def update_windows_certificates(self):
""" Update the root Windows certificates"""
try:
cert_file = os.path.join(self.persistent_work_dir, 'root_certs.sst')
if not os.path.isdir(self.persistent_work_dir):
os.makedirs(self.persistent_work_dir)
needs_update = True
if os.path.isfile(cert_file):
days = (time.time() - os.path.getmtime(cert_file)) / 86400
if days < 5:
needs_update = False
if needs_update:
logging.debug("Updating Windows root certificates...")
if os.path.isfile(cert_file):
os.unlink(cert_file)
from internal.os_util import run_elevated
run_elevated('certutil.exe', '-generateSSTFromWU "{0}"'.format(cert_file))
if os.path.isfile(cert_file):
run_elevated('certutil.exe', '-addstore -f Root "{0}"'.format(cert_file))
except Exception:
pass
def requires(self, module, module_name=None):
"""Try importing a module and installing it if it isn't available"""
ret = False
if module_name is None:
module_name = module
try:
__import__(module)
ret = True
except ImportError:
pass
if not ret:
from internal.os_util import run_elevated
logging.debug('Trying to install %s...', module_name)
run_elevated(sys.executable, '-m pip install {0}'.format(module_name))
try:
__import__(module)
ret = True
except ImportError:
pass
if not ret:
print "Missing {0} module. Please run 'pip install {1}'".format(module, module_name)
return ret
def find_browsers():
"""Find the various known-browsers in case they are not explicitly configured"""
browsers = parse_ini(os.path.join(os.path.dirname(__file__), "browsers.ini"))
if browsers is None:
browsers = {}
plat = platform.system()
if plat == "Windows":
local_appdata = os.getenv('LOCALAPPDATA')
program_files = str(os.getenv('ProgramFiles'))
program_files_x86 = str(os.getenv('ProgramFiles(x86)'))
# Allow 32-bit python to detect 64-bit browser installs
if program_files == program_files_x86 and program_files.find(' (x86)') >= 0:
program_files = program_files.replace(' (x86)', '')
# Chrome
paths = [program_files, program_files_x86, local_appdata]
channels = ['Chrome', 'Chrome Beta', 'Chrome Dev']
for channel in channels:
for path in paths:
if path is not None and channel not in browsers:
chrome_path = os.path.join(path, 'Google', channel,
'Application', 'chrome.exe')
if os.path.isfile(chrome_path):
browsers[channel] = {'exe': chrome_path}
if local_appdata is not None and 'Canary' not in browsers:
canary_path = os.path.join(local_appdata, 'Google', 'Chrome SxS',
'Application', 'chrome.exe')
if os.path.isfile(canary_path):
browsers['Canary'] = {'exe': canary_path}
browsers['Chrome Canary'] = {'exe': canary_path}
# Fall back to Chrome dev for Canary if Canary isn't available but dev is
if 'Chrome Dev' in browsers and 'Canary' not in browsers:
browsers['Chrome Canary'] = dict(browsers['Chrome Dev'])
browsers['Canary'] = dict(browsers['Chrome Dev'])
# Opera (same engine as Chrome)
paths = [program_files, program_files_x86]
channels = ['Opera', 'Opera beta', 'Opera developer']
for channel in channels:
for path in paths:
if path is not None and channel not in browsers:
opera_path = os.path.join(path, channel, 'launcher.exe')
if os.path.isfile(opera_path):
browsers[channel] = {'exe': opera_path, 'other_exes': ['opera.exe']}
# Firefox browsers
paths = [program_files, program_files_x86]
for path in paths:
if path is not None and 'Firefox' not in browsers:
firefox_path = os.path.join(path, 'Mozilla Firefox', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'}
if path is not None and 'Firefox' not in browsers:
firefox_path = os.path.join(path, 'Firefox', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'}
if path is not None and 'Firefox ESR' not in browsers:
firefox_path = os.path.join(path, 'Mozilla Firefox ESR', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox ESR'] = {'exe': firefox_path, 'type': 'Firefox'}
if path is not None and 'Firefox Beta' not in browsers:
firefox_path = os.path.join(path, 'Mozilla Firefox Beta', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox Beta'] = {'exe': firefox_path, 'type': 'Firefox'}
if path is not None and 'Firefox Beta' not in browsers:
firefox_path = os.path.join(path, 'Firefox Beta', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox Beta'] = {'exe': firefox_path, 'type': 'Firefox'}
if path is not None and 'Firefox Dev' not in browsers:
firefox_path = os.path.join(path, 'Mozilla Firefox Dev', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox Dev'] = {'exe': firefox_path, 'type': 'Firefox'}
if path is not None and 'Firefox Dev' not in browsers:
firefox_path = os.path.join(path, 'Firefox Dev', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox Dev'] = {'exe': firefox_path, 'type': 'Firefox'}
if path is not None and 'Firefox Nightly' not in browsers:
firefox_path = os.path.join(path, 'Nightly', 'firefox.exe')
if os.path.isfile(firefox_path):
browsers['Firefox Nightly'] = {'exe': firefox_path,
'type': 'Firefox',
'log_level': 5}
# Microsoft Edge (Legacy)
edge = None
build = get_windows_build()
if build >= 10240:
edge_exe = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'internal',
'support', 'edge', 'current', 'MicrosoftWebDriver.exe')
if not os.path.isfile(edge_exe):
if build > 17134:
from internal.os_util import run_elevated
logging.debug('Installing latest webdriver for Microsoft Edge...')
run_elevated('DISM.exe', '/Online /Add-Capability '
'/CapabilityName:Microsoft.WebDriver~~~~0.0.1.0')
edge_exe = os.path.join(os.environ['windir'], 'System32',
'MicrosoftWebDriver.exe')
else:
if build >= 17000:
edge_version = 17
elif build >= 16000:
edge_version = 16
elif build >= 15000:
edge_version = 15
elif build >= 14000:
edge_version = 14
elif build >= 10586:
edge_version = 13
else:
edge_version = 12
edge_exe = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'internal',
'support', 'edge', str(edge_version),
'MicrosoftWebDriver.exe')
if os.path.isfile(edge_exe):
edge = {'exe': edge_exe}
if edge is not None:
edge['type'] = 'Edge'
if 'Microsoft Edge (EdgeHTML)' not in browsers:
browsers['Microsoft Edge (EdgeHTML)'] = dict(edge)
if 'Microsoft Edge' not in browsers:
browsers['Microsoft Edge'] = dict(edge)
if 'Edge' not in browsers:
browsers['Edge'] = dict(edge)
# Microsoft Edge (Chromium)
paths = [program_files, program_files_x86, local_appdata]
channels = ['Edge Dev']
for channel in channels:
for path in paths:
if path is not None and channel not in browsers:
edge_path = os.path.join(path, 'Microsoft', channel,
'Application', 'msedge.exe')
if os.path.isfile(edge_path):
browser_name = 'Microsoft {0} (Chromium)'.format(channel)
if browser_name not in browsers:
browsers[browser_name] = {'exe': edge_path}
if local_appdata is not None and 'Microsoft Edge Canary (Chromium)' not in browsers:
edge_path = os.path.join(local_appdata, 'Microsoft', 'Edge SxS',
'Application', 'msedge.exe')
if os.path.isfile(edge_path):
browsers['Microsoft Edge Canary (Chromium)'] = {'exe': edge_path}
# Internet Explorer
paths = [program_files, program_files_x86]
for path in paths:
if path is not None and 'IE' not in browsers:
ie_path = os.path.join(path, 'Internet Explorer', 'iexplore.exe')
if os.path.isfile(ie_path):
browsers['ie'] = {'exe': ie_path, 'type': 'IE'}
elif plat == "Linux":
chrome_path = '/opt/google/chrome/chrome'
if 'Chrome' not in browsers and os.path.isfile(chrome_path):
browsers['Chrome'] = {'exe': chrome_path}
beta_path = '/opt/google/chrome-beta/chrome'
if 'Chrome Beta' not in browsers and os.path.isfile(beta_path):
browsers['Chrome Beta'] = {'exe': beta_path}
# google-chrome-unstable is the closest thing to Canary for Linux
canary_path = '/opt/google/chrome-unstable/chrome'
if os.path.isfile(canary_path):
if 'Chrome Dev' not in browsers:
browsers['Chrome Dev'] = {'exe': canary_path}
if 'Chrome Canary' not in browsers:
browsers['Chrome Canary'] = {'exe': canary_path}
if 'Canary' not in browsers:
browsers['Canary'] = {'exe': canary_path}
# Chromium
chromium_path = '/usr/lib/chromium-browser/chromium-browser'
if 'Chromium' not in browsers and os.path.isfile(chromium_path):
browsers['Chromium'] = {'exe': chromium_path}
if 'Chrome' not in browsers and os.path.isfile(chromium_path):
browsers['Chrome'] = {'exe': chromium_path}
chromium_path = '/usr/bin/chromium-browser'
if 'Chromium' not in browsers and os.path.isfile(chromium_path):
browsers['Chromium'] = {'exe': chromium_path}
if 'Chrome' not in browsers and os.path.isfile(chromium_path):
browsers['Chrome'] = {'exe': chromium_path}
# Opera
opera_path = '/usr/lib/x86_64-linux-gnu/opera/opera'
if 'Opera' not in browsers and os.path.isfile(opera_path):
browsers['Opera'] = {'exe': opera_path}
opera_path = '/usr/lib64/opera/opera'
if 'Opera' not in browsers and os.path.isfile(opera_path):
browsers['Opera'] = {'exe': opera_path}
beta_path = '/usr/lib/x86_64-linux-gnu/opera-beta/opera-beta'
if 'Opera beta' not in browsers and os.path.isfile(beta_path):
browsers['Opera beta'] = {'exe': beta_path}
beta_path = '/usr/lib64/opera-beta/opera-beta'
if 'Opera beta' not in browsers and os.path.isfile(beta_path):
browsers['Opera beta'] = {'exe': beta_path}
dev_path = '/usr/lib/x86_64-linux-gnu/opera-developer/opera-developer'
if 'Opera developer' not in browsers and os.path.isfile(dev_path):
browsers['Opera developer'] = {'exe': dev_path}
dev_path = '/usr/lib64/opera-developer/opera-developer'
if 'Opera developer' not in browsers and os.path.isfile(dev_path):
browsers['Opera developer'] = {'exe': dev_path}
# Firefox browsers
firefox_path = '/usr/lib/firefox/firefox'
if 'Firefox' not in browsers and os.path.isfile(firefox_path):
browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'}
firefox_path = '/usr/bin/firefox'
if 'Firefox' not in browsers and os.path.isfile(firefox_path):
browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'}
firefox_path = '/usr/lib/firefox-esr/firefox-esr'
if 'Firefox' not in browsers and os.path.isfile(firefox_path):
browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'}
if 'Firefox ESR' not in browsers and os.path.isfile(firefox_path):
browsers['Firefox ESR'] = {'exe': firefox_path, 'type': 'Firefox'}
nightly_path = '/usr/lib/firefox-trunk/firefox-trunk'
if 'Firefox Nightly' not in browsers and os.path.isfile(nightly_path):
browsers['Firefox Nightly'] = {'exe': nightly_path,
'type': 'Firefox',
'log_level': 5}
nightly_path = '/usr/bin/firefox-trunk'
if 'Firefox Nightly' not in browsers and os.path.isfile(nightly_path):
browsers['Firefox Nightly'] = {'exe': nightly_path,
'type': 'Firefox',
'log_level': 5}
elif plat == "Darwin":
chrome_path = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
if 'Chrome' not in browsers and os.path.isfile(chrome_path):
browsers['Chrome'] = {'exe': chrome_path}
canary_path = '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'
if os.path.isfile(canary_path):
if 'Chrome Dev' not in browsers:
browsers['Chrome Dev'] = {'exe': canary_path}
if 'Chrome Canary' not in browsers:
browsers['Chrome Canary'] = {'exe': canary_path}
if 'Canary' not in browsers:
browsers['Canary'] = {'exe': canary_path}
firefox_path = '/Applications/Firefox.app/Contents/MacOS/firefox'
if 'Firefox' not in browsers and os.path.isfile(firefox_path):
browsers['Firefox'] = {'exe': firefox_path, 'type': 'Firefox'}
nightly_path = '/Applications/FirefoxNightly.app/Contents/MacOS/firefox'
if 'Firefox Nightly' not in browsers and os.path.isfile(nightly_path):
browsers['Firefox Nightly'] = {'exe': nightly_path,
'type': 'Firefox',
'log_level': 5}
logging.debug('Detected Browsers:')
for browser in browsers:
logging.debug('%s: %s', browser, browsers[browser]['exe'])
if 'Firefox' in browsers:
try:
# make sure marionette is up to date
from internal.os_util import run_elevated
run_elevated(sys.executable, '-m pip install --upgrade marionette_driver')
except Exception:
pass
return browsers
def startup(self):
"""Validate that all of the external dependencies are installed"""
ret = True
# default /tmp/wptagent as an alive file on Linux
if self.options.alive is None:
if platform.system() == "Linux":
self.options.alive = '/tmp/wptagent'
else:
self.options.alive = os.path.join(os.path.dirname(__file__), 'wptagent.alive')
self.alive()
ret = self.requires('dns', 'dnspython') and ret
ret = self.requires('monotonic') and ret
ret = self.requires('PIL', 'pillow') and ret
ret = self.requires('psutil') and ret
ret = self.requires('requests') and ret
if not self.options.android and not self.options.iOS:
ret = self.requires('tornado') and ret
# Windows-specific imports
if platform.system() == "Windows":
ret = self.requires('win32api', 'pywin32') and ret
# Optional imports
self.requires('brotli')
self.requires('fontTools', 'fonttools')
# Try patching ws4py with a faster lib
try:
self.requires('wsaccel')
import wsaccel
wsaccel.patch_ws4py()
except Exception:
pass
try:
subprocess.check_output(['python', '--version'])
except Exception:
print "Make sure python 2.7 is available in the path."
ret = False
try:
subprocess.check_output('{0} -version'.format(self.image_magick['convert']), shell=True)
except Exception:
print "Missing convert utility. Please install ImageMagick " \
"and make sure it is in the path."
ret = False
try:
subprocess.check_output('{0} -version'.format(self.image_magick['mogrify']), shell=True)
except Exception:
print "Missing mogrify utility. Please install ImageMagick " \
"and make sure it is in the path."
ret = False
if platform.system() == "Linux":
try:
subprocess.check_output(['traceroute', '--version'])
except Exception:
logging.debug("Traceroute is missing, installing...")
subprocess.call(['sudo', 'apt-get', '-yq', 'install', 'traceroute'])
# If we are on Linux and there is no display, enable xvfb by default
if platform.system() == "Linux" and not self.options.android and \
not self.options.iOS and 'DISPLAY' not in os.environ:
self.options.xvfb = True
if self.options.xvfb:
ret = self.requires('xvfbwrapper') and ret
if ret:
from xvfbwrapper import Xvfb
self.xvfb = Xvfb(width=1920, height=1200, colordepth=24)
self.xvfb.start()
# Figure out which display to capture from
if platform.system() == "Linux" and 'DISPLAY' in os.environ:
logging.debug('Display: %s', os.environ['DISPLAY'])
self.capture_display = os.environ['DISPLAY']
elif platform.system() == "Darwin":
proc = subprocess.Popen('ffmpeg -f avfoundation -list_devices true -i ""',
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
_, err = proc.communicate()
for line in err.splitlines():
matches = re.search(r'\[(\d+)\] Capture screen', line)
if matches:
self.capture_display = matches.group(1)
break
elif platform.system() == "Windows":
self.capture_display = 'desktop'
if self.options.throttle:
try:
subprocess.check_output('sudo cgset -h', shell=True)
except Exception:
print "Missing cgroups, make sure cgroup-tools is installed."
ret = False
# Fix Lighthouse install permissions
if platform.system() != "Windows":
from internal.os_util import run_elevated
run_elevated('chmod', '-R 777 ~/.config/configstore/')
try:
import getpass
run_elevated('chown', '-R {0}:{0} ~/.config'.format(getpass.getuser()))
except Exception:
pass
# Check for Node 10+
if self.get_node_version() < 10.0:
if platform.system() == "Linux":
# This only works on debian-based systems
logging.debug('Updating Node.js to 10.x')
subprocess.call('curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash -',
shell=True)
subprocess.call(['sudo', 'apt-get', 'install', '-y', 'nodejs'])
if self.get_node_version() < 10.0:
logging.warning("Node.js 10 or newer is required for Lighthouse testing")
# Check the iOS install
if self.ios is not None:
ret = self.ios.check_install()
if not self.options.android and not self.options.iOS and not self.options.noidle:
self.wait_for_idle(300)
if self.adb is not None:
if not self.adb.start():
print "Error configuring adb. Make sure it is installed and in the path."
ret = False
self.shaper.remove()
if not self.shaper.install():
if platform.system() == "Windows":
print "Error configuring traffic shaping, make sure secure boot is disabled."
else:
print "Error configuring traffic shaping, make sure it is installed."
ret = False
# Update the Windows root certs
if platform.system() == "Windows":
self.update_windows_certificates()
return ret
def startup(self):
"""Validate that all of the external dependencies are installed"""
ret = True
self.alive()
ret = self.requires('dns', 'dnspython') and ret
ret = self.requires('monotonic') and ret
ret = self.requires('PIL', 'pillow') and ret
ret = self.requires('psutil') and ret
ret = self.requires('requests') and ret
if not self.options.android and not self.options.iOS:
ret = self.requires('tornado') and ret
# Windows-specific imports
if platform.system() == "Windows":
ret = self.requires('win32api', 'pypiwin32') and ret
try:
subprocess.check_output(['python', '--version'])
except Exception:
print "Make sure python 2.7 is available in the path."
ret = False
try:
subprocess.check_output('{0} -version'.format(
self.image_magick['convert']),
shell=True)
except Exception:
print "Missing convert utility. Please install ImageMagick " \
"and make sure it is in the path."
ret = False
try:
subprocess.check_output('{0} -version'.format(
self.image_magick['mogrify']),
shell=True)
except Exception:
print "Missing mogrify utility. Please install ImageMagick " \
"and make sure it is in the path."
ret = False
if platform.system() == "Linux":
try:
subprocess.check_output(['traceroute', '--version'])
except Exception:
logging.debug("Traceroute is missing, installing...")
subprocess.call(
['sudo', 'apt-get', '-yq', 'install', 'traceroute'])
# if we are on Linux and there is no display, enable xvfb by default
if platform.system() == "Linux" and not self.options.android and \
not self.options.iOS and 'DISPLAY' not in os.environ:
self.options.xvfb = True
if self.options.xvfb:
ret = self.requires('xvfbwrapper') and ret
if ret:
from xvfbwrapper import Xvfb
self.xvfb = Xvfb(width=1920, height=1200, colordepth=24)
self.xvfb.start()
# Figure out which display to capture from
if platform.system() == "Linux" and 'DISPLAY' in os.environ:
logging.debug('Display: %s', os.environ['DISPLAY'])
self.capture_display = os.environ['DISPLAY']
elif platform.system() == "Darwin":
proc = subprocess.Popen(
'ffmpeg -f avfoundation -list_devices true -i ""',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
_, err = proc.communicate()
for line in err.splitlines():
matches = re.search(r'\[(\d+)\] Capture screen', line)
if matches:
self.capture_display = matches.group(1)
break
elif platform.system() == "Windows":
self.capture_display = 'desktop'
if self.options.throttle:
try:
subprocess.check_output('sudo cgset -h', shell=True)
except Exception:
print "Missing cgroups, make sure cgroup-tools is installed."
ret = False
# Fix Lighthouse install permissions
if platform.system() != "Windows":
from internal.os_util import run_elevated
run_elevated('chmod', '-R 777 ~/.config/configstore/')
# Check the iOS install
if self.ios is not None:
ret = self.ios.check_install()
if not self.options.android and not self.options.iOS:
self.wait_for_idle(300)
if self.adb is not None:
if not self.adb.start():
print "Error configuring adb. Make sure it is installed and in the path."
ret = False
self.shaper.remove()
if not self.shaper.install():
if platform.system() == "Windows":
print "Error configuring traffic shaping, make sure secure boot is disabled."
else:
print "Error configuring traffic shaping, make sure it is installed."
ret = False
return ret
