def GetChromeInfo(args): """Finds chrome either locally or remotely and returns path and version info. Version is not reported if local chrome is used. """ if args.use_local_chrome: chrome_path = GetLocalChromePath(args.chrome_path) if not chrome_path: raise ChromeNotFound('Could not find chrome locally. You can supply it ' 'manually using --chrome_path') return ChromeInfo(path=chrome_path, version=None) else: channel = args.channel if sys.platform == 'linux2' and channel == 'canary': channel = 'dev' assert channel in ['stable', 'beta', 'dev', 'canary'] binary = 'chrome' print('Fetching the', channel, binary, 'binary via the binary_manager.') chrome_manager = binary_manager.BinaryManager([CHROME_BINARIES_CONFIG]) os_name, arch = dependency_util.GetOSAndArchForCurrentDesktopPlatform() chrome_path, version = chrome_manager.FetchPathWithVersion( '%s_%s' % (binary, channel), os_name, arch) print('Finished fetching the', binary, 'binary to', chrome_path) return ChromeInfo(path=chrome_path, version=version)
def __init__(self): self.bm = binary_manager.BinaryManager([_NodeBinariesConfigPath()]) self.os_name = dependency_util.GetOSNameForCurrentDesktopPlatform() self.arch_name = dependency_util.GetArchForCurrentDesktopPlatform( self.os_name) self.node_path = self.bm.FetchPath('node', self.os_name, self.arch_name) self.npm_path = self.bm.FetchPath('npm', self.os_name, self.arch_name) self.node_initialized = False
def _GetDownloader(self): """Gets the downloader used to download wpr_go binary from GCS.""" if ReplayServer._go_binary_path: # If the _go_binary_path was already set, then no need to use downloader # to download via binary_manager. self._downloader = None elif not self._downloader: configs = [CHROME_BINARY_CONFIG, TELEMETRY_PROJECT_CONFIG] self._downloader = binary_manager.BinaryManager(configs).FetchPath return self._downloader
def _FetchReferenceBrowserBinary(platform): os_name = platform.GetOSName() arch_name = platform.GetArchName() manager = binary_manager.BinaryManager([CHROME_BINARY_CONFIG]) if os_name == 'android': os_version = dependency_util.GetChromeApkOsVersion( platform.GetOSVersionName()) manager.FetchPath('chrome_stable', os_name, arch_name, os_version) else: manager.FetchPath('reference_build', os_name, arch_name)
def testFailedFetchPathMissingDep(self): manager = binary_manager.BinaryManager([self.base_config]) with self.assertRaises(exceptions.NoPathFoundError): manager.FetchPath('missing_dep', 'linux', 'x86_64') with self.assertRaises(exceptions.NoPathFoundError): manager.FetchPath('missing_dep', 'android', 'x86', 'l') with self.assertRaises(exceptions.NoPathFoundError): manager.FetchPath('dep_1', 'linux', 'bad_arch') with self.assertRaises(exceptions.NoPathFoundError): manager.FetchPath('dep_1', 'bad_os', 'x86')
def __init__(self): self.bm = binary_manager.BinaryManager( [_NodeBinariesConfigPath()]) self.os_name = dependency_util.GetOSNameForCurrentDesktopPlatform() self.arch_name = dependency_util.GetArchForCurrentDesktopPlatform( self.os_name) self.node_path = self.bm.FetchPath('node', self.os_name, self.arch_name) self.npm_path = os.path.abspath(os.path.join( os.path.dirname(self.node_path), '..', 'lib', 'node_modules', 'npm', 'bin', 'npm-cli.js')) self.node_initialized = False
def InitDependencyManager(client_configs): if GetBinaryManager(): raise exceptions.InitializationError( 'Trying to re-initialize the binary manager with config %s' % client_configs) configs = [] if client_configs: configs += client_configs configs += [TELEMETRY_PROJECT_CONFIG, CHROME_BINARY_CONFIG] SetBinaryManager(binary_manager.BinaryManager(configs)) devil_env.config.Initialize()
def RunWct(base_dir, dep_dirs, persist=False, chrome_channel='stable', prefix=''): wct_bin = os.environ.get('WCT', 'wct') if os.system('which %s > /dev/null' % wct_bin): print 'FATAL ERROR: wct not found. Install it and add it to your path:' print 'cipd install -root ~/cipd infra/testing/wct/linux-amd64 prod' print 'export PATH=~/cipd:$PATH' return 1 xvfb_process = None chrome_bin = run_dev_server_tests.GetLocalChromePath(None) if not chrome_bin: chrome_manager = binary_manager.BinaryManager( [run_dev_server_tests.CHROME_BINARIES_CONFIG]) arch, os_name = dependency_util.GetOSAndArchForCurrentDesktopPlatform() chrome_bin = chrome_manager.FetchPathWithVersion( 'chrome_%s' % chrome_channel, arch, os_name)[0] if not chrome_bin or os.system('which %s > /dev/null' % chrome_bin): print 'FATAL ERROR: chrome not found.' return 1 if xvfb.ShouldStartXvfb(): print 'Starting xvfb...' xvfb_process = xvfb.StartXvfb() user_data_dir = tempfile.mkdtemp() command = [wct_bin] command += ['-chrome', chrome_bin] command += ['-dir', user_data_dir] command += ['-base', base_dir] command += ['-prefix', prefix] if persist: command += ['-persist'] for dep in dep_dirs: command += ['-dep', dep] logging.info('Starting WCT: %r', command) try: return subprocess.call(command) finally: if xvfb_process: xvfb_process.kill() try: shutil.rmtree(user_data_dir) except OSError as e: logging.error('Error cleaning up temp dir %r: %s', user_data_dir, e)
def InitDependencyManager(client_configs): global _binary_manager # pylint: disable=global-statement if _binary_manager: raise exceptions.InitializationError( 'Trying to re-initialize the binary manager with config %s' % client_configs) configs = [] if client_configs: configs += client_configs configs += [TELEMETRY_PROJECT_CONFIG, CHROME_BINARY_CONFIG] _binary_manager = binary_manager.BinaryManager(configs) devil_env.config.Initialize()
def _AddMissingURLsToArchive(self, replay_out_file): existing_wpr = self._ExistingWpr() if not existing_wpr: return missing_urls = _ExtractMissingURLsFromLog(replay_out_file) if not missing_urls: return if not self.wpr_go_bin: self.wpr_go_bin = ( binary_manager.BinaryManager([TELEMETRY_BIN_DEPS_CONFIG]).FetchPath( 'wpr_go', py_utils.GetHostArchName(), py_utils.GetHostOsName())) subprocess.check_call([self.wpr_go_bin, 'add', existing_wpr] + missing_urls)
def __init__(self, archive_path, replay_host, http_port, https_port, replay_options, binary_downloader=None): """Initialize ReplayServer. Args: archive_path: a path to a specific WPR archive. replay_host: the hostname to serve traffic. http_port: an integer port on which to serve HTTP traffic. May be zero to let the OS choose an available port. https_port: an integer port on which to serve HTTPS traffic. May be zero to let the OS choose an available port. replay_options: an iterable of options strings to forward to replay.py. binary_downloader: a function to be used to fetch binary. May be None to use py_utils.binary_manager.FetchPath as default downloader. """ self.archive_path = archive_path self._replay_host = replay_host self._started_ports = {} # a dict such as {'http': 80, 'https': 443} # A temporary path for storing stdout & stderr of the webpagereplay # subprocess. self._temp_log_file_path = None # Assign the downloader func and binary_manager downloader = None if binary_downloader: downloader = binary_downloader else: configs = [CHROME_BINARY_CONFIG, TELEMETRY_PROJECT_CONFIG] downloader = binary_manager.BinaryManager(configs).FetchPath self._cmd_line = self._GetCommandLine( self._GetGoBinaryPath(downloader=downloader), http_port, https_port, replay_options, archive_path) if RECORD in replay_options or 'record' in replay_options: self._AssertPathExists('archive directory', os.path.dirname(self.archive_path)) elif not os.path.exists(self.archive_path): self._AssertPathExists('archive file', self.archive_path) self.replay_process = None
def testSuccessfulFetchPathFallbackToNoOsVersion(self): manager = binary_manager.BinaryManager([self.base_config]) found_path = manager.FetchPath('dep_2', 'linux', 'x86_64', 'fake_version') self.assertEqual(self.expected_dep2_linux_file, found_path)
def testInitializationWithConfig(self): with self.assertRaises(ValueError): manager = binary_manager.BinaryManager(self.base_config) manager = binary_manager.BinaryManager([self.base_config]) self.assertItemsEqual(self.expected_dependencies, manager._dependency_manager._lookup_dict)
def testInitializationMissingConfig(self): with self.assertRaises(ValueError): binary_manager.BinaryManager(os.path.join('missing', 'path'))
def testInitializationNoConfig(self): with self.assertRaises(ValueError): binary_manager.BinaryManager(None)
def GetWPRArchiveFromConfig(test_name, config): """Downloads the wpr archive from given config and test name.""" return binary_manager.BinaryManager(config).FetchPath( test_name, py_utils.GetHostOsName(), py_utils.GetHostArchName())
def testSuccessfulLocalPathNoOsVersion(self): manager = binary_manager.BinaryManager([self.base_config]) found_path = manager.LocalPath('dep_2', 'linux', 'x86_64') self.assertEqual(self.expected_dep2_linux_file, found_path)
def testSuccessfulLocalPathOsVersion(self): manager = binary_manager.BinaryManager([self.base_config]) found_path = manager.LocalPath('dep_2', 'android', 'x86', 'l') self.assertEqual(self.expected_dep2_android_file, found_path)
def Main(argv): try: parser = argparse.ArgumentParser( description='Run dev_server tests for a project.') parser.add_argument('--chrome_path', type=str, help='Path to Chrome browser binary.') parser.add_argument('--no-use-local-chrome', dest='use_local_chrome', action='store_false') parser.add_argument('--no-install-hooks', dest='install_hooks', action='store_false') parser.add_argument( '--tests', type=str, help='Set of tests to run (tracing or perf_insights)') parser.add_argument('--channel', type=str, default='stable', help='Chrome channel to run (stable or canary)') parser.add_argument('--presentation-json', type=str, help='Recipe presentation-json output file path') parser.set_defaults(install_hooks=True) parser.set_defaults(use_local_chrome=True) args = parser.parse_args(argv[1:]) if args.install_hooks: install.InstallHooks() user_data_dir = tempfile.mkdtemp() tmpdir = None xvfb_process = None server_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'bin', 'run_dev_server') # TODO(anniesullie): Make OS selection of port work on Windows. See #1235. if sys.platform == 'win32': port = DEFAULT_PORT else: port = '0' server_command = [server_path, '--no-install-hooks', '--port', port] if sys.platform.startswith('win'): server_command = ['python.exe'] + server_command print "Starting dev_server..." server_process = subprocess.Popen(server_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1) time.sleep(1) if sys.platform != 'win32': output = server_process.stderr.readline() port = re.search(r'Now running on http://127.0.0.1:([\d]+)', output).group(1) chrome_info = None if args.use_local_chrome: chrome_path = GetLocalChromePath(args.chrome_path) if not chrome_path: logging.error('Could not find path to chrome.') sys.exit(1) chrome_info = 'with command `%s`' % chrome_path else: channel = args.channel if sys.platform == 'linux2' and channel == 'canary': channel = 'dev' assert channel in ['stable', 'beta', 'dev', 'canary'] print 'Fetching the %s chrome binary via the binary_manager.' % channel chrome_manager = binary_manager.BinaryManager( [CHROME_BINARIES_CONFIG]) arch, os_name = dependency_util.GetOSAndArchForCurrentDesktopPlatform( ) chrome_path, version = chrome_manager.FetchPathWithVersion( 'chrome_%s' % channel, arch, os_name) if xvfb.ShouldStartXvfb(): xvfb_process = xvfb.StartXvfb() chrome_info = 'version %s from channel %s' % (version, channel) chrome_command = [ chrome_path, '--user-data-dir=%s' % user_data_dir, '--no-sandbox', '--no-experiments', '--no-first-run', '--noerrdialogs', '--window-size=1280,1024', ('http://localhost:%s/%s/tests.html?' % (port, args.tests)) + 'headless=true&testTypeToRun=all', ] print "Starting Chrome %s..." % chrome_info chrome_process = subprocess.Popen(chrome_command, stdout=sys.stdout, stderr=sys.stderr) print 'chrome process command: %s' % ' '.join(chrome_command) print "Waiting for tests to finish..." server_out, server_err = server_process.communicate() print "Killing Chrome..." if sys.platform == 'win32': # Use taskkill on Windows to make sure Chrome and all subprocesses are # killed. subprocess.call( ['taskkill', '/F', '/T', '/PID', str(chrome_process.pid)]) else: chrome_process.kill() if server_process.returncode != 0: logging.error('Tests failed!') logging.error('Server stderr:') logging.error(server_err) logging.error('Server stdout:') logging.error(server_out) else: print server_out if args.presentation_json: with open(args.presentation_json, 'w') as recipe_out: # Add a link to the buildbot status for the step saying which version # of Chrome the test ran on. The actual linking feature is not used, # but there isn't a way to just add text. link_name = 'Chrome Version %s' % version presentation_info = {'links': {link_name: CHROME_CONFIG_URL}} json.dump(presentation_info, recipe_out) finally: # Wait for Chrome to be killed before deleting temp Chrome dir. Only have # this timing issue on Windows. if sys.platform == 'win32': time.sleep(5) if tmpdir: try: shutil.rmtree(tmpdir) shutil.rmtree(user_data_dir) except OSError as e: logging.error('Error cleaning up temp dirs %s and %s: %s', tmpdir, user_data_dir, e) if xvfb_process: xvfb_process.kill() sys.exit(server_process.returncode)