def SetupWindowsCrossCompileToolchain(target_arch): # First retrieve various MSVC and Windows SDK paths. output = subprocess.check_output([ os.path.join(CHROMIUM_ROOT_DIR, 'build', 'vs_toolchain.py'), 'get_toolchain_dir' ]) new_args = [ '--enable-cross-compile', '--cc=clang-cl', '--ld=lld-link', '--nm=llvm-nm', '--ar=llvm-ar', # Separate from optflags because configure strips it from msvc builds... '--extra-cflags=-O2', ] if target_arch == 'ia32': new_args += ['--extra-cflags=-m32'] if target_arch == 'ia32': target_arch = 'x86' # Turn this into a dictionary. win_dirs = gn_helpers.FromGNArgs(output) # Use those paths with a second script which will tell us the proper include # and lib paths to specify for cflags and ldflags respectively. output = subprocess.check_output([ 'python', os.path.join(CHROMIUM_ROOT_DIR, 'build', 'toolchain', 'win', 'setup_toolchain.py'), win_dirs['vs_path'], win_dirs['sdk_path'], win_dirs['runtime_dirs'], 'win', target_arch, 'none' ]) flags = gn_helpers.FromGNArgs(output) cwd = os.getcwd() for cflag in flags['include_flags_imsvc'].split(' '): # Apparently setup_toolchain prefers relative include paths, which # may work for chrome, but it does not work for ffmpeg, so let's make # them asbolute again. cflag = cflag.strip('"') if cflag.startswith("-imsvc"): cflag = "-imsvc" + os.path.join(cwd, cflag[6:]) new_args += ['--extra-cflags=' + cflag] # TODO(dalecurtis): Why isn't the ucrt path printed? flags['vc_lib_ucrt_path'] = flags['vc_lib_um_path'].replace( '/um/', '/ucrt/') # Unlike the cflags, the lib include paths are each in a separate variable. for k in flags: if 'lib' in k: new_args += ['--extra-ldflags=-libpath:' + flags[k]] return new_args
def SetupWindowsCrossCompileToolchain(target_arch): # First retrieve various MSVC and Windows SDK paths. output = subprocess.check_output([ os.path.join(CHROMIUM_ROOT_DIR, 'build', 'vs_toolchain.py'), 'get_toolchain_dir' ]) # Turn this into a dictionary. win_dirs = gn_helpers.FromGNArgs(output) # Use those paths with a second script which will tell us the proper include # and lib paths to specify for cflags and ldflags respectively. output = subprocess.check_output([ 'python', os.path.join(CHROMIUM_ROOT_DIR, 'build', 'toolchain', 'win', 'setup_toolchain.py'), win_dirs['vs_path'], win_dirs['sdk_path'], win_dirs['runtime_dirs'], 'win', target_arch, 'none' ]) flags = gn_helpers.FromGNArgs(output) cwd = os.getcwd() target_env = os.environ # Each path is of the form: # "/I../depot_tools/win_toolchain/vs_files/20d5f2553f/Windows Kits/10/Include/10.0.19041.0/winrt" # # Since there's a space in the include path, inputs are quoted in |flags|, we # can helpfully use shlex to split on spaces while preserving quoted strings. include_paths = [] for include_path in shlex.split(flags['include_flags_I']): # Apparently setup_toolchain prefers relative include paths, which # may work for chrome, but it does not work for dav1d, so let's make # them asbolute again. include_path = os.path.abspath(os.path.join(cwd, include_path[2:])) include_paths.append(include_path) # TODO(dalecurtis): Why isn't the ucrt path printed? flags['vc_lib_ucrt_path'] = flags['vc_lib_um_path'].replace( '/um/', '/ucrt/') # Unlike the cflags, the lib include paths are each in a separate variable. lib_paths = [] for k in flags: # libpath_flags is like cflags. Since it is also redundant, skip it. if 'lib' in k and k != 'libpath_flags': lib_paths.append(flags[k]) target_env = os.environ target_env['INCLUDE'] = ';'.join(include_paths) target_env['LIB'] = ';'.join(lib_paths) return target_env
def SetupWindowsCrossCompileToolchain(target_arch): # First retrieve various MSVC and Windows SDK paths. output = subprocess.check_output([ os.path.join(CHROMIUM_ROOT_DIR, 'build', 'vs_toolchain.py'), 'get_toolchain_dir' ]) # Turn this into a dictionary. win_dirs = gn_helpers.FromGNArgs(output) # Use those paths with a second script which will tell us the proper include # and lib paths to specify for cflags and ldflags respectively. output = subprocess.check_output([ 'python', os.path.join(CHROMIUM_ROOT_DIR, 'build', 'toolchain', 'win', 'setup_toolchain.py'), win_dirs['vs_path'], win_dirs['sdk_path'], win_dirs['runtime_dirs'], 'win', target_arch, 'none' ]) flags = gn_helpers.FromGNArgs(output) cwd = os.getcwd() target_env = os.environ include_paths = [] for cflag in flags['include_flags_imsvc'].split(' '): # Apparently setup_toolchain prefers relative include paths, which # may work for chrome, but it does not work for ffmpeg, so let's make # them asbolute again. include_path = cflag.strip('"') if include_path.startswith('-imsvc'): include_path = os.path.abspath(os.path.join(cwd, include_path[6:])) include_paths.append(include_path) # TODO(dalecurtis): Why isn't the ucrt path printed? flags['vc_lib_ucrt_path'] = flags['vc_lib_um_path'].replace( '/um/', '/ucrt/') # Unlike the cflags, the lib include paths are each in a separate variable. lib_paths = [] for k in flags: # libpath_flags is like cflags. Since it is also redundant, skip it. if 'lib' in k and k != 'libpath_flags': lib_paths.append(flags[k]) target_env = os.environ target_env['INCLUDE'] = ';'.join(include_paths) target_env['LIB'] = ';'.join(lib_paths) return target_env
def GNArgs(self, vals): if vals['cros_passthrough']: if not 'GN_ARGS' in os.environ: raise MBErr('MB is expecting GN_ARGS to be in the environment') gn_args = os.environ['GN_ARGS'] if not re.search('target_os.*=.*"chromeos"', gn_args): raise MBErr( 'GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' % gn_args) else: gn_args = vals['gn_args'] if self.args.goma_dir: gn_args += ' goma_dir="%s"' % self.args.goma_dir android_version_code = self.args.android_version_code if android_version_code: gn_args += ' android_default_version_code="%s"' % android_version_code android_version_name = self.args.android_version_name if android_version_name: gn_args += ' android_default_version_name="%s"' % android_version_name # Canonicalize the arg string into a sorted, newline-separated list # of key-value pairs, and de-dup the keys if need be so that only # the last instance of each arg is listed. gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args)) args_file = vals.get('args_file', None) if args_file: gn_args = ('import("%s")\n' % vals['args_file']) + gn_args return gn_args
def _BuildWithChromium(): """Returns value of gclient's |build_with_chromium|.""" gni_path = os.path.join(_BUILD_DIR, 'config', 'gclient_args.gni') with open(gni_path) as f: data = f.read() args = gn_helpers.FromGNArgs(data) return args['build_with_chromium']
def GNArgs(self, vals, expand_imports=False): if vals['cros_passthrough']: if not 'GN_ARGS' in os.environ: raise MBErr('MB is expecting GN_ARGS to be in the environment') gn_args = os.environ['GN_ARGS'] if not re.search('target_os.*=.*"chromeos"', gn_args): raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' % gn_args) else: gn_args = vals['gn_args'] if self.args.goma_dir: gn_args += ' goma_dir="%s"' % self.args.goma_dir android_version_code = self.args.android_version_code if android_version_code: gn_args += ' android_default_version_code="%s"' % android_version_code android_version_name = self.args.android_version_name if android_version_name: gn_args += ' android_default_version_name="%s"' % android_version_name args_gn_lines = [] parsed_gn_args = {} args_file = vals.get('args_file', None) if args_file: if expand_imports: content = self.ReadFile(self.ToAbsPath(args_file)) parsed_gn_args = gn_helpers.FromGNArgs(content) else: args_gn_lines.append('import("%s")' % args_file) # Canonicalize the arg string into a sorted, newline-separated list # of key-value pairs, and de-dup the keys if need be so that only # the last instance of each arg is listed. parsed_gn_args.update(gn_helpers.FromGNArgs(gn_args)) args_gn_lines.append(gn_helpers.ToGNString(parsed_gn_args)) return '\n'.join(args_gn_lines)
def test_FromGNArgs(self): # Booleans and numbers should work; whitespace is allowed works. self.assertEqual(gn_helpers.FromGNArgs('foo = true\nbar = 1\n'), {'foo': True, 'bar': 1}) # Whitespace is not required; strings should also work. self.assertEqual(gn_helpers.FromGNArgs('foo="bar baz"'), {'foo': 'bar baz'}) # Comments should work (and be ignored). gn_args_lines = [ '# Top-level comment.', 'foo = true', 'bar = 1 # In-line comment.', ] self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)), {'foo': True, 'bar': 1}) # Lists should work. self.assertEqual(gn_helpers.FromGNArgs('foo=[1, 2, 3]'), {'foo': [1, 2, 3]}) # Empty strings should return an empty dict. self.assertEqual(gn_helpers.FromGNArgs(''), {}) self.assertEqual(gn_helpers.FromGNArgs(' \n '), {}) # Non-identifiers should raise an exception. with self.assertRaises(gn_helpers.GNException): gn_helpers.FromGNArgs('123 = true') # References to other variables should raise an exception. with self.assertRaises(gn_helpers.GNException): gn_helpers.FromGNArgs('foo = bar') # References to functions should raise an exception. with self.assertRaises(gn_helpers.GNException): gn_helpers.FromGNArgs('foo = exec_script("//build/baz.py")') # Underscores in identifiers should work. self.assertEqual(gn_helpers.FromGNArgs('_foo = true'), {'_foo': True}) self.assertEqual(gn_helpers.FromGNArgs('foo_bar = true'), {'foo_bar': True}) self.assertEqual(gn_helpers.FromGNArgs('foo_=true'), {'foo_': True})
def main(): parser = argparse.ArgumentParser() parser.add_argument('--swarming-os', help='OS specifier for Swarming.') parser.add_argument('--target-os', default='detect', help='gn target_os') parser.add_argument('--arch', '-a', default='detect', help='CPU architecture of the test binary.') parser.add_argument('--build', dest='build', action='store_true', help='Build before isolating.') parser.add_argument('--no-build', dest='build', action='store_false', help='Do not build, just isolate (default).') parser.add_argument('--isolate-map-file', '-i', help='path to isolate map file if not using default') parser.add_argument('--copies', '-n', type=int, default=1, help='Number of copies to spawn.') parser.add_argument( '--device-os', help='Run tests on the given version of Android.') parser.add_argument( '--device-type', help='device_type specifier for Swarming' ' from https://chromium-swarm.appspot.com/botlist .') parser.add_argument('--pool', default='chromium.tests', help='Use the given swarming pool.') parser.add_argument('--results', '-r', default='results', help='Directory in which to store results.') parser.add_argument( '--gtest_filter', help='Deprecated. Pass as test runner arg instead, like \'-- ' '--gtest_filter="*#testFoo"\'') parser.add_argument( '--gtest_repeat', help='Deprecated. Pass as test runner arg instead, like \'-- ' '--gtest_repeat=99\'') parser.add_argument( '--test-launcher-shard-index', help='Shard index to run. Use with --test-launcher-total-shards.') parser.add_argument('--test-launcher-total-shards', help='Number of shards to split the test into. Use with' ' --test-launcher-shard-index.') parser.add_argument('--no-test-flags', action='store_true', help='Do not add --test-launcher-summary-output and ' '--system-log-file flags to the comment.') parser.add_argument('out_dir', type=str, help='Build directory.') parser.add_argument('target_name', type=str, help='Name of target to run.') parser.add_argument( 'runner_args', nargs='*', type=str, help='Arguments to pass to the test runner, e.g. gtest_filter and ' 'gtest_repeat.') args = parser.parse_intermixed_args() with open(os.path.join(args.out_dir, 'args.gn')) as f: gn_args = gn_helpers.FromGNArgs(f.read()) if args.target_os == 'detect': if 'target_os' in gn_args: args.target_os = gn_args['target_os'].strip('"') else: args.target_os = { 'darwin': 'mac', 'linux2': 'linux', 'win32': 'win' }[ sys.platform] if args.swarming_os is None: args.swarming_os = { 'mac': 'Mac', 'win': 'Windows', 'linux': 'Linux', 'android': 'Android', 'fuchsia': 'Linux' }[args.target_os] if args.target_os == 'win' and args.target_name.endswith('.exe'): # The machinery expects not to have a '.exe' suffix. args.target_name = os.path.splitext(args.target_name)[0] # Determine the CPU architecture of the test binary, if not specified. if args.arch == 'detect': if args.target_os not in ('android', 'mac', 'win'): executable_info = subprocess.check_output( ['file', os.path.join(args.out_dir, args.target_name)]) if 'ARM aarch64' in executable_info: args.arch = 'arm64', else: args.arch = 'x86-64' elif args.target_os == 'android': args.arch = gn_args.get('target_cpu', 'detect') # TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic # once mb.py is in python3 mb_cmd = ['tools/mb/mb', 'isolate'] if not args.build: mb_cmd.append('--no-build') if args.isolate_map_file: mb_cmd += ['--isolate-map-file', args.isolate_map_file] mb_cmd += ['//' + args.out_dir, args.target_name] subprocess.check_call(mb_cmd, shell=os.name == 'nt') print('If you get authentication errors, follow:') print( ' https://chromium.googlesource.com/chromium/src/+/HEAD/docs/workflow/debugging-with-swarming.md#authenticating' ) print('Uploading to isolate server, this can take a while...') isolate = os.path.join(args.out_dir, args.target_name + '.isolate') archive_json = os.path.join(args.out_dir, args.target_name + '.archive.json') subprocess.check_output([ 'tools/luci-go/isolate', 'archive', '-cas-instance', 'chromium-swarm', '-isolate', isolate, '-dump-json', archive_json ]) with open(archive_json) as f: cas_digest = json.load(f).get(args.target_name) # TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic # once mb.py is in python3 mb_cmd = ['tools/mb/mb', 'get-swarming-command', '--as-list'] if not args.build: mb_cmd.append('--no-build') if args.isolate_map_file: mb_cmd += ['--isolate-map-file', args.isolate_map_file] mb_cmd += ['//' + args.out_dir, args.target_name] mb_output = subprocess.check_output(mb_cmd, shell=os.name == 'nt') swarming_cmd = json.loads(mb_output) if os.path.isdir(args.results): shutil.rmtree(args.results) os.makedirs(args.results) try: print('Triggering %d tasks...' % args.copies) # Use dummy since threadpools give better exception messages # than process pools do, and threads work fine for what we're doing. pool = multiprocessing.dummy.Pool() spawn_args = [(i, args, cas_digest, swarming_cmd) for i in range(args.copies)] spawn_results = pool.imap_unordered(_Spawn, spawn_args) exit_codes = [] collect_results = pool.imap_unordered(_Collect, spawn_results) for result in collect_results: exit_codes.append(result) successes = sum(1 for x in exit_codes if x == 0) errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE) failures = len(exit_codes) - successes - errors clear_to_eol = '\033[K' print( '\r[%d/%d] collected: ' '%d successes, %d failures, %d bot errors...%s' % (len(exit_codes), args.copies, successes, failures, errors, clear_to_eol), end=' ') sys.stdout.flush() print() print('Results logs collected into', os.path.abspath(args.results) + '.') finally: pool.close() pool.join() return 0
def test_FromGNArgs(self): # Booleans and numbers should work; whitespace is allowed works. self.assertEqual(gn_helpers.FromGNArgs('foo = true\nbar = 1\n'), { 'foo': True, 'bar': 1 }) # Whitespace is not required; strings should also work. self.assertEqual(gn_helpers.FromGNArgs('foo="bar baz"'), {'foo': 'bar baz'}) # Comments should work (and be ignored). gn_args_lines = [ '# Top-level comment.', 'foo = true', 'bar = 1 # In-line comment followed by whitespace.', ' ', 'baz = false', ] self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)), { 'foo': True, 'bar': 1, 'baz': False }) # Lists should work. self.assertEqual(gn_helpers.FromGNArgs('foo=[1, 2, 3]'), {'foo': [1, 2, 3]}) # Empty strings should return an empty dict. self.assertEqual(gn_helpers.FromGNArgs(''), {}) self.assertEqual(gn_helpers.FromGNArgs(' \n '), {}) # Comments should work everywhere (and be ignored). gn_args_lines = [ '# Top-level comment.', '', '# Variable comment.', 'foo = true', 'bar = [', ' # Value comment in list.', ' 1,', ' 2,', ']', '', 'baz # Comment anywhere, really', ' = # also here', ' 4', ] self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)), { 'foo': True, 'bar': [1, 2], 'baz': 4 }) # Scope should be parsed, even empty ones. gn_args_lines = [ 'foo = {', ' a = 1', ' b = [', ' { },', ' {', ' c = 1', ' },', ' ]', '}', ] self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)), {'foo': { 'a': 1, 'b': [ {}, { 'c': 1, }, ] }}) # Non-identifiers should raise an exception. with self.assertRaises(gn_helpers.GNError): gn_helpers.FromGNArgs('123 = true') # References to other variables should raise an exception. with self.assertRaises(gn_helpers.GNError): gn_helpers.FromGNArgs('foo = bar') # References to functions should raise an exception. with self.assertRaises(gn_helpers.GNError): gn_helpers.FromGNArgs('foo = exec_script("//build/baz.py")') # Underscores in identifiers should work. self.assertEqual(gn_helpers.FromGNArgs('_foo = true'), {'_foo': True}) self.assertEqual(gn_helpers.FromGNArgs('foo_bar = true'), {'foo_bar': True}) self.assertEqual(gn_helpers.FromGNArgs('foo_=true'), {'foo_': True})