def SetupWindowsCrossCompileToolchain(target_arch):
# First retrieve various MSVC and Windows SDK paths.
output = subprocess.check_output([
os.path.join(CHROMIUM_ROOT_DIR, 'build', 'vs_toolchain.py'),
'get_toolchain_dir'
])
new_args = [
'--enable-cross-compile',
'--cc=clang-cl',
'--ld=lld-link',
'--nm=llvm-nm',
'--ar=llvm-ar',
# Separate from optflags because configure strips it from msvc builds...
'--extra-cflags=-O2',
]
if target_arch == 'ia32':
new_args += ['--extra-cflags=-m32']
if target_arch == 'ia32':
target_arch = 'x86'
# Turn this into a dictionary.
win_dirs = gn_helpers.FromGNArgs(output)
# Use those paths with a second script which will tell us the proper include
# and lib paths to specify for cflags and ldflags respectively.
output = subprocess.check_output([
'python',
os.path.join(CHROMIUM_ROOT_DIR, 'build', 'toolchain', 'win',
'setup_toolchain.py'), win_dirs['vs_path'],
win_dirs['sdk_path'], win_dirs['runtime_dirs'], 'win', target_arch,
'none'
])
flags = gn_helpers.FromGNArgs(output)
cwd = os.getcwd()
for cflag in flags['include_flags_imsvc'].split(' '):
# Apparently setup_toolchain prefers relative include paths, which
# may work for chrome, but it does not work for ffmpeg, so let's make
# them asbolute again.
cflag = cflag.strip('"')
if cflag.startswith("-imsvc"):
cflag = "-imsvc" + os.path.join(cwd, cflag[6:])
new_args += ['--extra-cflags=' + cflag]
# TODO(dalecurtis): Why isn't the ucrt path printed?
flags['vc_lib_ucrt_path'] = flags['vc_lib_um_path'].replace(
'/um/', '/ucrt/')
# Unlike the cflags, the lib include paths are each in a separate variable.
for k in flags:
if 'lib' in k:
new_args += ['--extra-ldflags=-libpath:' + flags[k]]
return new_args
def SetupWindowsCrossCompileToolchain(target_arch):
# First retrieve various MSVC and Windows SDK paths.
output = subprocess.check_output([
os.path.join(CHROMIUM_ROOT_DIR, 'build', 'vs_toolchain.py'),
'get_toolchain_dir'
])
# Turn this into a dictionary.
win_dirs = gn_helpers.FromGNArgs(output)
# Use those paths with a second script which will tell us the proper include
# and lib paths to specify for cflags and ldflags respectively.
output = subprocess.check_output([
'python',
os.path.join(CHROMIUM_ROOT_DIR, 'build', 'toolchain', 'win',
'setup_toolchain.py'), win_dirs['vs_path'],
win_dirs['sdk_path'], win_dirs['runtime_dirs'], 'win', target_arch,
'none'
])
flags = gn_helpers.FromGNArgs(output)
cwd = os.getcwd()
target_env = os.environ
# Each path is of the form:
# "/I../depot_tools/win_toolchain/vs_files/20d5f2553f/Windows Kits/10/Include/10.0.19041.0/winrt"
#
# Since there's a space in the include path, inputs are quoted in |flags|, we
# can helpfully use shlex to split on spaces while preserving quoted strings.
include_paths = []
for include_path in shlex.split(flags['include_flags_I']):
# Apparently setup_toolchain prefers relative include paths, which
# may work for chrome, but it does not work for dav1d, so let's make
# them asbolute again.
include_path = os.path.abspath(os.path.join(cwd, include_path[2:]))
include_paths.append(include_path)
# TODO(dalecurtis): Why isn't the ucrt path printed?
flags['vc_lib_ucrt_path'] = flags['vc_lib_um_path'].replace(
'/um/', '/ucrt/')
# Unlike the cflags, the lib include paths are each in a separate variable.
lib_paths = []
for k in flags:
# libpath_flags is like cflags. Since it is also redundant, skip it.
if 'lib' in k and k != 'libpath_flags':
lib_paths.append(flags[k])
target_env = os.environ
target_env['INCLUDE'] = ';'.join(include_paths)
target_env['LIB'] = ';'.join(lib_paths)
return target_env
def SetupWindowsCrossCompileToolchain(target_arch):
# First retrieve various MSVC and Windows SDK paths.
output = subprocess.check_output([
os.path.join(CHROMIUM_ROOT_DIR, 'build', 'vs_toolchain.py'),
'get_toolchain_dir'
])
# Turn this into a dictionary.
win_dirs = gn_helpers.FromGNArgs(output)
# Use those paths with a second script which will tell us the proper include
# and lib paths to specify for cflags and ldflags respectively.
output = subprocess.check_output([
'python',
os.path.join(CHROMIUM_ROOT_DIR, 'build', 'toolchain', 'win',
'setup_toolchain.py'), win_dirs['vs_path'],
win_dirs['sdk_path'], win_dirs['runtime_dirs'], 'win', target_arch,
'none'
])
flags = gn_helpers.FromGNArgs(output)
cwd = os.getcwd()
target_env = os.environ
include_paths = []
for cflag in flags['include_flags_imsvc'].split(' '):
# Apparently setup_toolchain prefers relative include paths, which
# may work for chrome, but it does not work for ffmpeg, so let's make
# them asbolute again.
include_path = cflag.strip('"')
if include_path.startswith('-imsvc'):
include_path = os.path.abspath(os.path.join(cwd, include_path[6:]))
include_paths.append(include_path)
# TODO(dalecurtis): Why isn't the ucrt path printed?
flags['vc_lib_ucrt_path'] = flags['vc_lib_um_path'].replace(
'/um/', '/ucrt/')
# Unlike the cflags, the lib include paths are each in a separate variable.
lib_paths = []
for k in flags:
# libpath_flags is like cflags. Since it is also redundant, skip it.
if 'lib' in k and k != 'libpath_flags':
lib_paths.append(flags[k])
target_env = os.environ
target_env['INCLUDE'] = ';'.join(include_paths)
target_env['LIB'] = ';'.join(lib_paths)
return target_env
def GNArgs(self, vals):
if vals['cros_passthrough']:
if not 'GN_ARGS' in os.environ:
raise MBErr('MB is expecting GN_ARGS to be in the environment')
gn_args = os.environ['GN_ARGS']
if not re.search('target_os.*=.*"chromeos"', gn_args):
raise MBErr(
'GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' %
gn_args)
else:
gn_args = vals['gn_args']
if self.args.goma_dir:
gn_args += ' goma_dir="%s"' % self.args.goma_dir
android_version_code = self.args.android_version_code
if android_version_code:
gn_args += ' android_default_version_code="%s"' % android_version_code
android_version_name = self.args.android_version_name
if android_version_name:
gn_args += ' android_default_version_name="%s"' % android_version_name
# Canonicalize the arg string into a sorted, newline-separated list
# of key-value pairs, and de-dup the keys if need be so that only
# the last instance of each arg is listed.
gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args))
args_file = vals.get('args_file', None)
if args_file:
gn_args = ('import("%s")\n' % vals['args_file']) + gn_args
return gn_args
def _BuildWithChromium():
"""Returns value of gclient's |build_with_chromium|."""
gni_path = os.path.join(_BUILD_DIR, 'config', 'gclient_args.gni')
with open(gni_path) as f:
data = f.read()
args = gn_helpers.FromGNArgs(data)
return args['build_with_chromium']
def GNArgs(self, vals, expand_imports=False):
if vals['cros_passthrough']:
if not 'GN_ARGS' in os.environ:
raise MBErr('MB is expecting GN_ARGS to be in the environment')
gn_args = os.environ['GN_ARGS']
if not re.search('target_os.*=.*"chromeos"', gn_args):
raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' %
gn_args)
else:
gn_args = vals['gn_args']
if self.args.goma_dir:
gn_args += ' goma_dir="%s"' % self.args.goma_dir
android_version_code = self.args.android_version_code
if android_version_code:
gn_args += ' android_default_version_code="%s"' % android_version_code
android_version_name = self.args.android_version_name
if android_version_name:
gn_args += ' android_default_version_name="%s"' % android_version_name
args_gn_lines = []
parsed_gn_args = {}
args_file = vals.get('args_file', None)
if args_file:
if expand_imports:
content = self.ReadFile(self.ToAbsPath(args_file))
parsed_gn_args = gn_helpers.FromGNArgs(content)
else:
args_gn_lines.append('import("%s")' % args_file)
# Canonicalize the arg string into a sorted, newline-separated list
# of key-value pairs, and de-dup the keys if need be so that only
# the last instance of each arg is listed.
parsed_gn_args.update(gn_helpers.FromGNArgs(gn_args))
args_gn_lines.append(gn_helpers.ToGNString(parsed_gn_args))
return '\n'.join(args_gn_lines)
def test_FromGNArgs(self):
# Booleans and numbers should work; whitespace is allowed works.
self.assertEqual(gn_helpers.FromGNArgs('foo = true\nbar = 1\n'),
{'foo': True, 'bar': 1})
# Whitespace is not required; strings should also work.
self.assertEqual(gn_helpers.FromGNArgs('foo="bar baz"'),
{'foo': 'bar baz'})
# Comments should work (and be ignored).
gn_args_lines = [
'# Top-level comment.',
'foo = true',
'bar = 1 # In-line comment.',
]
self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)),
{'foo': True, 'bar': 1})
# Lists should work.
self.assertEqual(gn_helpers.FromGNArgs('foo=[1, 2, 3]'),
{'foo': [1, 2, 3]})
# Empty strings should return an empty dict.
self.assertEqual(gn_helpers.FromGNArgs(''), {})
self.assertEqual(gn_helpers.FromGNArgs(' \n '), {})
# Non-identifiers should raise an exception.
with self.assertRaises(gn_helpers.GNException):
gn_helpers.FromGNArgs('123 = true')
# References to other variables should raise an exception.
with self.assertRaises(gn_helpers.GNException):
gn_helpers.FromGNArgs('foo = bar')
# References to functions should raise an exception.
with self.assertRaises(gn_helpers.GNException):
gn_helpers.FromGNArgs('foo = exec_script("//build/baz.py")')
# Underscores in identifiers should work.
self.assertEqual(gn_helpers.FromGNArgs('_foo = true'),
{'_foo': True})
self.assertEqual(gn_helpers.FromGNArgs('foo_bar = true'),
{'foo_bar': True})
self.assertEqual(gn_helpers.FromGNArgs('foo_=true'),
{'foo_': True})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--swarming-os', help='OS specifier for Swarming.')
parser.add_argument('--target-os', default='detect', help='gn target_os')
parser.add_argument('--arch', '-a', default='detect',
help='CPU architecture of the test binary.')
parser.add_argument('--build',
dest='build',
action='store_true',
help='Build before isolating.')
parser.add_argument('--no-build',
dest='build',
action='store_false',
help='Do not build, just isolate (default).')
parser.add_argument('--isolate-map-file', '-i',
help='path to isolate map file if not using default')
parser.add_argument('--copies', '-n', type=int, default=1,
help='Number of copies to spawn.')
parser.add_argument(
'--device-os', help='Run tests on the given version of Android.')
parser.add_argument(
'--device-type',
help='device_type specifier for Swarming'
' from https://chromium-swarm.appspot.com/botlist .')
parser.add_argument('--pool',
default='chromium.tests',
help='Use the given swarming pool.')
parser.add_argument('--results', '-r', default='results',
help='Directory in which to store results.')
parser.add_argument(
'--gtest_filter',
help='Deprecated. Pass as test runner arg instead, like \'-- '
'--gtest_filter="*#testFoo"\'')
parser.add_argument(
'--gtest_repeat',
help='Deprecated. Pass as test runner arg instead, like \'-- '
'--gtest_repeat=99\'')
parser.add_argument(
'--test-launcher-shard-index',
help='Shard index to run. Use with --test-launcher-total-shards.')
parser.add_argument('--test-launcher-total-shards',
help='Number of shards to split the test into. Use with'
' --test-launcher-shard-index.')
parser.add_argument('--no-test-flags', action='store_true',
help='Do not add --test-launcher-summary-output and '
'--system-log-file flags to the comment.')
parser.add_argument('out_dir', type=str, help='Build directory.')
parser.add_argument('target_name', type=str, help='Name of target to run.')
parser.add_argument(
'runner_args',
nargs='*',
type=str,
help='Arguments to pass to the test runner, e.g. gtest_filter and '
'gtest_repeat.')
args = parser.parse_intermixed_args()
with open(os.path.join(args.out_dir, 'args.gn')) as f:
gn_args = gn_helpers.FromGNArgs(f.read())
if args.target_os == 'detect':
if 'target_os' in gn_args:
args.target_os = gn_args['target_os'].strip('"')
else:
args.target_os = { 'darwin': 'mac', 'linux2': 'linux', 'win32': 'win' }[
sys.platform]
if args.swarming_os is None:
args.swarming_os = {
'mac': 'Mac',
'win': 'Windows',
'linux': 'Linux',
'android': 'Android',
'fuchsia': 'Linux'
}[args.target_os]
if args.target_os == 'win' and args.target_name.endswith('.exe'):
# The machinery expects not to have a '.exe' suffix.
args.target_name = os.path.splitext(args.target_name)[0]
# Determine the CPU architecture of the test binary, if not specified.
if args.arch == 'detect':
if args.target_os not in ('android', 'mac', 'win'):
executable_info = subprocess.check_output(
['file', os.path.join(args.out_dir, args.target_name)])
if 'ARM aarch64' in executable_info:
args.arch = 'arm64',
else:
args.arch = 'x86-64'
elif args.target_os == 'android':
args.arch = gn_args.get('target_cpu', 'detect')
# TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic
# once mb.py is in python3
mb_cmd = ['tools/mb/mb', 'isolate']
if not args.build:
mb_cmd.append('--no-build')
if args.isolate_map_file:
mb_cmd += ['--isolate-map-file', args.isolate_map_file]
mb_cmd += ['//' + args.out_dir, args.target_name]
subprocess.check_call(mb_cmd, shell=os.name == 'nt')
print('If you get authentication errors, follow:')
print(
' https://chromium.googlesource.com/chromium/src/+/HEAD/docs/workflow/debugging-with-swarming.md#authenticating'
)
print('Uploading to isolate server, this can take a while...')
isolate = os.path.join(args.out_dir, args.target_name + '.isolate')
archive_json = os.path.join(args.out_dir, args.target_name + '.archive.json')
subprocess.check_output([
'tools/luci-go/isolate', 'archive', '-cas-instance', 'chromium-swarm',
'-isolate', isolate, '-dump-json', archive_json
])
with open(archive_json) as f:
cas_digest = json.load(f).get(args.target_name)
# TODO(crbug.com/1268955): Use sys.executable and remove os-specific logic
# once mb.py is in python3
mb_cmd = ['tools/mb/mb', 'get-swarming-command', '--as-list']
if not args.build:
mb_cmd.append('--no-build')
if args.isolate_map_file:
mb_cmd += ['--isolate-map-file', args.isolate_map_file]
mb_cmd += ['//' + args.out_dir, args.target_name]
mb_output = subprocess.check_output(mb_cmd, shell=os.name == 'nt')
swarming_cmd = json.loads(mb_output)
if os.path.isdir(args.results):
shutil.rmtree(args.results)
os.makedirs(args.results)
try:
print('Triggering %d tasks...' % args.copies)
# Use dummy since threadpools give better exception messages
# than process pools do, and threads work fine for what we're doing.
pool = multiprocessing.dummy.Pool()
spawn_args = [(i, args, cas_digest, swarming_cmd)
for i in range(args.copies)]
spawn_results = pool.imap_unordered(_Spawn, spawn_args)
exit_codes = []
collect_results = pool.imap_unordered(_Collect, spawn_results)
for result in collect_results:
exit_codes.append(result)
successes = sum(1 for x in exit_codes if x == 0)
errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE)
failures = len(exit_codes) - successes - errors
clear_to_eol = '\033[K'
print(
'\r[%d/%d] collected: '
'%d successes, %d failures, %d bot errors...%s' %
(len(exit_codes), args.copies, successes, failures, errors,
clear_to_eol),
end=' ')
sys.stdout.flush()
print()
print('Results logs collected into', os.path.abspath(args.results) + '.')
finally:
pool.close()
pool.join()
return 0
def test_FromGNArgs(self):
# Booleans and numbers should work; whitespace is allowed works.
self.assertEqual(gn_helpers.FromGNArgs('foo = true\nbar = 1\n'), {
'foo': True,
'bar': 1
})
# Whitespace is not required; strings should also work.
self.assertEqual(gn_helpers.FromGNArgs('foo="bar baz"'),
{'foo': 'bar baz'})
# Comments should work (and be ignored).
gn_args_lines = [
'# Top-level comment.',
'foo = true',
'bar = 1 # In-line comment followed by whitespace.',
' ',
'baz = false',
]
self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)), {
'foo': True,
'bar': 1,
'baz': False
})
# Lists should work.
self.assertEqual(gn_helpers.FromGNArgs('foo=[1, 2, 3]'),
{'foo': [1, 2, 3]})
# Empty strings should return an empty dict.
self.assertEqual(gn_helpers.FromGNArgs(''), {})
self.assertEqual(gn_helpers.FromGNArgs(' \n '), {})
# Comments should work everywhere (and be ignored).
gn_args_lines = [
'# Top-level comment.',
'',
'# Variable comment.',
'foo = true',
'bar = [',
' # Value comment in list.',
' 1,',
' 2,',
']',
'',
'baz # Comment anywhere, really',
' = # also here',
' 4',
]
self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)), {
'foo': True,
'bar': [1, 2],
'baz': 4
})
# Scope should be parsed, even empty ones.
gn_args_lines = [
'foo = {',
' a = 1',
' b = [',
' { },',
' {',
' c = 1',
' },',
' ]',
'}',
]
self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)),
{'foo': {
'a': 1,
'b': [
{},
{
'c': 1,
},
]
}})
# Non-identifiers should raise an exception.
with self.assertRaises(gn_helpers.GNError):
gn_helpers.FromGNArgs('123 = true')
# References to other variables should raise an exception.
with self.assertRaises(gn_helpers.GNError):
gn_helpers.FromGNArgs('foo = bar')
# References to functions should raise an exception.
with self.assertRaises(gn_helpers.GNError):
gn_helpers.FromGNArgs('foo = exec_script("//build/baz.py")')
# Underscores in identifiers should work.
self.assertEqual(gn_helpers.FromGNArgs('_foo = true'), {'_foo': True})
self.assertEqual(gn_helpers.FromGNArgs('foo_bar = true'),
{'foo_bar': True})
self.assertEqual(gn_helpers.FromGNArgs('foo_=true'), {'foo_': True})
