def main(): logging.basicConfig(level=logging.WARN) parser = argparse.ArgumentParser(description='Deploy a new build of mojo.') parser.add_argument('deploy_root', type=str) args = parser.parse_args() # Always use android release? rel_build_dir = os.path.join('out', 'android_Release') build_dir = os.path.join(Paths().src_root, rel_build_dir) paths = Paths(build_dir=build_dir) def deploy_path(rel_path): return os.path.join(args.deploy_root, rel_path) def src_path(rel_path): return os.path.join(paths.src_root, rel_path) copy(paths.build_dir, deploy_path('mojo'), mojo_filter) copy(src_path('sky/examples'), deploy_path('sky/examples'), sky_filter) copy(src_path('sky/framework'), deploy_path('sky/framework'), sky_filter) copy(os.path.join(paths.build_dir, 'gen'), deploy_path('gen'), sky_filter) shutil.copy(os.path.join(paths.build_dir, 'apks', 'MojoShell.apk'), args.deploy_root) subprocess.check_call(['git', 'add', '.'], cwd=args.deploy_root) subprocess.check_call([ 'git', 'commit', '-m', '%s from %s' % (rel_build_dir, git_revision()) ], cwd=args.deploy_root)
def upload_symbols(config, build_dir, breakpad_upload_urls, dry_run): dump_syms_exe = os.path.join(Paths().src_root, "mojo", "tools", "linux64", "dump_syms") symupload_exe = os.path.join(Paths().src_root, "mojo", "tools", "linux64", "symupload") dest_dir = "gs://mojo/symbols/" symbols_dir = os.path.join(build_dir, "symbols") with open(os.devnull, "w") as devnull: for name in os.listdir(symbols_dir): path = os.path.join(symbols_dir, name) with open(path) as f: signature = signatures.get_signature(f, elffile) if signature is not None: dest = dest_dir + signature upload(config, path, dest, dry_run) if breakpad_upload_urls: with tempfile.NamedTemporaryFile() as temp: check_call([dump_syms_exe, path], dry_run, stdout=temp, stderr=devnull) temp.flush() for upload_url in breakpad_upload_urls: check_call([symupload_exe, temp.name, upload_url], dry_run)
def _MakeMaybeXvfbEntry(config, name, command): """Makes an entry with the given name to run the given command, using xvfb if appropriate.""" real_command = ["python"] if config.target_os == Config.OS_LINUX: paths = Paths(config) real_command += ["./testing/xvfb.py", paths.SrcRelPath(paths.build_dir)] real_command += command return _MakeEntry(name, real_command)
def main(): logging.basicConfig(level=logging.WARN) parser = argparse.ArgumentParser(description='Deploy a new build of mojo.') parser.add_argument('deploy_root', type=str) args = parser.parse_args() # Always use android release? rel_build_dir = os.path.join('out', 'android_Release') build_dir = os.path.join(Paths().src_root, rel_build_dir) paths = Paths(build_dir=build_dir) dart_pkg_dir = os.path.join(paths.build_dir, 'gen', 'dart-pkg') sky_pkg_dir = os.path.join(dart_pkg_dir, 'sky') sky_pkg_lib_dir = os.path.join(sky_pkg_dir, 'lib') dart_pkg_packages_dir = os.path.join(dart_pkg_dir, 'packages') def deploy_path(rel_path): return os.path.join(args.deploy_root, rel_path) def src_path(rel_path): return os.path.join(paths.src_root, rel_path) # Verify that material-design-icons have been downloaded. icons_dir = os.path.join(dart_pkg_packages_dir, 'sky/assets/material-design-icons') if not os.path.isdir(icons_dir): print('NOTE: Running `download_material_design_icons` for you.') subprocess.check_call( [os.path.join(sky_pkg_lib_dir, 'download_material_design_icons')]) # Copy sky/sdk/example into example/ copy(src_path('sky/sdk/example'), deploy_path('example'), examples_filter) copy(src_path('sky/sdk/home.dart'), args.deploy_root) # Deep copy packages/. This follows symlinks and flattens them. packages_root = deploy_path('packages') copy(dart_pkg_packages_dir, packages_root, packages_filter, True) # Write out license. with open(deploy_path('LICENSES.sky'), 'w') as license_file: subprocess.check_call([src_path('tools/licenses.py'), 'credits'], stdout=license_file) # Run git commands. subprocess.check_call(['git', 'add', '.'], cwd=args.deploy_root) subprocess.check_call([ 'git', 'commit', '-m', '%s from %s' % (rel_build_dir, git_revision()) ], cwd=args.deploy_root)
def upload_system_thunks_lib(config, dry_run): paths = Paths(config) version = Version().version dest = config.values['upload_location'] + \ ("system_thunks/%s/%s/libsystem_thunks.a" % (target(config), version)) source_path = paths.build_dir + "/obj/mojo/libsystem_thunks.a" upload(config, source_path, dest, dry_run)
def upload_dart_snapshotter(config, dry_run, verbose): # Only built for Linux and Mac. if not config.target_os in [Config.OS_LINUX, Config.OS_MAC]: return paths = Paths(config) zipfile_name = target(config) version = Version().version dest = config.values['upload_location'] + "dart_snapshotter/" + version + \ "/" + zipfile_name + ".zip" with tempfile.NamedTemporaryFile() as zip_file: with zipfile.ZipFile(zip_file, 'w') as z: dart_snapshotter_path = paths.target_dart_snapshotter_path with open(dart_snapshotter_path) as dart_snapshotter_binary: dart_snapshotter_filename = os.path.basename(dart_snapshotter_path) zipinfo = zipfile.ZipInfo(dart_snapshotter_filename) zipinfo.external_attr = 0777 << 16L compress_type = zipfile.ZIP_DEFLATED zipinfo.compress_type = compress_type zipinfo.date_time = time.gmtime(os.path.getmtime(dart_snapshotter_path)) if verbose: print "zipping %s" % dart_snapshotter_path z.writestr(zipinfo, dart_snapshotter_binary.read()) upload(config, zip_file.name, dest, dry_run, gzip=False) # Update the LATEST file to contain the version of the new binary. latest_file = config.values['upload_location'] + \ ("dart_snapshotter/%s/LATEST" % target(config)) write_file_to_gs(version, latest_file, config, dry_run)
def upload_shell(config, dry_run, verbose): paths = Paths(config) shell_path = paths.target_mojo_shell_path target = "%s-%s" % (config.target_os, config.target_arch) dest_prefix = "gs://mojo/shell/" + Version().version + "/" + target if config.target_os == Config.OS_ANDROID: # The APK is already compressed, so zipping it is redundant. We could zip it # using zipfile.ZIP_STORED, but not all unzip utilities can correctly unzip # a file zipped that way. shell_binary_name = os.path.basename(shell_path) dest = dest_prefix + "/" + shell_binary_name upload(config, shell_path, dest, dry_run) return # Zip the shell binary and upload it. with tempfile.NamedTemporaryFile() as zip_file: with zipfile.ZipFile(zip_file, 'w') as z: dest = dest_prefix + ".zip" with open(shell_path) as shell_binary: shell_filename = os.path.basename(shell_path) zipinfo = zipfile.ZipInfo(shell_filename) zipinfo.external_attr = 0777 << 16L zipinfo.compress_type = zipfile.ZIP_DEFLATED zipinfo.date_time = time.gmtime(os.path.getmtime(shell_path)) if verbose: print "zipping %s" % shell_path z.writestr(zipinfo, shell_binary.read()) upload(config, zip_file.name, dest, dry_run)
def _run_test_with_xvfb(config, shell, args, apptest): '''Run the test with xvfb; return the output or raise an exception.''' env = os.environ.copy() if (config.target_os != Config.OS_LINUX or '--gtest_list_tests' in args or not xvfb.should_start_xvfb(env)): return _run_test_with_timeout(config, shell, args, apptest, env) try: # Simply prepending xvfb.py to the command line precludes direct control of # test subprocesses, and prevents easily getting output when tests timeout. xvfb_proc = None openbox_proc = None global XVFB_DISPLAY_ID display_string = ':' + str(XVFB_DISPLAY_ID) (xvfb_proc, openbox_proc) = xvfb.start_xvfb(env, Paths(config).build_dir, display=display_string) XVFB_DISPLAY_ID = (XVFB_DISPLAY_ID + 1) % 50000 if not xvfb_proc or not xvfb_proc.pid: raise Exception('Xvfb failed to start; aborting test run.') if not openbox_proc or not openbox_proc.pid: raise Exception('Openbox failed to start; aborting test run.') logging.getLogger().debug( 'Running Xvfb %s (pid %d) and Openbox (pid %d).' % (display_string, xvfb_proc.pid, openbox_proc.pid)) return _run_test_with_timeout(config, shell, args, apptest, env) finally: xvfb.kill(xvfb_proc) xvfb.kill(openbox_proc)
def PrepareShellRun(config): """ Returns a context allowing a shell to be run. This will start an internal http server to serve mojo applications, forward a local port on the device to this http server and install the current version of the mojo shell. """ build_dir = Paths(config).build_dir constants.SetOutputDirectort(build_dir) httpd = _SilentTCPServer(('127.0.0.1', 0), _GetHandlerClassForPath(build_dir)) atexit.register(httpd.shutdown) host_port = httpd.server_address[1] http_thread = threading.Thread(target=httpd.serve_forever) http_thread.daemon = True http_thread.start() device = android_commands.AndroidCommands( android_commands.GetAttachedDevices()[0]) device.EnableAdbRoot() device.ManagedInstall(os.path.join(build_dir, 'apks', 'MojoShell.apk'), keep_data=True, package_name=MOJO_SHELL_PACKAGE_NAME) atexit.register(forwarder.Forwarder.UnmapAllDevicePorts, device) forwarder.Forwarder.Map([(0, host_port)], device) context = Context(device, forwarder.Forwarder.DevicePortForHostPort(host_port)) atexit.register(StopShell, context) return context
def main(): parser = argparse.ArgumentParser( description="Runs pure Go tests in the " "Mojo source tree from a list of directories specified in a data file." ) parser.add_argument('go_tool_path', metavar='go-tool-path', help="the path to the 'go' binary") parser.add_argument("test_list_file", type=file, metavar='test-list-file', help="a file listing directories containing go tests " "to run") args = parser.parse_args() go_tool = os.path.abspath(args.go_tool_path) # Execute the Python script specified in args.test_list_file. # This will populate a list of Go directories. test_list_globals = {} exec args.test_list_file in test_list_globals test_dirs = test_list_globals["test_dirs"] src_root = Paths().src_root assert os.path.isabs(src_root) # |test_dirs| is a list of "/"-delimited, |src_root|-relative, paths # of directories containing a Go package. for path in test_dirs: test_dir = os.path.join(src_root, *path.split('/')) os.chdir(test_dir) print "Running Go tests in %s..." % test_dir call_result = InvokeGo(go_tool, ["test"], src_root=src_root) if call_result: return call_result
def upload_shell(config, dry_run, verbose): paths = Paths(config) zipfile_name = target(config) version = Version().version # Upload the binary. # TODO(blundell): Change this to be in the same structure as the LATEST files, # e.g., gs://mojo/shell/linux-x64/<version>/shell.zip. dest = "gs://mojo/shell/" + version + "/" + zipfile_name + ".zip" with tempfile.NamedTemporaryFile() as zip_file: with zipfile.ZipFile(zip_file, 'w') as z: shell_path = paths.target_mojo_shell_path with open(shell_path) as shell_binary: shell_filename = os.path.basename(shell_path) zipinfo = zipfile.ZipInfo(shell_filename) zipinfo.external_attr = 0777 << 16L compress_type = zipfile.ZIP_DEFLATED if config.target_os == Config.OS_ANDROID: # The APK is already compressed. compress_type = zipfile.ZIP_STORED zipinfo.compress_type = compress_type zipinfo.date_time = time.gmtime(os.path.getmtime(shell_path)) if verbose: print "zipping %s" % shell_path z.writestr(zipinfo, shell_binary.read()) upload(config, zip_file.name, dest, dry_run, gzip=False) # Update the LATEST file to contain the version of the new binary. latest_file = "gs://mojo/shell/%s/LATEST" % target(config) write_file_to_gs(version, latest_file, config, dry_run)
def main(): parser = argparse.ArgumentParser( description="Upload binaries for apps and " "the Mojo shell to google storage (by default on Linux, but this can be " "changed via options).") parser.add_argument("-n", "--dry_run", help="Dry run, do not actually " + "upload", action="store_true") parser.add_argument("-v", "--verbose", help="Verbose mode", action="store_true") parser.add_argument("--android", action="store_true", help="Upload the shell and apps for Android") args = parser.parse_args() target_os = Config.OS_LINUX if args.android: target_os = Config.OS_ANDROID config = Config(target_os=target_os, is_debug=False) upload_shell(config, args.dry_run, args.verbose) update_version(config, "shell", args.dry_run) apps_to_upload = find_apps_to_upload( gn.BuildDirectoryForConfig(config, Paths(config).src_root)) for app in apps_to_upload: upload_app(app, config, args.dry_run) update_version(config, "services", args.dry_run) return 0
def get_shell_executable(config): paths = Paths(config=config) if config.target_os == Config.OS_ANDROID: return os.path.join(paths.src_root, "mojo", "tools", "android_mojo_shell.py") else: return paths.mojo_shell_path
def main(): logging.basicConfig() parser = argparse.ArgumentParser(usage=USAGE) debug_group = parser.add_mutually_exclusive_group() debug_group.add_argument('--debug', help='Debug build (default)', default=True, action='store_true') debug_group.add_argument('--release', help='Release build', default=False, dest='debug', action='store_false') parser.add_argument('--target-cpu', help='CPU architecture to run for.', choices=['x64', 'x86', 'arm']) parser.add_argument('--origin', help='Origin for mojo: URLs.', default='localhost') parser.add_argument('--target-device', help='Device to run on.') launcher_args, args = parser.parse_known_args() config = Config(target_os=Config.OS_ANDROID, target_cpu=launcher_args.target_cpu, is_debug=launcher_args.debug, apk_name="MojoRunner.apk") paths = Paths(config) shell = AndroidShell(paths.target_mojo_shell_path, paths.build_dir, paths.adb_path, launcher_args.target_device) extra_shell_args = shell.PrepareShellRun(launcher_args.origin) args.extend(extra_shell_args) shell.CleanLogs() p = shell.ShowLogs() shell.StartShell(args, sys.stdout, p.terminate) return 0
def get_gsutil(): paths = Paths() sys.path.insert(0, os.path.join(paths.src_root, "tools")) # pylint: disable=F0401 import find_depot_tools depot_tools_path = find_depot_tools.add_depot_tools_to_path() return os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil")
def __init__(self, args): self._args = args self._benchmark_dir = os.path.dirname(os.path.realpath(__file__)) if args.release: build_directory = os.path.join('out', 'Release') else: build_directory = os.path.join('out', 'Debug') self._paths = Paths(build_dir=build_directory)
def _build_command_line(config, args, apptest): """Build the apptest command line. This value isn't executed on Android.""" paths = Paths(config) # On Linux, always run tests with xvfb, but not for --gtest_list_tests. use_xvfb = (config.target_os == Config.OS_LINUX and not "--gtest_list_tests" in args) prefix = [paths.xvfb, paths.build_dir] if use_xvfb else [] return prefix + [paths.mojo_runner] + args + [apptest]
def main(): parser = argparse.ArgumentParser(description="Upload binaries for apps and " "the Mojo shell to google storage (by default on Linux, but this can be " "changed via options).") parser.add_argument("-n", "--dry_run", help="Dry run, do not actually "+ "upload", action="store_true") parser.add_argument("-v", "--verbose", help="Verbose mode", action="store_true") parser.add_argument("--android", action="store_true", help="Upload the shell and apps for Android") parser.add_argument("--official", action="store_true", help="Upload the official build of the Android shell") parser.add_argument("-p", "--use-default-path-for-gsutil", action="store_true", help=("Use default $PATH location for finding gsutil." " Helpful when gcloud sdk has been installed and $PATH has been set.")) parser.add_argument("--symbols-upload-url", action="append", default=[], help="URL of the server to upload breakpad symbols to") parser.add_argument("-u", "--upload-location", default="gs://mojo/", help="root URL of the server to upload binaries to") args = parser.parse_args() is_official_build = args.official target_os = Config.OS_LINUX if args.android: target_os = Config.OS_ANDROID elif is_official_build: print "Uploading official builds is only supported for android." return 1 config = Config(target_os=target_os, is_debug=False, is_official_build=is_official_build, upload_location=args.upload_location, use_default_path_for_gsutil=args.use_default_path_for_gsutil, verbose=args.verbose) upload_shell(config, args.dry_run, args.verbose) if is_official_build: print "Skipping uploading apps (official apk build)." return 0 build_directory = gn.BuildDirectoryForConfig(config, Paths(config).src_root) apps_to_upload = find_apps_to_upload(build_directory) for app in apps_to_upload: upload_app(app, config, args.dry_run) upload_symbols(config, build_directory, args.symbols_upload_url, args.dry_run) upload_dart_snapshotter(config, args.dry_run, args.verbose) upload_system_thunks_lib(config, args.dry_run) return 0
def main(): parser = argparse.ArgumentParser( description="Upload binaries for apps and " "the Mojo shell to google storage (by default on Linux, but this can be " "changed via options).") parser.add_argument("-n", "--dry_run", help="Dry run, do not actually " + "upload", action="store_true") parser.add_argument("-v", "--verbose", help="Verbose mode", action="store_true") parser.add_argument("--android", action="store_true", help="Upload the shell and apps for Android") parser.add_argument("--official", action="store_true", help="Upload the official build of the Android shell") parser.add_argument("--symbols-upload-url", action="append", default=[], help="URL of the server to upload breakpad symbols to") args = parser.parse_args() is_official_build = args.official target_os = Config.OS_LINUX if args.android: target_os = Config.OS_ANDROID elif is_official_build: print "Uploading official builds is only supported for android." return 1 config = Config(target_os=target_os, is_debug=False, is_official_build=is_official_build) upload_shell(config, args.dry_run, args.verbose) if is_official_build: print "Skipping uploading apps (official apk build)." return 0 build_directory = gn.BuildDirectoryForConfig(config, Paths(config).src_root) apps_to_upload = find_apps_to_upload(build_directory) for app in apps_to_upload: upload_app(app, config, args.dry_run) files_to_upload = find_architecture_independent_files(build_directory) for file_to_upload in files_to_upload: upload_file(file_to_upload, config, args.dry_run) upload_symbols(config, build_directory, args.symbols_upload_url, args.dry_run) return 0
def _get_gsutil_exe(): """Get the path to gsutil executable.""" config = Config(target_os=Config.OS_ANDROID, is_debug=False, is_official_build=True) paths = Paths(config) sys.path.insert(0, os.path.join(paths.src_root, "tools")) # pylint: disable=F0401 import find_depot_tools depot_tools_path = find_depot_tools.add_depot_tools_to_path() gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil") return gsutil_exe
def main(): logging.basicConfig() # Uncomment to debug: # _logging.setLevel(logging.DEBUG) parser = argparse.ArgumentParser(description='A test runner for gtest ' 'application tests.') parser.add_argument('apptest_list_file', type=file) parser.add_argument('build_dir', type=str) args = parser.parse_args() _logging.debug("Test list file: %s", args.apptest_list_file) apptest_list = ast.literal_eval(args.apptest_list_file.read()) _logging.debug("Test list: %s" % apptest_list) mopy.gtest.set_color() mojo_shell = Paths(build_dir=args.build_dir).mojo_shell_path exit_code = 0 for apptest_dict in apptest_list: if apptest_dict.get("disabled"): continue apptest = apptest_dict["test"] apptest_args = apptest_dict.get("test-args", []) shell_args = apptest_dict.get("shell-args", []) print "Running " + apptest + "...", sys.stdout.flush() # List the apptest fixtures so they can be run independently for isolation. # TODO(msw): Run some apptests without fixture isolation? fixtures = mopy.gtest.get_fixtures(mojo_shell, apptest) if not fixtures: print "Failed with no tests found." exit_code = 1 continue apptest_result = "Succeeded" for fixture in fixtures: args_for_apptest = " ".join( ["--args-for=" + apptest, "--gtest_filter=" + fixture] + apptest_args) command = [mojo_shell, apptest, args_for_apptest] + shell_args if not mopy.gtest.run_test(command): apptest_result = "Failed test(s) in " + apptest exit_code = 1 print apptest_result return exit_code
def upload(config, source, dest, dry_run): paths = Paths(config) sys.path.insert(0, os.path.join(paths.src_root, "tools")) # pylint: disable=F0401 import find_depot_tools depot_tools_path = find_depot_tools.add_depot_tools_to_path() gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil") if dry_run: print str([gsutil_exe, "cp", source, dest]) else: subprocess.check_call([gsutil_exe, "cp", source, dest])
def upload(config, source, dest, dry_run, gzip=False): paths = Paths(config) sys.path.insert(0, os.path.join(paths.src_root, "tools")) # pylint: disable=F0401 import find_depot_tools depot_tools_path = find_depot_tools.add_depot_tools_to_path() gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil") command_line = [gsutil_exe, "cp"] if gzip and "." in source: extension = source.split(".")[-1] command_line.extend(["-z", extension]) command_line.extend([source, dest]) check_call(command_line, dry_run)
def main(): logging.basicConfig(level=logging.WARN) parser = argparse.ArgumentParser(description='Dump bloat treeview.') args = parser.parse_args() check_deps() # Always use android release? rel_build_dir = os.path.join('out', 'android_Release') src_root = Paths().src_root build_dir = os.path.join(src_root, rel_build_dir) tools_dir = os.path.join(src_root, ANDROID_TOOLS_DIR) tools_prefix = 'arm-linux-androideabi-' nm = os.path.join(tools_dir, tools_prefix + 'nm') objdump = os.path.join(tools_dir, tools_prefix + 'objdump') for name, binary in binaries(build_dir): print 'Analyzing', name nm_path = name + '.nm' objdump_path = name + '.objdump' json_path = name + '.json' html_path = name + '.html' with open(nm_path, 'w') as nm_file: args = [nm, '-C', '-S', '-l', binary] subprocess.check_call(args, stdout=nm_file) with open(objdump_path, 'w') as objdump_file: subprocess.check_call([objdump, '-h', binary], stdout=objdump_file) with open(json_path, 'w') as json_file: subprocess.check_call([ sys.executable, 'bloat/bloat.py', '--nm-output=' + nm_path, '--objdump-output=' + objdump_path, '--strip-prefix=' + src_root + '/', 'syms' ], stdout=json_file) source = None with open('bloat/index.html', 'r') as source_file: source = source_file.read().replace('bloat.json', json_path) with open(html_path, 'w') as html_file: html_file.write(source)
def main(): parser = argparse.ArgumentParser( description="A test runner for application " "tests.") parser.add_argument("--verbose", help="be verbose (multiple times for more)", default=0, dest="verbose_count", action="count") parser.add_argument("test_list_file", type=file, help="a file listing apptests to run") parser.add_argument("build_dir", type=str, help="the build output directory") args = parser.parse_args() InitLogging(args.verbose_count) config = ConfigForGNArgs(ParseGNConfig(args.build_dir)) paths = Paths(config) extra_args = [] if config.target_os == Config.OS_ANDROID: shell = AndroidShell(paths.adb_path) device_status, error = shell.CheckDevice() if not device_status: print 'Device check failed: ' + error return 1 shell.InstallApk(paths.target_mojo_shell_path) extra_args.extend( shell_arguments.ConfigureLocalOrigin(shell, paths.build_dir, fixed_port=True)) else: shell = LinuxShell(paths.mojo_shell_path) gtest.set_color() test_list_globals = {"config": config} exec args.test_list_file in test_list_globals apptests_result = run_apptests(shell, extra_args, test_list_globals["tests"]) return 0 if apptests_result else 1
def main(): parser = argparse.ArgumentParser( description="A test runner for application " "tests.") parser.add_argument("--verbose", help="be verbose (multiple times for more)", default=0, dest="verbose_count", action="count") parser.add_argument("test_list_file", type=str, help="a file listing apptests to run") parser.add_argument("build_dir", type=str, help="the build output directory") args = parser.parse_args() InitLogging(args.verbose_count) config = ConfigForGNArgs(ParseGNConfig(args.build_dir)) paths = Paths(config) command_line = [ os.path.join(os.path.dirname(__file__), os.path.pardir, "devtools", "common", "mojo_test"), str(args.test_list_file) ] if config.target_os == Config.OS_ANDROID: command_line.append("--android") command_line.append("--adb-path=" + paths.adb_path) command_line.append("--origin=" + paths.build_dir) command_line.append("--shell-path=" + paths.target_mojo_shell_path) if args.verbose_count: command_line.append("--verbose") gtest.set_color() print "Running " + str(command_line) ret = subprocess.call(command_line) return ret
def main(): os.chdir(Paths().src_root) parser = argparse.ArgumentParser(description='A script to make building' '/testing Mojo components easier.') parent_parser = argparse.ArgumentParser(add_help=False) parent_parser.add_argument('--verbose', help='Be verbose (multiple times for more)', default=0, dest='verbose_count', action='count') parent_parser.add_argument('--asan', help='Use Address Sanitizer', action='store_true') parent_parser.add_argument('--dcheck_always_on', help='DCHECK and MOJO_DCHECK are fatal even in ' 'release builds', action='store_true') debug_group = parent_parser.add_mutually_exclusive_group() debug_group.add_argument('--debug', help='Debug build (default)', default=True, action='store_true') debug_group.add_argument('--release', help='Release build', default=False, dest='debug', action='store_false') # The official build is a release build suitable for distribution, with a # different package name. debug_group.add_argument('--official', help='Official build', default=False, dest='official', action='store_true') os_group = parent_parser.add_mutually_exclusive_group() os_group.add_argument('--android', help='Build for Android', action='store_true') os_group.add_argument('--ios', help='Build for iOS', action='store_true') os_group.add_argument('--fnl', help='Build for FNL', action='store_true') parent_parser.add_argument('--simulator', help='Build for a simulator of the target', action='store_true') parent_parser.add_argument('--target-cpu', help='CPU architecture to build for.', choices=['x64', 'x86', 'arm']) parent_parser.add_argument('--target-sysroot', help='Location of sysroot for target', default='', dest='target_sysroot') parent_parser.add_argument('--toolchain-prefix', help='Toolchain prefix', default='', dest='toolchain_prefix') subparsers = parser.add_subparsers() sync_parser = subparsers.add_parser( 'sync', parents=[parent_parser], help='Sync using gclient (does not run gn).') sync_parser.set_defaults(func=_sync) gn_parser = subparsers.add_parser('gn', parents=[parent_parser], help='Run gn for mojo (does not sync).') gn_parser.set_defaults(func=_gn) gn_parser.add_argument('--args', help='Specify extra args', default=None, dest='gn_args') # Note: no default, if nothing is specified on the command line GN decides. gn_parser.add_argument('--nacl', help='Add in NaCl', action='store_true', default=argparse.SUPPRESS) gn_parser.add_argument('--no-nacl', help='Remove NaCl', action='store_false', default=argparse.SUPPRESS, dest='nacl') clang_group = gn_parser.add_mutually_exclusive_group() clang_group.add_argument('--clang', help='Use Clang (default)', default=None, action='store_true') clang_group.add_argument('--gcc', help='Use GCC', dest='clang', action='store_false') goma_group = gn_parser.add_mutually_exclusive_group() goma_group.add_argument('--goma', help='Use Goma (if $GOMA_DIR is set or $HOME/goma ' 'exists; default)', default=True, action='store_true') goma_group.add_argument('--no-goma', help='Don\'t use Goma', default=False, dest='goma', action='store_false') build_parser = subparsers.add_parser('build', parents=[parent_parser], help='Build') build_parser.set_defaults(func=_build) test_parser = subparsers.add_parser( 'test', parents=[parent_parser], help='Run unit tests (does not build).') test_parser.set_defaults(func=_test) test_parser.add_argument('--dry-run', help='Print instead of executing commands', default=False, action='store_true') perftest_parser = subparsers.add_parser( 'perftest', parents=[parent_parser], help='Run perf tests (does not build).') perftest_parser.set_defaults(func=_perftest) pytest_parser = subparsers.add_parser( 'pytest', parents=[parent_parser], help='Run Python unit tests (does not build).') pytest_parser.set_defaults(func=_pytest) analyze_dart_parser = subparsers.add_parser( 'analyze-dart', parents=[parent_parser], help='Run the dart analyzer (does not build.') analyze_dart_parser.add_argument('package_name', nargs='?', default=None) analyze_dart_parser.set_defaults(func=_analyzedart) args = parser.parse_args() global _verbose_count _verbose_count = args.verbose_count InitLogging(_verbose_count) if args.simulator and not args.ios: sys.exit("Currently, the simulator target is only configured for iOS") return args.func(_args_to_config(args))
def _args_to_config(args): # Default to host OS. target_os = None if args.android: target_os = Config.OS_ANDROID elif args.ios: target_os = Config.OS_IOS elif args.fnl: target_os = Config.OS_FNL target_cpu = args.target_cpu additional_args = {} if 'clang' in args: additional_args['is_clang'] = args.clang if 'asan' in args and args.asan: additional_args['sanitizer'] = Config.SANITIZER_ASAN # Additional non-standard config entries: if 'goma' in args: goma_dir = os.environ.get('GOMA_DIR') goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma') if args.goma and goma_dir: additional_args['use_goma'] = True additional_args['goma_dir'] = goma_dir elif args.goma and os.path.exists(goma_home_dir): additional_args['use_goma'] = True additional_args['goma_dir'] = goma_home_dir else: additional_args['use_goma'] = False additional_args['goma_dir'] = None if 'nacl' in args: additional_args['use_nacl'] = args.nacl if not ('asan' in args and args.asan): go_dir = os.path.join(Paths().src_root, 'third_party', 'go', 'tool') if args.android: additional_args['mojo_use_go'] = True additional_args['go_build_tool'] = os.path.join( go_dir, 'android_arm', 'bin', 'go') elif target_os is None and Config.GetHostOS() == Config.OS_LINUX: additional_args['mojo_use_go'] = True additional_args['go_build_tool'] = os.path.join( go_dir, 'linux_amd64', 'bin', 'go') if 'dry_run' in args: additional_args['dry_run'] = args.dry_run if 'builder_name' in args: additional_args['builder_name'] = args.builder_name if 'build_number' in args: additional_args['build_number'] = args.build_number if 'master_name' in args: additional_args['master_name'] = args.master_name if 'test_results_server' in args: additional_args['test_results_server'] = args.test_results_server if 'gn_args' in args: additional_args['gn_args'] = args.gn_args is_debug = args.debug and not args.official if 'target_sysroot' in args and args.target_sysroot: additional_args['target_sysroot'] = os.path.abspath( args.target_sysroot) if 'toolchain_prefix' in args and args.toolchain_prefix: additional_args['toolchain_prefix'] = args.toolchain_prefix if 'package_name' in args: additional_args['package_name'] = args.package_name return Config(target_os=target_os, target_cpu=target_cpu, is_debug=is_debug, is_official_build=args.official, dcheck_always_on=args.dcheck_always_on, is_simulator=args.simulator, **additional_args)
def _get_out_dir(config): """Gets the build output directory (e.g., out/Debug), relative to src, for the given config.""" paths = Paths(config) return paths.SrcRelPath(paths.build_dir)
def GetTestList(config, verbose_count=0): """Gets the list of tests to run for the given config. The test list (which is returned) is just a list of dictionaries, each dictionary having two required fields: { "name": "Short name", "command": ["python", "test_runner.py", "--some", "args"] } """ types_to_run = set(config.test_types) # See above for a description of the test list. test_list = [] paths = Paths(config) build_dir = paths.SrcRelPath(paths.build_dir) target_os = config.target_os verbose_flags = verbose_count * ["--verbose"] # Utility functions ---------------------------------------------------------- # Call this to determine if a test matching classes this_tests_types should # run: e.g., ShouldRunTest(Config.TEST_TYPE_DEFAULT, "sky") returns true if # the test list being requested specifies the default set or the "sky" set. def ShouldRunTest(*this_tests_types): return not types_to_run.isdisjoint(this_tests_types) # Call this to add the given command to the test list. def AddEntry(name, command): if config.sanitizer == Config.SANITIZER_ASAN: command = ([ "python", os.path.join("mojo", "tools", "run_command_through_symbolizer.py") ] + command) test_list.append({"name": name, "command": command}) # Call this to add the given command to the test list. If appropriate, the # command will be run under xvfb. def AddXvfbEntry(name, command): real_command = ["python"] if config.target_os == Config.OS_LINUX: real_command += [ "./testing/xvfb.py", paths.SrcRelPath(paths.build_dir) ] real_command += command AddEntry(name, real_command) # ---------------------------------------------------------------------------- # TODO(vtl): Currently, we only know how to run tests for Android, Linux, or # Windows. if target_os not in (Config.OS_ANDROID, Config.OS_LINUX, Config.OS_WINDOWS, Config.OS_IOS): return test_list # Tests run by default ------------------------------------------------------- # C++ unit tests: if ShouldRunTest(Config.TEST_TYPE_DEFAULT, Config.TEST_TYPE_UNIT): AddXvfbEntry("Unit tests", [ os.path.join("mojo", "tools", "test_runner.py"), os.path.join("mojo", "tools", "data", "unittests"), build_dir ] + verbose_flags) # NaCl tests (Linux only): if (target_os == Config.OS_LINUX and config.sanitizer != Config.SANITIZER_ASAN): AddEntry("SFI NaCl tests", [ os.path.join(build_dir, "monacl_shell_sfi"), os.path.join(build_dir, "irt_" + config.target_cpu, "irt_mojo.nexe"), os.path.join(build_dir, "clang_newlib_" + config.target_cpu, "monacl_test.nexe") ]) # C++ app tests: if ShouldRunTest(Config.TEST_TYPE_DEFAULT, "app"): AddXvfbEntry("App tests", [ os.path.join("mojo", "tools", "apptest_runner.py"), os.path.join("mojo", "tools", "data", "apptests"), build_dir ] + verbose_flags) # Non-SFI NaCl app tests (Linux including Android, no asan): if config.sanitizer != Config.SANITIZER_ASAN: AddXvfbEntry("Non-SFI NaCl App tests", [ os.path.join("mojo", "tools", "apptest_runner.py"), os.path.join("mojo", "tools", "data", "nacl_nonsfi_apptests"), build_dir ] + verbose_flags) # NaCl app tests (Linux excluding Android): if (target_os == Config.OS_LINUX and config.sanitizer != Config.SANITIZER_ASAN): # TODO(smklein): Add for Android once tests are pexes. AddXvfbEntry("NaCl app tests", [ os.path.join("mojo", "tools", "apptest_runner.py"), os.path.join("mojo", "tools", "data", "nacl_apptests"), build_dir ] + verbose_flags) # Go unit tests (Linux-only): if (target_os == Config.OS_LINUX and config.sanitizer != Config.SANITIZER_ASAN and ShouldRunTest( Config.TEST_TYPE_DEFAULT, Config.TEST_TYPE_UNIT, "go")): # Go system tests: AddEntry("Go system tests", [os.path.join(build_dir, "obj", "mojo", "go", "system_test")]) # Pure Go unit tests: assert paths.go_tool_path is not None go_tool = paths.go_tool_path AddEntry("Go unit tests", [ "python", os.path.join("mojo", "tools", "run_pure_go_tests.py"), go_tool, os.path.join("mojo", "tools", "data", "gotests") ]) # Python unit tests: if ShouldRunTest(Config.TEST_TYPE_DEFAULT, Config.TEST_TYPE_UNIT, "python"): AddEntry("Python unit tests", [ "python", os.path.join("mojo", "tools", "run_mojo_python_tests.py") ]) # Python bindings tests (Linux-only): # See http://crbug.com/438781 for details on asan exclusion. if (target_os == Config.OS_LINUX and ShouldRunTest( Config.TEST_TYPE_DEFAULT, Config.TEST_TYPE_UNIT, "python") and config.sanitizer != Config.SANITIZER_ASAN): AddEntry("Python bindings tests", [ "python", os.path.join("mojo", "tools", "run_mojo_python_bindings_tests.py"), "--build-dir=" + build_dir ]) AddEntry("Mojom translator python tests", [ "python", os.path.join("mojo", "tools", "run_mojom_translator_python_tests.py"), "--build-dir=" + build_dir ]) # mojo tools unit tests: if ShouldRunTest(Config.TEST_TYPE_DEFAULT, Config.TEST_TYPE_UNIT, "tools"): AddEntry("Mojo tools unit tests", [ "python", os.path.join("mojo", "tools", "testing", "mojom_fetcher", "mojom_fetcher_tests.py") ]) # Observatory tests (Linux-only): if target_os == Config.OS_LINUX and ShouldRunTest( Config.TEST_TYPE_DEFAULT): AddEntry("Dart Observatory tests", [ "python", os.path.join("mojo", "dart", "unittests", "observatory_tester", "runner.py"), "--build-dir=" + build_dir, "--dart-exe=third_party/dart-sdk/dart-sdk/bin/dart" ]) AddEntry("Dart HTTP Load test", [ "python", os.path.join("mojo", "dart", "unittests", "http_load_test", "runner.py"), "--build-dir=" + build_dir, "--dart-exe=third_party/dart-sdk/dart-sdk/bin/dart" ]) if target_os == Config.OS_LINUX: # Dart mojom package generate.dart script tests: if ShouldRunTest(Config.TEST_TYPE_DEFAULT): AddEntry("Dart mojom package generate tests", [ os.path.join("third_party", "dart-sdk", "dart-sdk", "bin", "dart"), "--checked", "-p", os.path.join(build_dir, "gen", "dart-pkg", "packages"), os.path.join("mojo", "dart", "packages", "mojom", "test", "generate_test.dart") ]) # Tests of Dart examples. if ShouldRunTest(Config.TEST_TYPE_DEFAULT): AddEntry("Dart examples tests", [ "python", os.path.join("examples", "dart", "example_tests.py"), "--build-dir", build_dir ]) # Test of Dart's snapshotter. if ShouldRunTest(Config.TEST_TYPE_DEFAULT): AddEntry("Dart snapshotter test", [ "python", os.path.join("mojo", "dart", "embedder", "snapshotter", "test", "dart_snapshotter_test.py"), "--build-dir=" + build_dir ]) # Dart analyzer test if ShouldRunTest(Config.TEST_TYPE_DEFAULT, "analyze-dart"): AddEntry("Dart Package Static Analysis", [ "python", os.path.join("mojo", "public", "tools", "dart_pkg_static_analysis.py"), "--dart-pkg-dir=" + os.path.join(build_dir, "gen", "dart-pkg"), "--dart-sdk=" + os.path.join("third_party", "dart-sdk", "dart-sdk"), "--package-root=" + os.path.join(build_dir, "gen", "dart-pkg", "packages") ]) # Perf tests ----------------------------------------------------------------- bot_name = "linux_%s" % ("debug" if config.is_debug else "release") if target_os == Config.OS_LINUX and ShouldRunTest(Config.TEST_TYPE_PERF): test_names = [ "mojo_public_system_perftests", "mojo_public_bindings_perftests", "mojo_edk_system_perftests" ] for test_name in test_names: command = [ "python", os.path.join("mojo", "tools", "perf_test_runner.py"), "--upload", "--server-url", _PERFORMANCE_DASHBOARD_URL, "--bot-name", bot_name, "--test-name", test_name, "--perf-data-path", os.path.join(build_dir, test_name + "_perf.log") ] if config.values.get("builder_name"): command += ["--builder-name", config.values["builder_name"]] if config.values.get("build_number"): command += ["--build-number", config.values["build_number"]] if config.values.get("master_name"): command += ["--master-name", config.values["master_name"]] command += [os.path.join(build_dir, test_name)] AddEntry(test_name, command) # Benchmarks ----------------------------------------------------------------- if target_os == Config.OS_LINUX and ShouldRunTest(Config.TEST_TYPE_PERF): spec_files = ["benchmarks", "rtt_benchmarks"] aggregate_run_count = 3 for spec_file in spec_files: command = [ "python", os.path.join("mojo", "devtools", "common", "mojo_benchmark"), os.path.join("mojo", "tools", "data", spec_file), "--aggregate", aggregate_run_count, "--upload", "--server-url", _PERFORMANCE_DASHBOARD_URL, "--bot-name", bot_name, "--test-name", "mojo_benchmarks" ] if config.values.get("builder_name"): command += ["--builder-name", config.values["builder_name"]] if config.values.get("build_number"): command += ["--build-number", config.values["build_number"]] if config.values.get("master_name"): command += ["--master-name", config.values["master_name"]] if not config.is_debug: command += ["--release"] AddEntry("benchmarks: " + spec_file, command) # Integration tests ---------------------------------------------------------- if target_os == Config.OS_ANDROID and ShouldRunTest( Config.TEST_TYPE_DEFAULT, Config.TEST_TYPE_INTEGRATION): AddEntry("Integration test (MojoTest)", [ "python", os.path.join("build", "android", "test_runner.py"), "instrumentation", "--test-apk=MojoTest", "--output-directory=%s" % build_dir, "--test_data=bindings:mojo/public/interfaces/bindings/tests/data" ] + verbose_flags) return test_list