def _PrintStaticInitializersCountFromApk(apk_filename, chartjson=None): print 'Finding static initializers (can take a minute)' with zipfile.ZipFile(apk_filename) as z: infolist = z.infolist() out_dir = constants.GetOutDirectory() si_count = 0 for zip_info in infolist: # Check file size to account for placeholder libraries. if zip_info.filename.endswith('.so') and zip_info.file_size > 0: lib_name = os.path.basename(zip_info.filename).replace( 'crazy.', '') unstripped_path = os.path.join(out_dir, 'lib.unstripped', lib_name) if os.path.exists(unstripped_path): si_count += _PrintStaticInitializersCount(unstripped_path) else: raise Exception('Unstripped .so not found. Looked here: %s', unstripped_path) ReportPerfResult(chartjson, 'StaticInitializersCount', 'count', si_count, 'count')
def _LogToFile(results, test_type, suite_name): """Log results to local files which can be used for aggregation later.""" log_file_path = os.path.join(constants.GetOutDirectory(), 'test_logs') if not os.path.exists(log_file_path): os.mkdir(log_file_path) full_file_name = os.path.join( log_file_path, re.sub(r'\W', '_', test_type).lower() + '.log') if not os.path.exists(full_file_name): with open(full_file_name, 'w') as log_file: print >> log_file, '\n%s results for %s build %s:' % ( test_type, os.environ.get('BUILDBOT_BUILDERNAME'), os.environ.get('BUILDBOT_BUILDNUMBER')) logging.info('Writing results to %s.', full_file_name) logging.info('Writing results to %s.', full_file_name) with open(full_file_name, 'a') as log_file: shortened_suite_name = suite_name[:25] + (suite_name[25:] and '...') print >> log_file, '%s%s' % (shortened_suite_name.ljust(30), results.GetShortForm())
def Install(self, device, incremental=False): if not incremental: device.Install(self._apk_helper, reinstall=True, permissions=self._permissions) return installer_script = os.path.join(constants.GetOutDirectory(), 'bin', 'install_%s_apk_incremental' % self._suite) try: install_wrapper = imp.load_source('install_wrapper', installer_script) except IOError: raise Exception(('Incremental install script not found: %s\n' 'Make sure to first build "%s_incremental"') % (installer_script, self._suite)) params = install_wrapper.GetInstallParameters() installer.Install(device, self._apk_helper, split_globs=params['splits'], native_libs=params['native_libs'], dex_files=params['dex_files'])
def _TriggerSetUp(self): """Set up the triggering of a test run.""" logging.info('Triggering test run.') if self._env.runner_type: logging.warning('Ignoring configured runner_type "%s"', self._env.runner_type) if not self._env.runner_package: runner_package = self.DEFAULT_RUNNER_PACKAGE logging.info('Using default runner package: %s', self.DEFAULT_RUNNER_PACKAGE) else: runner_package = self._env.runner_package dummy_app_path = os.path.join( constants.GetOutDirectory(), 'apks', 'remote_device_dummy.apk') # pylint: disable=protected-access with tempfile.NamedTemporaryFile(suffix='.flags.txt') as flag_file: env_vars = dict(self._test_instance.extras) if gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in env_vars: env_vars[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int( self._test_instance.shard_timeout * 1e9) flags = [] filter_string = self._test_instance._GenerateDisabledFilterString(None) if filter_string: flags.append('--gtest_filter=%s' % filter_string) if self._test_instance.test_arguments: flags.append(self._test_instance.test_arguments) if flags: flag_file.write('_ ' + ' '.join(flags)) flag_file.flush() env_vars[_EXTRA_COMMAND_LINE_FILE] = os.path.basename(flag_file.name) self._test_instance._data_deps.append( (os.path.abspath(flag_file.name), None)) self._AmInstrumentTestSetup( dummy_app_path, self._test_instance.apk, runner_package, environment_variables=env_vars)
def GenerateDepsDirUsingIsolate(suite_name, isolate_file_path, isolate_file_paths, deps_exclusion_list): """Generate the dependency dir for the test suite using isolate. Args: suite_name: Name of the test suite (e.g. base_unittests). isolate_file_path: .isolate file path to use. If there is a default .isolate file path for the suite_name, this will override it. isolate_file_paths: Dictionary with the default .isolate file paths for the test suites. deps_exclusion_list: A list of files that are listed as dependencies in the .isolate files but should not be pushed to the device. """ if isolate_file_path: if os.path.isabs(isolate_file_path): isolate_abs_path = isolate_file_path else: isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_file_path) else: isolate_rel_path = isolate_file_paths.get(suite_name) if not isolate_rel_path: logging.info('Did not find an isolate file for the test suite.') return isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path) isolated_abs_path = os.path.join(constants.GetOutDirectory(), '%s.isolated' % suite_name) assert os.path.exists( isolate_abs_path), 'Cannot find %s' % isolate_abs_path i = isolator.Isolator(constants.ISOLATE_DEPS_DIR) i.Clear() i.Remap(isolate_abs_path, isolated_abs_path) # We're relying on the fact that timestamps are preserved # by the remap command (hardlinked). Otherwise, all the data # will be pushed to the device once we move to using time diff # instead of md5sum. Perform a sanity check here. i.VerifyHardlinks() i.PurgeExcluded(deps_exclusion_list) i.MoveOutputDeps()
def SetupPrebuiltTools(adb): # TODO(bulach): build the host tools for mac, and the targets for x86/mips. # Prebuilt tools from r226197. has_prebuilt = sys.platform.startswith('linux') if has_prebuilt: abi = adb.system_properties['ro.product.cpu.abi'] has_prebuilt = abi.startswith('armeabi') if not has_prebuilt: logging.error( 'Prebuilt android tools only available for Linux host and ARM device.') return False prebuilt_tools = [ 'forwarder_dist/device_forwarder', 'host_forwarder', 'md5sum_dist/md5sum_bin', 'md5sum_bin_host', 'purge_ashmem', ] build_type = None for t in prebuilt_tools: src = os.path.basename(t) android_prebuilt_profiler_helper.GetIfChanged(src) bin_path = util.FindSupportBinary(t) if not build_type: build_type = GetBuildTypeOfPath(bin_path) or 'Release' constants.SetBuildType(build_type) dest = os.path.join(constants.GetOutDirectory(), t) if not bin_path: logging.warning('Setting up prebuilt %s', dest) if not os.path.exists(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) prebuilt_path = android_prebuilt_profiler_helper.GetHostPath(src) if not os.path.exists(prebuilt_path): raise NotImplementedError(""" %s must be checked into cloud storage. Instructions: http://www.chromium.org/developers/telemetry/upload_to_cloud_storage """ % t) shutil.copyfile(prebuilt_path, dest) os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) return True
def _RunCygprofileUnitTests(self): """Builds, deploys and runs cygprofile_unittests.""" # There an no unittests (yet) for the lightweight instrumentation. # TODO(lizeb): Fix this. if self._options.lightweight_instrumentation: return tools_compiler = ClankCompiler( os.path.dirname(constants.GetOutDirectory()), self._step_recorder, self._options.arch, self._options.jobs, self._options.max_load, self._options.use_goma, self._options.goma_dir, self._options.lightweight_instrumentation) tools_compiler.Build(instrumented=False, target='android_tools') self._compiler.Build(instrumented=True, target='cygprofile_unittests') self._step_recorder.BeginStep('Deploy and run cygprofile_unittests') exit_code = self._profiler.RunCygprofileTests() if exit_code != 0: self._step_recorder.FailStep( 'cygprofile_unittests exited with non-0 status: %d' % exit_code)
def ProcessCommonOptions(args): """Processes and handles all common options.""" run_tests_helper.SetLogLevel(args.verbose_count) constants.SetBuildType(args.build_type) if args.build_directory: constants.SetBuildDirectory(args.build_directory) if args.output_directory: constants.SetOutputDirectory(args.output_directory) devil_custom_deps = None if args.adb_path: devil_custom_deps = {'adb': {devil_env.GetPlatform(): [args.adb_path]}} devil_chromium.Initialize(output_directory=constants.GetOutDirectory(), custom_deps=devil_custom_deps) # Some things such as Forwarder require ADB to be in the environment path. adb_dir = os.path.dirname(constants.GetAdbPath()) if adb_dir and adb_dir not in os.environ['PATH'].split(os.pathsep): os.environ['PATH'] = adb_dir + os.pathsep + os.environ['PATH']
def RunTest(self, _test): """Runs junit tests from |self._test_suite|.""" with tempfile.NamedTemporaryFile() as json_file: java_script = os.path.join( constants.GetOutDirectory(), 'bin', self._test_suite) command = [java_script, '-test-jars', self._test_suite + '.jar', '-json-results-file', json_file.name] if self._test_filter: command.extend(['-gtest-filter', self._test_filter]) if self._package_filter: command.extend(['-package-filter', self._package_filter]) if self._runner_filter: command.extend(['-runner-filter', self._runner_filter]) if self._sdk_version: command.extend(['-sdk-version', self._sdk_version]) return_code = cmd_helper.RunCmd(command) results_list = json_results.ParseResultsFromJson( json.loads(json_file.read())) return (results_list, return_code)
def __initializeDataDependencyAttributes(self, args, isolate_delegate): self._data_deps = [] if args.isolate_file_path: self._isolate_abs_path = os.path.abspath(args.isolate_file_path) self._isolate_delegate = isolate_delegate self._isolated_abs_path = os.path.join( constants.GetOutDirectory(), '%s.isolated' % self._test_package) else: self._isolate_delegate = None # TODO(jbudorick): Deprecate and remove --test-data once data dependencies # are fully converted to isolate. if args.test_data: logging.info('Data dependencies specified via --test-data') self._test_data = args.test_data else: self._test_data = None if not self._isolate_delegate and not self._test_data: logging.warning('No data dependencies will be pushed.')
def __init__(self, suite_name): """ Args: suite_name: Name of the test suite (e.g. base_unittests). """ TestPackage.__init__(self, suite_name) self.suite_path = os.path.join(constants.GetOutDirectory(), '%s_apk' % suite_name, '%s-debug.apk' % suite_name) if suite_name == 'content_browsertests': self._package_info = constants.PACKAGE_INFO['content_browsertests'] elif suite_name == 'components_browsertests': self._package_info = constants.PACKAGE_INFO[ 'components_browsertests'] else: self._package_info = constants.PACKAGE_INFO['gtest'] if suite_name == 'net_unittests': self._extras = {'RunInSubThread': ''} else: self._extras = []
def CalculateHostMd5Sums(paths): """Calculates the MD5 sum value for all items in |paths|. Directories are traversed recursively and the MD5 sum of each file found is reported in the result. Args: paths: A list of host paths to md5sum. Returns: A dict mapping file paths to their respective md5sum checksums. """ if isinstance(paths, basestring): paths = [paths] md5sum_bin_host_path = os.path.join(constants.GetOutDirectory(), 'md5sum_bin_host') if not os.path.exists(md5sum_bin_host_path): raise IOError('File not built: %s' % md5sum_bin_host_path) out = cmd_helper.GetCmdOutput([md5sum_bin_host_path] + [p for p in paths]) return _ParseMd5SumOutput(out.splitlines())
def InstallCommands(device): if device.IsUserBuild(): raise Exception( 'chromium_commands currently requires a userdebug build.') chromium_commands_jar_path = os.path.join(constants.GetOutDirectory(), constants.SDK_BUILD_JAVALIB_DIR, 'chromium_commands.dex.jar') if not os.path.exists(chromium_commands_jar_path): raise Exception('%s not found. Please build chromium_commands.' % chromium_commands_jar_path) device.RunShellCommand(['mkdir', BIN_DIR, _FRAMEWORK_DIR]) for command, main_class in _COMMANDS.iteritems(): shell_command = _SHELL_COMMAND_FORMAT % (constants.TEST_EXECUTABLE_DIR, main_class) shell_file = '%s/%s' % (BIN_DIR, command) device.WriteFile(shell_file, shell_command) device.RunShellCommand(['chmod', '755', shell_file], check_return=True) device.adb.Push(chromium_commands_jar_path, '%s/chromium_commands.jar' % _FRAMEWORK_DIR)
def ProcessUIAutomatorOptions(options, error_func): """Processes UIAutomator options/arguments. Args: options: optparse.Options object. error_func: Function to call with the error message in case of an error. Returns: A UIAutomatorOptions named tuple which contains all options relevant to uiautomator tests. """ ProcessJavaTestOptions(options) if not options.package: error_func('--package is required.') if options.package not in constants.PACKAGE_INFO: error_func('Invalid package.') if not options.test_jar: error_func('--test-jar must be specified.') if os.path.exists(options.test_jar): # The dexed JAR is fully qualified, assume the info JAR lives along side. options.uiautomator_jar = options.test_jar else: options.uiautomator_jar = os.path.join(constants.GetOutDirectory(), constants.SDK_BUILD_JAVALIB_DIR, '%s.dex.jar' % options.test_jar) options.uiautomator_info_jar = ( options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] + '_java.jar') return uiautomator_test_options.UIAutomatorOptions( options.tool, options.cleanup_test_files, options.push_deps, options.annotations, options.exclude_annotations, options.test_filter, options.test_data, options.save_perf_json, options.screenshot_failures, options.uiautomator_jar, options.uiautomator_info_jar, options.package)
def RunTest(self, _test): """Runs junit tests from |self._test_suite|.""" with tempfile.NamedTemporaryFile() as json_file: java_script = os.path.join(constants.GetOutDirectory(), 'bin', 'helper', self._test_suite) command = [java_script] # Add Jar arguments. jar_args = [ '-test-jars', self._test_suite + '.jar', '-json-results-file', json_file.name ] if self._test_filter: jar_args.extend(['-gtest-filter', self._test_filter]) if self._package_filter: jar_args.extend(['-package-filter', self._package_filter]) if self._runner_filter: jar_args.extend(['-runner-filter', self._runner_filter]) if self._sdk_version: jar_args.extend(['-sdk-version', self._sdk_version]) command.extend(['--jar-args', '"%s"' % ' '.join(jar_args)]) # Add JVM arguments. jvm_args = [] if self._coverage_dir: if not os.path.exists(self._coverage_dir): os.makedirs(self._coverage_dir) elif not os.path.isdir(self._coverage_dir): raise Exception( '--coverage-dir takes a directory, not file path.') jvm_args.append('-Demma.coverage.out.file=%s' % os.path.join( self._coverage_dir, '%s.ec' % self._test_suite)) if jvm_args: command.extend(['--jvm-args', '"%s"' % ' '.join(jvm_args)]) return_code = cmd_helper.RunCmd(command) results_list = json_results.ParseResultsFromJson( json.loads(json_file.read())) return (results_list, return_code)
def ExtractAndResolveNativeStackTraces(self, data_to_symbolize, device_abi, include_stack=True): """Run the stack tool for given input. Args: data_to_symbolize: a list of strings to symbolize. include_stack: boolean whether to include stack data in output. device_abi: the default ABI of the device which generated the tombstone. Yields: A string for each line of resolved stack output. """ self.UnzipAPKIfNecessary() arch = _DeviceAbiToArch(device_abi) if not arch: logging.warning('No device_abi can be found.') return cmd = [ _STACK_TOOL, '--arch', arch, '--output-directory', constants.GetOutDirectory(), '--more-info' ] if self._non_native_packed_relocations and self._apk_libs: for apk_lib in self._apk_libs: cmd.extend(['--packed-lib', apk_lib]) env = dict(os.environ) env['PYTHONDONTWRITEBYTECODE'] = '1' with tempfile.NamedTemporaryFile() as f: f.write('\n'.join(data_to_symbolize)) f.flush() _, output = cmd_helper.GetCmdStatusAndOutput(cmd + [f.name], env=env) for line in output.splitlines(): if not include_stack and 'Stack Data:' in line: break yield line
def CalculateDeviceMd5Sums(paths, device): """Calculates the MD5 sum value for all items in |paths|. Directories are traversed recursively and the MD5 sum of each file found is reported in the result. Args: paths: A list of device paths to md5sum. Returns: A dict mapping file paths to their respective md5sum checksums. """ if isinstance(paths, basestring): paths = [paths] if not device.FileExists(MD5SUM_DEVICE_BIN_PATH): md5sum_dist_path = os.path.join(constants.GetOutDirectory(), 'md5sum_dist') if not os.path.exists(md5sum_dist_path): raise IOError('File not built: %s' % md5sum_dist_path) device.adb.Push(md5sum_dist_path, MD5SUM_DEVICE_LIB_PATH) out = [] with tempfile.NamedTemporaryFile() as md5sum_script_file: with device_temp_file.DeviceTempFile( device.adb) as md5sum_device_script_file: md5sum_script = (MD5SUM_DEVICE_SCRIPT_FORMAT.format( path=p, md5sum_lib=MD5SUM_DEVICE_LIB_PATH, md5sum_bin=MD5SUM_DEVICE_BIN_PATH) for p in paths) md5sum_script_file.write('; '.join(md5sum_script)) md5sum_script_file.flush() device.adb.Push(md5sum_script_file.name, md5sum_device_script_file.name) out = device.RunShellCommand( ['sh', md5sum_device_script_file.name]) return _ParseMd5SumOutput(out)
def ExtractAndResolveNativeStackTraces(self, data_to_symbolize, device_abi, include_stack=True): """Run the stack tool for given input. Args: data_to_symbolize: a list of strings to symbolize. include_stack: boolean whether to include stack data in output. device_abi: the default ABI of the device which generated the tombstone. Yields: A string for each line of resolved stack output. """ if not os.path.exists(_STACK_TOOL): logging.warning('%s missing. Unable to resolve native stack traces.', _STACK_TOOL) return arch = _DeviceAbiToArch(device_abi) if not arch: logging.warning('No device_abi can be found.') return cmd = [_STACK_TOOL, '--arch', arch, '--output-directory', constants.GetOutDirectory(), '--more-info'] env = dict(os.environ) env['PYTHONDONTWRITEBYTECODE'] = '1' with tempfile.NamedTemporaryFile(mode='w') as f: f.write('\n'.join(data_to_symbolize)) f.flush() start = time.time() try: _, output = cmd_helper.GetCmdStatusAndOutput(cmd + [f.name], env=env) finally: self._time_spent_symbolizing += time.time() - start for line in output.splitlines(): if not include_stack and 'Stack Data:' in line: break yield line
def _CreateClassToFileNameDict(test_apk): """Creates a dict mapping classes to file names from size-info apk.""" constants.CheckOutputDirectory() test_apk_size_info = os.path.join(constants.GetOutDirectory(), 'size-info', os.path.basename(test_apk) + '.jar.info') class_to_file_dict = {} # Some tests such as webview_cts_tests use a separately downloaded apk to run # tests. This means the apk may not have been built by the system and hence # no size info file exists. if not os.path.exists(test_apk_size_info): logging.debug('Apk size file not found. %s', test_apk_size_info) return class_to_file_dict with open(test_apk_size_info, 'r') as f: for line in f: file_class, file_name = line.rstrip().split(',', 1) # Only want files that are not prebuilt. if file_name.startswith('../../'): class_to_file_dict[file_class] = str( file_name.replace('../../', '//', 1)) return class_to_file_dict
def __init__(self, shard_index, test_server_port, test_server_path): """Sets up a Python driven test server on the host machine. Args: shard_index: Index of the current shard. test_server_port: Port to run the test server on. This is multiplexed with the shard index. To retrieve the real port access the member variable |port|. test_server_path: The path (relative to the root src dir) of the server """ self.host = _TEST_SERVER_HOST self.port = test_server_port + shard_index src_dir = constants.DIR_SOURCE_ROOT # Make dirs into a list of absolute paths. abs_dirs = [os.path.join(src_dir, d) for d in _PYTHONPATH_DIRS] # Add the generated python files to the path abs_dirs.extend([ os.path.join(src_dir, constants.GetOutDirectory(), d) for d in _GENERATED_PYTHONPATH_DIRS ]) current_python_path = os.environ.get('PYTHONPATH') extra_python_path = ':'.join(abs_dirs) if current_python_path: python_path = current_python_path + ':' + extra_python_path else: python_path = extra_python_path # NOTE: A separate python process is used to simplify getting the right # system path for finding includes. cmd = [ 'python', os.path.join(src_dir, test_server_path), '--log-to-console', ('--host=%s' % self.host), ('--port=%d' % self.port) ] self._test_server_process = subprocess.Popen( cmd, env={'PYTHONPATH': python_path})
def ProcessUIAutomatorOptions(args): """Processes UIAutomator options/arguments. Args: args: argparse.Namespace object. Returns: A UIAutomatorOptions named tuple which contains all options relevant to uiautomator tests. """ ProcessJavaTestOptions(args) if os.path.exists(args.test_jar): # The dexed JAR is fully qualified, assume the info JAR lives along side. args.uiautomator_jar = args.test_jar else: args.uiautomator_jar = os.path.join( constants.GetOutDirectory(), constants.SDK_BUILD_JAVALIB_DIR, '%s.dex.jar' % args.test_jar) args.uiautomator_info_jar = ( args.uiautomator_jar[:args.uiautomator_jar.find('.dex.jar')] + '_java.jar') return uiautomator_test_options.UIAutomatorOptions( args.tool, args.annotations, args.exclude_annotations, args.test_filter, args.test_data, args.save_perf_json, args.screenshot_failures, args.uiautomator_jar, args.uiautomator_info_jar, args.package, args.set_asserts)
def _GetNative(relative_func, target_names): """Returns an object containing native c++ sources list and its included path Iterate through all target_names and their deps to get the list of included paths and sources.""" out_dir = constants.GetOutDirectory() with open(os.path.join(out_dir, 'project.json'), 'r') as project_file: projects = json.load(project_file) project_targets = projects['targets'] root_dir = projects['build_settings']['root_path'] includes = set() processed_target = set() targets_stack = list(target_names) sources = [] while targets_stack: target_name = targets_stack.pop() if target_name in processed_target: continue processed_target.add(target_name) target = project_targets[target_name] includes.update(target.get('include_dirs', [])) targets_stack.extend(target.get('deps', [])) # Ignore generated files sources.extend(f for f in target.get('sources', []) if f.endswith('.cc') and not f.startswith('//out')) def process_paths(paths): # Ignores leading // return relative_func( sorted(os.path.join(root_dir, path[2:]) for path in paths)) return { 'sources': process_paths(sources), 'includes': process_paths(includes), }
def _PrintStaticInitializersCountFromApk(apk_filename, tools_prefix, chartjson=None): with zipfile.ZipFile(apk_filename) as z: so_files = [f for f in z.infolist() if f.filename.endswith('.so') and f.file_size > 0] # Skip checking static initializers for 32 bit .so files when 64 bit .so files # are present since the 32 bit versions will be checked by bots that only # build the 32 bit version. This avoids the complexity of finding 32 bit .so # files in the output directory in 64 bit builds. has_64 = any('64' in f.filename for f in so_files) files_to_check = [f for f in so_files if not has_64 or '64' in f.filename] out_dir = constants.GetOutDirectory() si_count = 0 for so_info in files_to_check: lib_name = os.path.basename(so_info.filename).replace('crazy.', '') unstripped_path = os.path.join(out_dir, 'lib.unstripped', lib_name) if os.path.exists(unstripped_path): si_count += _PrintStaticInitializersCount( apk_filename, so_info.filename, unstripped_path, tools_prefix) else: raise Exception('Unstripped .so not found. Looked here: %s', unstripped_path) ReportPerfResult(chartjson, 'StaticInitializersCount', 'count', si_count, 'count')
def GetDataDependencies(runtime_deps_path): """Returns a list of device data dependencies. Args: runtime_deps_path: A str path to the .runtime_deps file. Returns: A list of (host_path, device_path) tuples. """ if not runtime_deps_path: return [] with open(runtime_deps_path, 'r') as runtime_deps_file: rel_host_files = [l.strip() for l in runtime_deps_file if l] output_directory = constants.GetOutDirectory() abs_host_files = [ os.path.abspath(os.path.join(output_directory, r)) for r in rel_host_files ] filtered_abs_host_files = _FilterDataDeps(abs_host_files) # TODO(crbug.com/752610): Filter out host executables, and investigate # whether other files could be filtered as well. return [(f, DevicePathComponentsFor(f, output_directory)) for f in filtered_abs_host_files]
def _GetTestsFromRunner(self): test_apk_path = self._test_instance.test_apk.path pickle_path = '%s-runner.pickle' % test_apk_path # For incremental APKs, the code doesn't live in the apk, so instead check # the timestamp of the target's .stamp file. if self._test_instance.test_apk_incremental_install_json: with open(self._test_instance.test_apk_incremental_install_json ) as f: data = json.load(f) out_dir = constants.GetOutDirectory() test_mtime = max( os.path.getmtime(os.path.join(out_dir, p)) for p in data['dex_files']) else: test_mtime = os.path.getmtime(test_apk_path) try: return instrumentation_test_instance.GetTestsFromPickle( pickle_path, test_mtime) except instrumentation_test_instance.TestListPickleException as e: logging.info('Could not get tests from pickle: %s', e) logging.info('Getting tests by having %s list them.', self._test_instance.junit4_runner_class) def list_tests(d): def _run(dev): with device_temp_file.DeviceTempFile( dev.adb, suffix='.json', dir=dev. GetExternalStoragePath()) as dev_test_list_json: junit4_runner_class = self._test_instance.junit4_runner_class test_package = self._test_instance.test_package extras = { 'log': 'true', # Workaround for https://github.com/mockito/mockito/issues/922 'notPackage': 'net.bytebuddy', } extras[_EXTRA_TEST_LIST] = dev_test_list_json.name target = '%s/%s' % (test_package, junit4_runner_class) timeout = 120 if self._test_instance.wait_for_java_debugger: timeout = None test_list_run_output = dev.StartInstrumentation( target, extras=extras, retries=0, timeout=timeout) if any(test_list_run_output): logging.error('Unexpected output while listing tests:') for line in test_list_run_output: logging.error(' %s', line) with tempfile_ext.NamedTemporaryDirectory() as host_dir: host_file = os.path.join(host_dir, 'list_tests.json') dev.PullFile(dev_test_list_json.name, host_file) with open(host_file, 'r') as host_file: return json.load(host_file) return crash_handler.RetryOnSystemCrash(_run, d) raw_test_lists = self._env.parallel_devices.pMap(list_tests).pGet(None) # If all devices failed to list tests, raise an exception. # Check that tl is not None and is not empty. if all(not tl for tl in raw_test_lists): raise device_errors.CommandFailedError( 'Failed to list tests on any device') # Get the first viable list of raw tests raw_tests = [tl for tl in raw_test_lists if tl][0] instrumentation_test_instance.SaveTestsToPickle(pickle_path, raw_tests) return raw_tests
def GetOutDir(): return constants.GetOutDirectory()
def _initializeApkAttributes(self, args, error_func): if args.apk_under_test: apk_under_test_path = args.apk_under_test if (not args.apk_under_test.endswith('.apk') and not args.apk_under_test.endswith('.apks')): apk_under_test_path = os.path.join( constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, '%s.apk' % args.apk_under_test) # TODO(jbudorick): Move the realpath up to the argument parser once # APK-by-name is no longer supported. apk_under_test_path = os.path.realpath(apk_under_test_path) if not os.path.exists(apk_under_test_path): error_func('Unable to find APK under test: %s' % apk_under_test_path) self._apk_under_test = apk_helper.ToHelper(apk_under_test_path) test_apk_path = args.test_apk if not os.path.exists(test_apk_path): test_apk_path = os.path.join( constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, '%s.apk' % args.test_apk) # TODO(jbudorick): Move the realpath up to the argument parser once # APK-by-name is no longer supported. test_apk_path = os.path.realpath(test_apk_path) if not os.path.exists(test_apk_path): error_func('Unable to find test APK: %s' % test_apk_path) self._test_apk = apk_helper.ToHelper(test_apk_path) self._suite = os.path.splitext(os.path.basename(args.test_apk))[0] self._apk_under_test_incremental_install_json = ( args.apk_under_test_incremental_install_json) self._test_apk_incremental_install_json = ( args.test_apk_incremental_install_json) if self._test_apk_incremental_install_json: assert self._suite.endswith('_incremental') self._suite = self._suite[:-len('_incremental')] self._modules = args.modules self._fake_modules = args.fake_modules self._test_jar = args.test_jar self._test_support_apk = apk_helper.ToHelper(os.path.join( constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR, '%sSupport.apk' % self._suite)) if not self._test_jar: logging.warning('Test jar not specified. Test runner will not have ' 'Java annotation info available. May not handle test ' 'timeouts correctly.') elif not os.path.exists(self._test_jar): error_func('Unable to find test JAR: %s' % self._test_jar) self._test_package = self._test_apk.GetPackageName() all_instrumentations = self._test_apk.GetAllInstrumentations() all_junit3_runner_classes = [ x for x in all_instrumentations if ('0xffffffff' in x.get( 'chromium-junit3', ''))] all_junit4_runner_classes = [ x for x in all_instrumentations if ('0xffffffff' not in x.get( 'chromium-junit3', ''))] if len(all_junit3_runner_classes) > 1: logging.warning('This test apk has more than one JUnit3 instrumentation') if len(all_junit4_runner_classes) > 1: logging.warning('This test apk has more than one JUnit4 instrumentation') self._junit3_runner_class = ( all_junit3_runner_classes[0]['android:name'] if all_junit3_runner_classes else self.test_apk.GetInstrumentationName()) self._junit4_runner_class = ( all_junit4_runner_classes[0]['android:name'] if all_junit4_runner_classes else None) if self._junit4_runner_class: if self._test_apk_incremental_install_json: self._junit4_runner_supports_listing = next( (True for x in self._test_apk.GetAllMetadata() if 'real-instr' in x[0] and x[1] in _TEST_LIST_JUNIT4_RUNNERS), False) else: self._junit4_runner_supports_listing = ( self._junit4_runner_class in _TEST_LIST_JUNIT4_RUNNERS) self._package_info = None if self._apk_under_test: package_under_test = self._apk_under_test.GetPackageName() for package_info in constants.PACKAGE_INFO.itervalues(): if package_under_test == package_info.package: self._package_info = package_info break if not self._package_info: logging.warning(("Unable to find package info for %s. " + "(This may just mean that the test package is" + "currently being installed.)"), self._test_package) for apk in args.additional_apks: if not os.path.exists(apk): error_func('Unable to find additional APK: %s' % apk) self._additional_apks = ( [apk_helper.ToHelper(x) for x in args.additional_apks])
def main(): parser = argparse.ArgumentParser() parser.add_argument('--output-directory', help='Path to the root build directory.') parser.add_argument('-v', '--verbose', dest='verbose_count', default=0, action='count', help='Verbose level') parser.add_argument( '--target', dest='targets', action='append', help='GN target to generate project for. Replaces set of ' 'default targets. May be repeated.') parser.add_argument( '--extra-target', dest='extra_targets', action='append', help='GN target to generate project for, in addition to ' 'the default ones. May be repeated.') parser.add_argument('--project-dir', help='Root of the output project.', default=os.path.join('$CHROMIUM_OUTPUT_DIR', 'gradle')) parser.add_argument('--all', action='store_true', help='Include all .java files reachable from any ' 'apk/test/binary target. On by default unless ' '--split-projects is used (--split-projects can ' 'slow down Studio given too many targets).') parser.add_argument('--use-gradle-process-resources', action='store_true', help='Have gradle generate R.java rather than ninja') parser.add_argument('--split-projects', action='store_true', help='Split projects by their gn deps rather than ' 'combining all the dependencies of each target') version_group = parser.add_mutually_exclusive_group() version_group.add_argument( '--beta', action='store_true', help='Generate a project that is compatible with ' 'Android Studio Beta.') version_group.add_argument( '--canary', action='store_true', help='Generate a project that is compatible with ' 'Android Studio Canary.') sdk_group = parser.add_mutually_exclusive_group() sdk_group.add_argument( '--sdk', choices=[ 'AndroidStudioCurrent', 'AndroidStudioDefault', 'ChromiumSdkRoot' ], default='ChromiumSdkRoot', help="Set the project's SDK root. This can be set to " "Android Studio's current SDK root, the default " "Android Studio SDK root, or Chromium's SDK " "root. The default is Chromium's SDK root, but " "using this means that updates and additions to " "the SDK (e.g. installing emulators), will " "modify this root, hence possibly causing " "conflicts on the next repository sync.") sdk_group.add_argument( '--sdk-path', help='An explict path for the SDK root, setting this ' 'is an alternative to setting the --sdk option') args = parser.parse_args() if args.output_directory: constants.SetOutputDirectory(args.output_directory) constants.CheckOutputDirectory() output_dir = constants.GetOutDirectory() devil_chromium.Initialize(output_directory=output_dir) run_tests_helper.SetLogLevel(args.verbose_count) if args.use_gradle_process_resources: assert args.split_projects, ( 'Gradle resources does not work without --split-projects.') _gradle_output_dir = os.path.abspath( args.project_dir.replace('$CHROMIUM_OUTPUT_DIR', output_dir)) jinja_processor = jinja_template.JinjaProcessor(_FILE_DIR) build_vars = _ReadPropertiesFile(os.path.join(output_dir, 'build_vars.txt')) source_properties = _ReadPropertiesFile( _RebasePath( os.path.join(build_vars['android_sdk_build_tools'], 'source.properties'))) if args.beta: channel = 'beta' elif args.canary: channel = 'canary' else: channel = 'stable' generator = _ProjectContextGenerator(_gradle_output_dir, build_vars, args.use_gradle_process_resources, jinja_processor, args.split_projects, channel) logging.warning('Creating project at: %s', generator.project_dir) # Generate for "all targets" by default when not using --split-projects (too # slow), and when no --target has been explicitly set. "all targets" means all # java targets that are depended on by an apk or java_binary (leaf # java_library targets will not be included). args.all = args.all or (not args.split_projects and not args.targets) targets_from_args = set(args.targets or _DEFAULT_TARGETS) if args.extra_targets: targets_from_args.update(args.extra_targets) if args.all: # Run GN gen if necessary (faster than running "gn gen" in the no-op case). _RunNinja(constants.GetOutDirectory(), ['build.ninja']) # Query ninja for all __build_config targets. targets = _QueryForAllGnTargets(output_dir) else: targets = [ re.sub(r'_test_apk$', '_test_apk__apk', t) for t in targets_from_args ] main_entries = [_ProjectEntry.FromGnTarget(t) for t in targets] logging.warning('Building .build_config files...') _RunNinja(output_dir, [e.NinjaBuildConfigTarget() for e in main_entries]) if args.all: # There are many unused libraries, so restrict to those that are actually # used by apks/binaries/tests or that are explicitly mentioned in --targets. main_entries = [ e for e in main_entries if (e.GetType() in ('android_apk', 'java_binary', 'junit_binary') or e.GnTarget() in targets_from_args or e.GnTarget().endswith('_test_apk__apk')) ] if args.split_projects: main_entries = _FindAllProjectEntries(main_entries) logging.info('Generating for %d targets.', len(main_entries)) entries = [e for e in _CombineTestEntries(main_entries) if e.IsValid()] logging.info('Creating %d projects for targets.', len(entries)) # When only one entry will be generated we want it to have a valid # build.gradle file with its own AndroidManifest. add_all_module = not args.split_projects and len(entries) > 1 logging.warning('Writing .gradle files...') project_entries = [] zip_tuples = [] generated_inputs = [] for entry in entries: data = _GenerateGradleFile(entry, generator, build_vars, source_properties, jinja_processor) if data: # Build all paths references by .gradle that exist within output_dir. generated_inputs.extend(generator.GeneratedInputs(entry)) zip_tuples.extend(( s, os.path.join(generator.EntryOutputDir(entry), _SRCJARS_SUBDIR)) for s in generator.AllSrcjars(entry)) zip_tuples.extend( (s, os.path.join(generator.EntryOutputDir(entry), _RES_SUBDIR)) for s in generator.AllResZips(entry)) if not add_all_module: project_entries.append(entry) _WriteFile( os.path.join(generator.EntryOutputDir(entry), _GRADLE_BUILD_FILE), data) if add_all_module: _GenerateModuleAll(_gradle_output_dir, generator, build_vars, source_properties, jinja_processor) _WriteFile(os.path.join(generator.project_dir, _GRADLE_BUILD_FILE), _GenerateRootGradle(jinja_processor, channel)) _WriteFile(os.path.join(generator.project_dir, 'settings.gradle'), _GenerateSettingsGradle(project_entries, add_all_module)) if args.sdk != "AndroidStudioCurrent": if args.sdk_path: sdk_path = _RebasePath(args.sdk_path) elif args.sdk == "AndroidStudioDefault": sdk_path = os.path.expanduser('~/Android/Sdk') else: sdk_path = _RebasePath(build_vars['android_sdk_root']) _WriteFile(os.path.join(generator.project_dir, 'local.properties'), _GenerateLocalProperties(sdk_path)) if generated_inputs: logging.warning('Building generated source files...') targets = _RebasePath(generated_inputs, output_dir) _RunNinja(output_dir, targets) if zip_tuples: _ExtractZips(generator.project_dir, zip_tuples) logging.warning('Project created!') logging.warning('Generated projects work with Android Studio %s', channel) logging.warning('For more tips: https://chromium.googlesource.com/chromium' '/src.git/+/master/docs/android_studio.md')
def main(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '-C', '--output-directory', help='If outdir is not provided, will attempt to guess.') parser.add_argument('--gn-labels', action='store_true', help='Print GN labels rather than ninja targets') parser.add_argument( '--nested', action='store_true', help='Do not convert nested targets to their top-level equivalents. ' 'E.g. Without this, foo_test__apk -> foo_test') parser.add_argument('--print-types', action='store_true', help='Print type of each target') parser.add_argument('--build-build-configs', action='store_true', help='Build all .build_config files.') parser.add_argument('--type', action='append', help='Restrict to targets of given type', choices=_VALID_TYPES) parser.add_argument('--stats', action='store_true', help='Print counts of each target type.') parser.add_argument('-v', '--verbose', default=0, action='count') args = parser.parse_args() args.build_build_configs |= bool(args.type or args.print_types or args.stats) logging.basicConfig( level=logging.WARNING - (10 * args.verbose), format='%(levelname).1s %(relativeCreated)6d %(message)s') if args.output_directory: constants.SetOutputDirectory(args.output_directory) constants.CheckOutputDirectory() output_dir = constants.GetOutDirectory() # Query ninja for all __build_config_crbug_908819 targets. targets = _query_for_build_config_targets(output_dir) entries = [_TargetEntry(t) for t in targets] if args.build_build_configs: logging.warning('Building %d .build_config files...', len(entries)) _run_ninja(output_dir, [e.ninja_build_config_target for e in entries]) if args.type: entries = [e for e in entries if e.get_type() in args.type] if args.stats: counts = collections.Counter(e.get_type() for e in entries) for entry_type, count in sorted(counts.items()): print(f'{entry_type}: {count}') else: for e in entries: if args.gn_labels: to_print = e.gn_target else: to_print = e.ninja_target # Convert to top-level target if not args.nested: to_print = to_print.replace('__test_apk__apk', '').replace('__apk', '') if args.print_types: to_print = f'{to_print}: {e.get_type()}' print(to_print)
class OrderfileGenerator(object): """A utility for generating a new orderfile for Clank. Builds an instrumented binary, profiles a run of the application, and generates an updated orderfile. """ _CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank') _CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'cyglog_to_orderfile.py') _CHECK_ORDERFILE_SCRIPT = os.path.join(constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py') _BUILD_ROOT = os.path.abspath( os.path.dirname(os.path.dirname( constants.GetOutDirectory()))) # Normally /path/to/src _UNPATCHED_ORDERFILE_FILENAME = os.path.join(_CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s') _MERGED_CYGLOG_FILENAME = os.path.join(constants.GetOutDirectory(), 'merged_cyglog') _PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles', 'orderfile.%s.out') # Previous orderfile_generator debug files would be overwritten. _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files' def _GetPathToOrderfile(self): """Gets the path to the architecture-specific orderfile.""" return self._PATH_TO_ORDERFILE % self._options.arch def _GetUnpatchedOrderfileFilename(self): """Gets the path to the architecture-specific unpatched orderfile.""" return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch def __init__(self, options, orderfile_updater_class): self._options = options self._instrumented_out_dir = os.path.join( self._BUILD_ROOT, self._options.arch + '_instrumented_out') self._uninstrumented_out_dir = os.path.join( self._BUILD_ROOT, self._options.arch + '_uninstrumented_out') if options.profile: output_directory = os.path.join(self._instrumented_out_dir, 'Release') host_profile_dir = os.path.join(output_directory, 'profile_data') urls = [profile_android_startup.AndroidProfileTool.TEST_URL] use_wpr = True simulate_user = False urls = options.urls use_wpr = not options.no_wpr simulate_user = options.simulate_user self._profiler = profile_android_startup.AndroidProfileTool( output_directory, host_profile_dir, use_wpr, urls, simulate_user, device=options.device) self._output_data = {} self._step_recorder = StepRecorder(options.buildbot) self._compiler = None assert issubclass(orderfile_updater_class, OrderfileUpdater) self._orderfile_updater = orderfile_updater_class( self._CLANK_REPO, self._step_recorder, options.branch, options.netrc) assert os.path.isdir( constants.DIR_SOURCE_ROOT), 'No src directory found' symbol_extractor.SetArchitecture(options.arch) @staticmethod def _RemoveBlanks(src_file, dest_file): """A utility to remove blank lines from a file. Args: src_file: The name of the file to remove the blanks from. dest_file: The name of the file to write the output without blanks. """ assert src_file != dest_file, 'Source and destination need to be distinct' try: src = open(src_file, 'r') dest = open(dest_file, 'w') for line in src: if line and not line.isspace(): dest.write(line) finally: src.close() dest.close() def _GenerateAndProcessProfile(self): """Invokes a script to merge the per-thread traces into one file.""" self._step_recorder.BeginStep('Generate Profile Data') files = [] try: logging.getLogger().setLevel(logging.DEBUG) files = self._profiler.CollectProfile( self._compiler.chrome_apk, constants.PACKAGE_INFO['chrome']) self._step_recorder.BeginStep('Process profile') assert os.path.exists(self._compiler.lib_chrome_so) offsets = process_profiles.GetReachedOffsetsFromDumpFiles( files, self._compiler.lib_chrome_so) if not offsets: raise Exception('No profiler offsets found in {}'.format( '\n'.join(files))) with open(self._MERGED_CYGLOG_FILENAME, 'w') as f: f.write('\n'.join(map(str, offsets))) except Exception: for f in files: self._SaveForDebugging(f) raise finally: self._profiler.Cleanup() logging.getLogger().setLevel(logging.INFO) try: command_args = [ '--target-arch=' + self._options.arch, '--native-library=' + self._compiler.lib_chrome_so, '--output=' + self._GetUnpatchedOrderfileFilename() ] command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME) self._step_recorder.RunCommand([self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args) except CommandError: self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME) self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so) raise def _DeleteTempFiles(self): """Deletes intermediate step output files.""" print 'Delete %s' % (self._MERGED_CYGLOG_FILENAME) if os.path.isfile(self._MERGED_CYGLOG_FILENAME): os.unlink(self._MERGED_CYGLOG_FILENAME) def _PatchOrderfile(self): """Patches the orderfile using clean version of libchrome.so.""" self._step_recorder.BeginStep('Patch Orderfile') patch_orderfile.GeneratePatchedOrderfile( self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so, self._GetPathToOrderfile()) def _VerifySymbolOrder(self): self._step_recorder.BeginStep('Verify Symbol Order') return_code = self._step_recorder.RunCommand([ self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so, self._GetPathToOrderfile(), '--target-arch=' + self._options.arch ], constants.DIR_SOURCE_ROOT, raise_on_error=False) if return_code: self._step_recorder.FailStep('Orderfile check returned %d.' % return_code) def _RecordHash(self, file_name): """Records the hash of the file into the output_data dictionary.""" self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(file_name) def _SaveFileLocally(self, file_name, file_sha1): """Saves the file to a temporary location and prints the sha1sum.""" if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES): os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES) shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES) print 'File: %s, saved in: %s, sha1sum: %s' % ( file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1) def _SaveForDebugging(self, filename): """Uploads the file to cloud storage or saves to a temporary location.""" file_sha1 = _GenerateHash(filename) if not self._options.buildbot: self._SaveFileLocally(filename, file_sha1) else: print 'Uploading file for debugging: ' + filename self._orderfile_updater.UploadToCloudStorage( filename, use_debug_location=True) def _SaveForDebuggingWithOverwrite(self, file_name): """Uploads and overwrites the file in cloud storage or copies locally. Should be used for large binaries like lib_chrome_so. Args: file_name: (str) File to upload. """ file_sha1 = _GenerateHash(file_name) if not self._options.buildbot: self._SaveFileLocally(file_name, file_sha1) else: print 'Uploading file for debugging: %s, sha1sum: %s' % (file_name, file_sha1) upload_location = '%s/%s' % (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name)) self._step_recorder.RunCommand( ['gsutil.py', 'cp', file_name, 'gs://' + upload_location]) print('Uploaded to: https://sandbox.google.com/storage/' + upload_location) def _MaybeArchiveOrderfile(self, filename): """In buildbot configuration, uploads the generated orderfile to Google Cloud Storage. Args: filename: (str) Orderfile to upload. """ # First compute hashes so that we can download them later if we need to self._step_recorder.BeginStep('Compute hash for ' + filename) self._RecordHash(filename) if self._options.buildbot: self._step_recorder.BeginStep('Archive ' + filename) self._orderfile_updater.UploadToCloudStorage( filename, use_debug_location=False) def _GetHashFilePathAndContents(self, base_file): """Gets the name and content of the hash file created from uploading the given file. Args: base_file: The file that was uploaded to cloud storage. Returns: A tuple of the hash file name, relative to the clank repo path, and the content, which should be the sha1 hash of the file ('base_file.sha1', hash) """ abs_file_name = base_file + '.sha1' rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO) with open(abs_file_name, 'r') as f: return (rel_file_name, f.read()) def Generate(self): """Generates and maybe upload an order.""" profile_uploaded = False orderfile_uploaded = False assert (bool(self._options.profile) ^ bool(self._options.manual_symbol_offsets)) if self._options.profile: try: _UnstashOutputDirectory(self._instrumented_out_dir) self._compiler = ClankCompiler( self._instrumented_out_dir, self._step_recorder, self._options.arch, self._options.jobs, self._options.max_load, self._options.use_goma, self._options.goma_dir) self._compiler.CompileChromeApk(True) self._GenerateAndProcessProfile() self._MaybeArchiveOrderfile( self._GetUnpatchedOrderfileFilename()) profile_uploaded = True finally: self._DeleteTempFiles() _StashOutputDirectory(self._instrumented_out_dir) elif self._options.manual_symbol_offsets: assert self._options.manual_libname assert self._options.manual_objdir with file(self._options.manual_symbol_offsets) as f: symbol_offsets = [int(x) for x in f.xreadlines()] processor = process_profiles.SymbolOffsetProcessor( self._options.manual_libname) generator = cyglog_to_orderfile.OffsetOrderfileGenerator( processor, cyglog_to_orderfile.ObjectFileProcessor( self._options.manual_objdir)) ordered_sections = generator.GetOrderedSections(symbol_offsets) if not ordered_sections: # Either None or empty is a problem. raise Exception('Failed to get ordered sections') with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile: orderfile.write('\n'.join(ordered_sections)) if self._options.patch: if self._options.profile: self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(), self._GetPathToOrderfile()) try: _UnstashOutputDirectory(self._uninstrumented_out_dir) self._compiler = ClankCompiler( self._uninstrumented_out_dir, self._step_recorder, self._options.arch, self._options.jobs, self._options.max_load, self._options.use_goma, self._options.goma_dir) self._compiler.CompileLibchrome(False) self._PatchOrderfile() # Because identical code folding is a bit different with and without # the orderfile build, we need to re-patch the orderfile with code # folding as close to the final version as possible. self._compiler.CompileLibchrome(False, force_relink=True) self._PatchOrderfile() self._compiler.CompileLibchrome(False, force_relink=True) self._VerifySymbolOrder() self._MaybeArchiveOrderfile(self._GetPathToOrderfile()) finally: _StashOutputDirectory(self._uninstrumented_out_dir) orderfile_uploaded = True if (self._options.buildbot and self._options.netrc and not self._step_recorder.ErrorRecorded()): unpatched_orderfile_filename = ( self._GetUnpatchedOrderfileFilename() if profile_uploaded else None) orderfile_filename = (self._GetPathToOrderfile() if orderfile_uploaded else None) self._orderfile_updater.CommitFileHashes( unpatched_orderfile_filename, orderfile_filename) self._step_recorder.EndStep() return not self._step_recorder.ErrorRecorded() def GetReportingData(self): """Get a dictionary of reporting data (timings, output hashes)""" self._output_data['timings'] = self._step_recorder.timings return self._output_data