def GetUpdatePayloadsFromLocalPath(path, payload_dir, src_image_to_delta=None, static_dir=DEFAULT_STATIC_DIR): """Generates update payloads from a local image path. This function wraps around ConvertLocalPathToXbuddy and GetUpdatePayloads, managing the creation and destruction of the necessary temporary directories required by this process. Args: path: Path to an image. payload_dir: The directory to store the payloads. On failure, the devserver log will be copied to |payload_dir|. src_image_to_delta: Image used as the base to generate the delta payloads. static_dir: Devserver static dir to use. """ with cros_build_lib.ContextManagerStack() as stack: image_tempdir = stack.Add(osutils.TempDir, base_dir=path_util.FromChrootPath('/tmp'), prefix='dev_server_wrapper_local_image', sudo_rm=True) static_tempdir = stack.Add(osutils.TempDir, base_dir=static_dir, prefix='local_image', sudo_rm=True) xbuddy_path = ConvertLocalPathToXbuddyPath(path, image_tempdir, static_tempdir, static_dir) GetUpdatePayloads(xbuddy_path, payload_dir, src_image_to_delta=src_image_to_delta, static_dir=static_dir)
def __init__(self, static_dir=None, port=None, log_dir=None, src_image=None, board=None): """Initialize a DevServerWrapper instance. Args: static_dir: The static directory to be used by the devserver. port: The port to used by the devserver. log_dir: Directory to store the log files. src_image: The path to the image to be used as the base to generate delta payloads. board: Override board to pass to the devserver for xbuddy pathing. """ super(DevServerWrapper, self).__init__() self.devserver_bin = 'start_devserver' # Set port if it is given. Otherwise, devserver will start at any # available port. self.port = None if not port else port self.src_image = src_image self.board = board self.tempdir = None self.log_dir = log_dir if not self.log_dir: self.tempdir = osutils.TempDir( base_dir=path_util.FromChrootPath('/tmp'), prefix='devserver_wrapper', sudo_rm=True) self.log_dir = self.tempdir.tempdir self.static_dir = static_dir self.log_file = os.path.join(self.log_dir, 'dev_server.log') self.port_file = os.path.join(self.log_dir, 'dev_server.port') self._pid_file = self._GetPIDFilePath() self._pid = None
def UniqueChrootFilePath(path): # If this fires, we can do stuff with tempfile. I don't think it # will, and I like path names without a lot of garbage in them. if os.path.exists(path_util.FromChrootPath(path)): raise ValueError('Path %r already exists; refusing to overwrite.' % path) return path
def DetermineAndroidBranch(self, package): """Returns the Android branch in use by the active container ebuild. Workspace version of cbuildbot_run.DetermineAndroidBranch(). Args: package: String name of Android package to get branch of. Returns: String with the android container branch name. """ ebuild_path = portage_util.FindEbuildForBoardPackage( package, self._current_board, buildroot=self._build_root) host_ebuild_path = path_util.FromChrootPath(ebuild_path, source_path=self._build_root) # We assume all targets pull from the same branch and that we always # have an ARM_TARGET or an X86_USERDEBUG_TARGET. targets = ['ARM_TARGET', 'X86_USERDEBUG_TARGET'] ebuild_content = osutils.SourceEnvironment(host_ebuild_path, targets) logging.info('Got ebuild env: %s', ebuild_content) for target in targets: if target in ebuild_content: branch = re.search(r'(.*?)-linux-', ebuild_content[target]) if branch is not None: return branch.group(1) raise cbuildbot_run.NoAndroidBranchError( 'Android branch could not be determined for %s (ebuild empty?)' % ebuild_path)
def _GetStatefulUpdateScript(self): """Returns the path to the stateful_update_bin on the target. Returns: <need_transfer, path>: need_transfer is True if stateful_update_bin is found in local path, False if we directly use stateful_update_bin on the host. path: If need_transfer is True, it represents the local path of stateful_update_bin, and is used for further transferring. Otherwise, it refers to the host path. """ # We attempt to load the local stateful update path in 2 different # ways. If this doesn't exist, we attempt to use the Chromium OS # Chroot path to the installed script. If all else fails, we use the # stateful update script on the host. stateful_update_path = path_util.FromChrootPath( self.STATEFUL_UPDATE_BIN) if not os.path.exists(stateful_update_path): logging.warning( 'Could not find chroot stateful_update script, falling ' 'back on client copy.') return False, self.REMOTE_STATEUL_UPDATE_PATH else: return True, stateful_update_path
def DetermineAndroidBranch(self, board): """Returns the Android branch in use by the active container ebuild.""" try: android_package = self.DetermineAndroidPackage(board) except cros_build_lib.RunCommandError: raise NoAndroidBranchError( 'Android branch could not be determined for %s' % board) if not android_package: raise NoAndroidBranchError( 'Android branch could not be determined for %s (no package?)' % board) ebuild_path = portage_util.FindEbuildForBoardPackage( android_package, board) host_ebuild_path = path_util.FromChrootPath(ebuild_path) # We assume all targets pull from the same branch and that we always # have at least one of the following targets. targets = constants.ANDROID_ALL_BUILD_TARGETS ebuild_content = osutils.SourceEnvironment(host_ebuild_path, targets) for target in targets: if target in ebuild_content: branch = re.search(r'(.*?)-linux-', ebuild_content[target]) if branch is not None: return branch.group(1) raise NoAndroidBranchError( 'Android branch could not be determined for %s (ebuild empty?)' % board)
def CreateMakeConfUser(): """Create default make.conf.user file in the chroot if it does not exist.""" path = '/etc/make.conf.user' if not cros_build_lib.IsInsideChroot(): path = path_util.FromChrootPath(path) if not os.path.exists(path): osutils.WriteFile(path, _DEFAULT_MAKE_CONF_USER, sudo=True)
def testTempDirInChroot(self): """Tests the correctness of TempDirInChroot.""" rm_check_dir = '' with chroot_util.TempDirInChroot() as tempdir: rm_check_dir = tempdir self.assertExists(tempdir) chroot_tempdir = path_util.FromChrootPath('/tmp') self.assertNotEqual(chroot_tempdir, tempdir) self.assertStartsWith(tempdir, chroot_tempdir) self.assertNotExists(rm_check_dir)
def main(args): opts = ParseArgs(args) # Build up test suites. loader = unittest.TestLoader() loader.suiteClass = image_test_lib.ImageTestSuite # We use a different prefix here so that unittest DO NOT pick up the # image tests automatically because they depend on a proper environment. loader.testMethodPrefix = 'Test' tests_namespace = 'chromite.cros.test.image_test' if opts.tests: tests = ['%s.%s' % (tests_namespace, x) for x in opts.tests] else: tests = (tests_namespace, ) all_tests = loader.loadTestsFromNames(tests) # If they just want to see the lists of tests, show them now. if opts.list: def _WalkSuite(suite): for test in suite: if isinstance(test, unittest.BaseTestSuite): for result in _WalkSuite(test): yield result else: yield (test.id()[len(tests_namespace) + 1:], test.shortDescription() or '') test_list = list(_WalkSuite(all_tests)) maxlen = max(len(x[0]) for x in test_list) for name, desc in test_list: print('%-*s %s' % (maxlen, name, desc)) return # Run them in the image directory. runner = image_test_lib.ImageTestRunner() runner.SetBoard(opts.board) runner.SetResultDir(opts.test_results_root) image_file = FindImage(opts.image) tmp_in_chroot = path_util.FromChrootPath('/tmp') with osutils.TempDir(base_dir=tmp_in_chroot) as temp_dir: with image_lib.LoopbackPartitions(image_file, temp_dir) as image: # Due to the lack of mount context, we mount the partitions # but do not reference directly. This will be removed with the # submission of http://crrev/c/1795578 _ = image.Mount((constants.PART_ROOT_A, ))[0] _ = image.Mount((constants.PART_STATE, ))[0] with osutils.ChdirContext(temp_dir): result = runner.run(all_tests) if result and not result.wasSuccessful(): return 1 return 0
def testTempDirInChrootWithBaseDir(self): """Tests the correctness of TempDirInChroot with a passed prefix.""" rm_check_dir = '' chroot_tempdir = path_util.FromChrootPath('/tmp/some-prefix') osutils.SafeMakedirs(chroot_tempdir) with chroot_util.TempDirInChroot(base_dir='/tmp/some-prefix') as tempdir: rm_check_dir = tempdir self.assertExists(tempdir) self.assertNotEqual(chroot_tempdir, tempdir) self.assertStartsWith(tempdir, chroot_tempdir) self.assertNotExists(rm_check_dir) osutils.RmDir(chroot_tempdir)
def _RunTest(self, test, remote, log_directory_base, fail_directory_base): """Runs a test or a suite of tests on a given remote. Runs a test target, whether an individual test or a suite of tests, with 'test_that'. Args: test: The test or suite to run. remote: The hostname of the remote DUT. log_directory_base: The base directory to store test logs. A sub directory specific to this test will be created there. fail_directory_base: The base directory to store test logs in case of a test failure. Returns: test: Same as |test|. This is useful when the caller wants to correlate results to the test name. percent_passed: Pass rate. test_report: Content of the test report generated by test_that. """ log_directory, _ = self._GetResultsDirectoryForTest( test, log_directory_base, fail_directory_base) log_directory_in_chroot = log_directory.rpartition('chroot')[2] # Copy GCE key file in a temporary file inside the chroot and # make sure to remove it before return. with tempfile.NamedTemporaryFile( dir=path_util.FromChrootPath('/tmp')) as gce_key_copy: shutil.copy(self.json_key_file, gce_key_copy.name) args = 'gce_project=%s gce_zone=%s gce_instance=%s gce_key_file=%s' % ( self.gce_context.project, self.gce_context.zone, self.instances[test], path_util.ToChrootPath(gce_key_copy.name)) cmd = ['test_that', '-b', self.board, '--no-quickmerge', '--results_dir=%s' % log_directory_in_chroot, remote, test, '--args=%s' % args] if self.ssh_private_key is not None: cmd.append('--ssh_private_key=%s' % path_util.ToChrootPath(self.ssh_private_key)) result = cros_build_lib.RunCommand(cmd, error_code_ok=True, enter_chroot=True, redirect_stdout=True, cwd=constants.CROSUTILS_DIR) percent_passed = self.ParseGeneratedTestOutput(result.output) test_report = self._GetTestReport(log_directory) # Returns the summarized test_report as it is more useful than the full # output, plus the entire log will always be linked in the failure report. return test, percent_passed, test_report
def _StageChrootFilesIntoDir(target_path, paths): """Install chroot files into a staging directory. Args: target_path: Path to the staging directory paths: An iterable of absolute paths inside the chroot """ for path in paths: chroot_path = path_util.FromChrootPath(path) install_path = os.path.join(target_path, os.path.relpath(path, '/')) install_parent = os.path.dirname(install_path) osutils.SafeMakedirs(install_parent, 0o775) shutil.copyfile(chroot_path, install_path) shutil.copymode(chroot_path, install_path)
def TransferStatefulUpdate(self): """Transfer files for stateful update. The stateful update bin and the corresponding payloads are copied to the target remote device for stateful update. Args: payload: The payload for stateful update. """ stateful_update_bin = path_util.FromChrootPath( self.STATEFUL_UPDATE_BIN) self.device.CopyToWorkDir(stateful_update_bin) logging.info('Copying stateful payload to device...') payload = os.path.join(self.payload_dir, ds_wrapper.STATEFUL_FILENAME) self.device.CopyToWorkDir(payload)
def PerformStage(self): test_root_in_chroot = commands.CreateTestRoot(self._build_root) test_root = path_util.FromChrootPath(test_root_in_chroot) results_dir = os.path.join(test_root, 'results') work_dir = os.path.join(test_root, 'workdir') osutils.SafeMakedirsNonRoot(results_dir) osutils.SafeMakedirsNonRoot(work_dir) try: self._PerformStage(work_dir, results_dir) except: logging.error( _ERROR_MSG % dict(test_name='MoblabVMTest', test_results='directory')) raise finally: self._ArchiveTestResults(results_dir)
def _GetTastSspFiles(self): """Build out the paths to the tast SSP files. Returns: list[str] - The paths to the files. """ files = [] if cros_build_lib.IsInsideChroot(): files.extend(self._TAST_SSP_CHROOT_FILES) else: files.extend(path_util.FromChrootPath(x) for x in self._TAST_SSP_CHROOT_FILES) for filename in self._TAST_SSP_SOURCE_FILES: files.append(os.path.join(constants.SOURCE_ROOT, filename)) return files
def _TempDirInChroot(): """A context to create and use a tempdir inside the chroot. Yields: A host path (not chroot path) to a tempdir inside the chroot. This tempdir is cleaned up when exiting the context. """ chroot_tempdir = cros_build_lib.RunCommand( ['mktemp', '-d'], capture_output=True, enter_chroot=True, ).output.strip() tempdir = path_util.FromChrootPath(chroot_tempdir) try: yield tempdir finally: osutils.RmDir(tempdir, ignore_missing=True)
def TranslatedPathToLocalPath(translated_path, static_dir): """Convert the translated path to a local path to the image file. Args: translated_path: the translated xbuddy path (e.g., peppy-release/R36-5760.0.0/chromiumos_image). static_dir: The static directory used by the devserver. Returns: A local path to the image file. """ real_path = osutils.ExpandPath(os.path.join(static_dir, translated_path)) if os.path.exists(real_path): return real_path else: return path_util.FromChrootPath(real_path)
def __init__(self, version_file=None, hooks_dir=None): if version_file: # We have one. Just here to skip the logic below since we don't need it. default_version_file = None elif cros_build_lib.IsInsideChroot(): # Use the absolute path since we're inside the chroot. default_version_file = CHROOT_VERSION_FILE else: # Otherwise convert to the path outside the chroot. default_version_file = path_util.FromChrootPath( CHROOT_VERSION_FILE) self._version_file = version_file or default_version_file self._hooks_dir = hooks_dir or _CHROOT_VERSION_HOOKS_DIR self._version = None self._latest_version = None self._hook_files = None
def TempDirInChroot(**kwargs): """A context to create and use a tempdir inside the chroot. Args: prefix: See tempfile.mkdtemp documentation. base_dir: The directory to place the temporary directory in the chroot. set_global: See osutils.TempDir documentation. delete: See osutils.TempDir documentation. sudo_rm: See osutils.TempDir documentation. Yields: A host path (not chroot path) to a tempdir inside the chroot. This tempdir is cleaned up when exiting the context. """ base_dir = kwargs.pop('base_dir', '/tmp') kwargs['base_dir'] = path_util.FromChrootPath(base_dir) tempdir = osutils.TempDir(**kwargs) yield tempdir.tempdir
def CreateAndUploadPayload(payload, cache, work_dir, sign=True, verify=True, dry_run=False, au_generator_uri=None): """Helper to create a PaygenPayloadLib instance and use it. Args: payload: An instance of utils.Payload describing the payload to generate. cache: An instance of DownloadCache for retrieving files. work_dir: A working directory that can hold scratch files. Will be cleaned up when done, and won't interfere with other users. None for /tmp. sign: Boolean saying if the payload should be signed (normally, you do). verify: whether the payload should be verified (default: True) dry_run: don't perform actual work au_generator_uri: URI to override standard au_generator.zip rules. """ # We need to create a temp directory inside the chroot so be able to access # from both inside and outside the chroot. temp_dir = path_util.FromChrootPath( cros_build_lib.RunCommand(['mktemp', '-d'], capture_output=True, enter_chroot=True).output.strip()) with osutils.TempDir(prefix='paygen_payload.', base_dir=temp_dir) as gen_dir: logging.info('* Starting payload generation') start_time = datetime.datetime.now() _PaygenPayload(payload, cache, gen_dir, sign, verify, au_generator_uri, dry_run=dry_run).Run() end_time = datetime.datetime.now() logging.info('* Finished payload generation in %s', end_time - start_time)
def UpdateChromeEbuildAFDOFile(board, profiles): """Update chrome ebuild with the dictionary of {arch: afdo_file} pairs. Modifies the Chrome ebuild to set the appropriate AFDO file for each given architecture. Regenerates the associated Manifest file and commits the new ebuild and Manifest. Args: board: board we are building Chrome for. profiles: {arch: afdo_file} pairs to put into the ebuild. These are profiles from selected benchmarks. """ # Find the Chrome ebuild file. equery_prog = 'equery' ebuild_prog = 'ebuild' if board: equery_prog += '-%s' % board ebuild_prog += '-%s' % board equery_cmd = [equery_prog, 'w', 'chromeos-chrome'] ebuild_file = cros_build_lib.run(equery_cmd, enter_chroot=True, stdout=True).output.rstrip() # Patch the ebuild file with the names of the available afdo_files. PatchChromeEbuildAFDOFile(ebuild_file, profiles) # Also patch the 9999 ebuild. This is necessary because the uprev # process starts from the 9999 ebuild file and then compares to the # current version to see if the uprev is really necessary. We dont # want the names of the available afdo_files to show as differences. # It also allows developers to do USE=afdo_use when using the 9999 # ebuild. ebuild_9999 = os.path.join(os.path.dirname(ebuild_file), 'chromeos-chrome-9999.ebuild') PatchChromeEbuildAFDOFile(ebuild_9999, profiles) UpdateManifest(ebuild_9999, ebuild_prog) ebuild_dir = path_util.FromChrootPath(os.path.dirname(ebuild_file)) CommitIfChanged(ebuild_dir, 'Update profiles and manifests for Chrome.')
def _BuildTarball(self, input_list, tarball_output, compressed=True, **kwargs): """Tars and zips files and directories from input_list to tarball_output. Args: input_list: A list of files and directories to be archived. tarball_output: Path of output tar archive file. compressed: Whether or not the tarball should be compressed with pbzip2. **kwargs: Keyword arguments to pass to CreateTarball. Returns: Return value of cros_build_lib.CreateTarball. """ for pathname in input_list: if os.path.exists(os.path.join(self.archive_basedir, pathname)): break else: # If any of them exist we can create an archive, but if none # do then we need to stop. For now, since we either pass in a # handful of directories we don't necessarily check, or actually # search the filesystem for lots of files, this is far more # efficient than building out a list of files that do exist. return None compressor = cros_build_lib.COMP_NONE chroot = None if compressed: compressor = cros_build_lib.COMP_BZIP2 if not cros_build_lib.IsInsideChroot(): chroot = path_util.FromChrootPath('/') return cros_build_lib.CreateTarball(tarball_output, self.archive_basedir, compression=compressor, chroot=chroot, inputs=input_list, **kwargs)
def PatchChromeEbuildAFDOFile(ebuild_file, arch_profiles): """Patch the Chrome ebuild with the dictionary of {arch: afdo_file} pairs. Args: ebuild_file: path of the ebuild file within the chroot. arch_profiles: {arch: afdo_file} pairs to put into the ebuild. """ original_ebuild = path_util.FromChrootPath(ebuild_file) modified_ebuild = '%s.new' % original_ebuild arch_patterns = {} arch_repls = {} arch_markers = {} for arch in arch_profiles.keys(): arch_patterns[arch] = re.compile(CHROME_EBUILD_AFDO_EXP % arch) arch_repls[arch] = CHROME_EBUILD_AFDO_REPL % arch_profiles[arch] arch_markers[arch] = False with open(original_ebuild, 'r') as original: with open(modified_ebuild, 'w') as modified: for line in original: for arch in arch_profiles.keys(): matched = arch_patterns[arch].match(line) if matched: arch_markers[arch] = True modified.write(arch_patterns[arch].sub( arch_repls[arch], line)) break else: # line without markers, just copy it. modified.write(line) for arch, found in arch_markers.iteritems(): if not found: raise MissingAFDOMarkers( 'Chrome ebuild file does not have appropriate ' 'AFDO markers for arch %s' % arch) os.rename(modified_ebuild, original_ebuild)
def main(args): opts = ParseArgs(args) # Build up test suites. loader = unittest.TestLoader() loader.suiteClass = image_test_lib.ImageTestSuite # We use a different prefix here so that unittest DO NOT pick up the # image tests automatically because they depend on a proper environment. loader.testMethodPrefix = 'Test' all_tests = loader.loadTestsFromName('chromite.cros.test.image_test') forgiving = image_test_lib.ImageTestSuite() non_forgiving = image_test_lib.ImageTestSuite() for suite in all_tests: for test in suite.GetTests(): if test.IsForgiving(): forgiving.addTest(test) else: non_forgiving.addTest(test) # Run them in the image directory. runner = image_test_lib.ImageTestRunner() runner.SetBoard(opts.board) runner.SetResultDir(opts.test_results_root) image_file = FindImage(opts.image_dir) tmp_in_chroot = path_util.FromChrootPath('/tmp') with osutils.TempDir(base_dir=tmp_in_chroot) as temp_dir: with osutils.MountImageContext(image_file, temp_dir): with osutils.ChdirContext(temp_dir): # Run non-forgiving tests first so that exceptions in forgiving tests # do not skip any required tests. logging.info('Running NON-forgiving tests.') result = runner.run(non_forgiving) logging.info('Running forgiving tests.') runner.run(forgiving) if result and not result.wasSuccessful(): return 1 return 0
def _BuildTarball(self, input_list, tarball_output, compressed=True, **kwargs): """Tars and zips files and directories from input_list to tarball_output. Args: input_list: A list of files and directories to be archived. tarball_output: Path of output tar archive file. compressed: Whether or not the tarball should be compressed with pbzip2. **kwargs: Keyword arguments to pass to CreateTarball. Returns: Return value of cros_build_lib.CreateTarball. """ compressor = cros_build_lib.COMP_NONE chroot = None if compressed: compressor = cros_build_lib.COMP_BZIP2 if not cros_build_lib.IsInsideChroot(): chroot = path_util.FromChrootPath('/') return cros_build_lib.CreateTarball( tarball_output, self.archive_basedir, compression=compressor, chroot=chroot, inputs=input_list, **kwargs)
def _GetStatefulUpdateScript(self): """Returns the path to the stateful_update_bin on the target. Returns: (need_transfer, path): need_transfer is True if stateful_update_bin is found in local path, False if we directly use stateful_update_bin on the host. path: If need_transfer is True, it represents the local path of stateful_update_bin, and is used for further transferring. Otherwise, it refers to the host path. """ # We attempt to load the local stateful update path in 2 different # ways. If this doesn't exist, we attempt to use the Chromium OS # Chroot path to the installed script. If all else fails, we use the # stateful update script on the host. stateful_update_path = path_util.FromChrootPath( self.LOCAL_CHROOT_STATEFUL_UPDATE_PATH) if not os.path.exists(stateful_update_path): logging.warning( 'Could not find chroot stateful_update script in %s, ' 'falling back to the client copy.', stateful_update_path) stateful_update_path = os.path.join( self._dev_dir, self.LOCAL_STATEFUL_UPDATE_FILENAME) if os.path.exists(stateful_update_path): logging.debug( 'Use stateful_update script in devserver path: %s', stateful_update_path) return True, stateful_update_path logging.debug( 'Cannot find stateful_update script, will use the script ' 'on the host') return False, self.REMOTE_STATEFUL_UPDATE_PATH else: return True, stateful_update_path
def UpdateChromeEbuildAFDOFile(board, arch_profiles): """Update chrome ebuild with the dictionary of {arch: afdo_file} pairs. Modifies the Chrome ebuild to set the appropriate AFDO file for each given architecture. Regenerates the associated Manifest file and commits the new ebuild and Manifest. Args: board: board we are building Chrome for. arch_profiles: {arch: afdo_file} pairs to put into the ebuild. """ # Find the Chrome ebuild file. equery_prog = 'equery' ebuild_prog = 'ebuild' if board: equery_prog += '-%s' % board ebuild_prog += '-%s' % board equery_cmd = [equery_prog, 'w', 'chromeos-chrome'] ebuild_file = cros_build_lib.RunCommand( equery_cmd, enter_chroot=True, redirect_stdout=True).output.rstrip() # Patch the ebuild file with the names of the available afdo_files. PatchChromeEbuildAFDOFile(ebuild_file, arch_profiles) # Also patch the 9999 ebuild. This is necessary because the uprev # process starts from the 9999 ebuild file and then compares to the # current version to see if the uprev is really necessary. We dont # want the names of the available afdo_files to show as differences. # It also allows developers to do USE=afdo_use when using the 9999 # ebuild. ebuild_9999 = os.path.join(os.path.dirname(ebuild_file), 'chromeos-chrome-9999.ebuild') PatchChromeEbuildAFDOFile(ebuild_9999, arch_profiles) # Regenerate the Manifest file. ebuild_gs_dir = None # If using the GS test location, pass this location to the # chrome ebuild. if AFDO_BASE_URL == AFDO_TEST_URL: ebuild_gs_dir = {'AFDO_GS_DIRECTORY': AFDO_TEST_URL} gen_manifest_cmd = [ebuild_prog, ebuild_file, 'manifest', '--force'] cros_build_lib.RunCommand(gen_manifest_cmd, enter_chroot=True, extra_env=ebuild_gs_dir, print_cmd=True) ebuild_dir = path_util.FromChrootPath(os.path.dirname(ebuild_file)) git.RunGit(ebuild_dir, ['add', 'Manifest']) # Check if anything changed compared to the previous version. mod_files = [ 'Manifest', os.path.basename(ebuild_file), os.path.basename(ebuild_9999) ] modifications = git.RunGit(ebuild_dir, ['status', '--porcelain', '--'] + mod_files, capture_output=True, print_cmd=True).output if not modifications: logging.info('AFDO info for the Chrome ebuild did not change. ' 'Nothing to commit') return # If there are changes to ebuild or Manifest, commit them. commit_msg = ('"Set {arch: afdo_file} pairs %s and updated Manifest"' % arch_profiles) git.RunGit(ebuild_dir, ['commit', '-m', commit_msg, '--'] + mod_files, print_cmd=True)
import urlparse from chromite.lib import constants from chromite.cli import command from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import gs from chromite.lib import osutils from chromite.lib import path_util from chromite.lib import remote_access from chromite.lib import timeout_util DEFAULT_PORT = 8080 DEVSERVER_PKG_DIR = os.path.join(constants.SOURCE_ROOT, 'src/platform/dev') DEFAULT_STATIC_DIR = path_util.FromChrootPath( os.path.join(constants.SOURCE_ROOT, 'src', 'platform', 'dev', 'static')) XBUDDY_REMOTE = 'remote' XBUDDY_LOCAL = 'local' ROOTFS_FILENAME = 'update.gz' STATEFUL_FILENAME = 'stateful.tgz' class ImagePathError(Exception): """Raised when the provided path can't be resolved to an image.""" def ConvertTranslatedPath(original_path, translated_path): """Converts a translated xbuddy path to an xbuddy path.
def __init__(self, goma_dir, goma_client_json, goma_tmp_dir=None, stage_name=None): """Initializes Goma instance. This ensures that |self.goma_log_dir| directory exists (if missing, creates it). Args: goma_dir: Path to the goma directory (outside of chroot). goma_client_json: Path to the service account json file to use goma. On bots, this must be specified, otherwise raise a ValueError. On local, this is optional, and can be set to None. goma_tmp_dir: Path to the GOMA_TMP_DIR to be passed to goma programs. If given, it is used. If not given, creates a directory under /tmp in the chroot, expecting that the directory is removed in the next run's clean up phase on bots. stage_name: optional name of the currently running stage. E.g. "build_packages" or "test_simple_chrome_workflow". If this is set deps cache is enabled. Raises: ValueError if 1) |goma_dir| does not point to a directory, 2) on bots, but |goma_client_json| is not given, 3) |goma_client_json| is given, but it does not point to a file, or 4) if |goma_tmp_dir| is given but it does not point to a directory. """ # Sanity checks of given paths. if not os.path.isdir(goma_dir): raise ValueError('goma_dir does not point a directory: %s' % (goma_dir,)) # If this script runs on bot, service account json file needs to be # provided, otherwise it cannot access to goma service. if cros_build_lib.HostIsCIBuilder() and goma_client_json is None: raise ValueError( 'goma is enabled on bot, but goma_client_json is not provided') # If goma_client_json file is provided, it must be an existing file. if goma_client_json and not os.path.isfile(goma_client_json): raise ValueError( 'Goma client json file is missing: %s' % (goma_client_json,)) # If goma_tmp_dir is provided, it must be an existing directory. if goma_tmp_dir and not os.path.isdir(goma_tmp_dir): raise ValueError( 'GOMA_TMP_DIR does not point a directory: %s' % (goma_tmp_dir,)) self.goma_dir = goma_dir self.goma_client_json = goma_client_json if stage_name: self.goma_cache = os.path.join(goma_dir, 'goma_cache', stage_name) osutils.SafeMakedirs(self.goma_cache) else: self.goma_cache = None if goma_tmp_dir is None: # If |goma_tmp_dir| is not given, create GOMA_TMP_DIR (goma # compiler_proxy's working directory), and its log directory. # Create unique directory by mkdtemp under chroot's /tmp. # Expect this directory is removed in next run's clean up phase. goma_tmp_dir = tempfile.mkdtemp( prefix='goma_tmp_dir.', dir=path_util.FromChrootPath('/tmp')) self.goma_tmp_dir = goma_tmp_dir # Create log directory if not exist. if not os.path.isdir(self.goma_log_dir): os.mkdir(self.goma_log_dir)
import re import shutil import tempfile from chromite.cbuildbot import constants from chromite.lib import auto_updater from chromite.lib import commandline from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import dev_server_wrapper as ds_wrapper from chromite.lib import operation from chromite.lib import osutils from chromite.lib import path_util from chromite.lib import remote_access DEVSERVER_STATIC_DIR = path_util.FromChrootPath( os.path.join(constants.CHROOT_SOURCE_ROOT, 'devserver', 'static')) class UsbImagerOperation(operation.ProgressBarOperation): """Progress bar for flashing image to operation.""" def __init__(self, image): super(UsbImagerOperation, self).__init__() self._size = os.path.getsize(image) self._transferred = 0. self._bytes = re.compile(r'(\d+) bytes') def _GetDDPid(self): """Get the Pid of dd.""" try: pids = cros_build_lib.RunCommand(['pgrep', 'dd'], capture_output=True,