def GetUpdatePayloadsFromLocalPath(path,
payload_dir,
src_image_to_delta=None,
static_dir=DEFAULT_STATIC_DIR):
"""Generates update payloads from a local image path.
This function wraps around ConvertLocalPathToXbuddy and GetUpdatePayloads,
managing the creation and destruction of the necessary temporary directories
required by this process.
Args:
path: Path to an image.
payload_dir: The directory to store the payloads. On failure, the devserver
log will be copied to |payload_dir|.
src_image_to_delta: Image used as the base to generate the delta payloads.
static_dir: Devserver static dir to use.
"""
with cros_build_lib.ContextManagerStack() as stack:
image_tempdir = stack.Add(osutils.TempDir,
base_dir=path_util.FromChrootPath('/tmp'),
prefix='dev_server_wrapper_local_image',
sudo_rm=True)
static_tempdir = stack.Add(osutils.TempDir,
base_dir=static_dir,
prefix='local_image',
sudo_rm=True)
xbuddy_path = ConvertLocalPathToXbuddyPath(path, image_tempdir,
static_tempdir, static_dir)
GetUpdatePayloads(xbuddy_path,
payload_dir,
src_image_to_delta=src_image_to_delta,
static_dir=static_dir)
def __init__(self, static_dir=None, port=None, log_dir=None, src_image=None,
board=None):
"""Initialize a DevServerWrapper instance.
Args:
static_dir: The static directory to be used by the devserver.
port: The port to used by the devserver.
log_dir: Directory to store the log files.
src_image: The path to the image to be used as the base to
generate delta payloads.
board: Override board to pass to the devserver for xbuddy pathing.
"""
super(DevServerWrapper, self).__init__()
self.devserver_bin = 'start_devserver'
# Set port if it is given. Otherwise, devserver will start at any
# available port.
self.port = None if not port else port
self.src_image = src_image
self.board = board
self.tempdir = None
self.log_dir = log_dir
if not self.log_dir:
self.tempdir = osutils.TempDir(
base_dir=path_util.FromChrootPath('/tmp'),
prefix='devserver_wrapper',
sudo_rm=True)
self.log_dir = self.tempdir.tempdir
self.static_dir = static_dir
self.log_file = os.path.join(self.log_dir, 'dev_server.log')
self.port_file = os.path.join(self.log_dir, 'dev_server.port')
self._pid_file = self._GetPIDFilePath()
self._pid = None
def UniqueChrootFilePath(path):
# If this fires, we can do stuff with tempfile. I don't think it
# will, and I like path names without a lot of garbage in them.
if os.path.exists(path_util.FromChrootPath(path)):
raise ValueError('Path %r already exists; refusing to overwrite.' %
path)
return path
def DetermineAndroidBranch(self, package):
"""Returns the Android branch in use by the active container ebuild.
Workspace version of cbuildbot_run.DetermineAndroidBranch().
Args:
package: String name of Android package to get branch of.
Returns:
String with the android container branch name.
"""
ebuild_path = portage_util.FindEbuildForBoardPackage(
package, self._current_board, buildroot=self._build_root)
host_ebuild_path = path_util.FromChrootPath(ebuild_path,
source_path=self._build_root)
# We assume all targets pull from the same branch and that we always
# have an ARM_TARGET or an X86_USERDEBUG_TARGET.
targets = ['ARM_TARGET', 'X86_USERDEBUG_TARGET']
ebuild_content = osutils.SourceEnvironment(host_ebuild_path, targets)
logging.info('Got ebuild env: %s', ebuild_content)
for target in targets:
if target in ebuild_content:
branch = re.search(r'(.*?)-linux-', ebuild_content[target])
if branch is not None:
return branch.group(1)
raise cbuildbot_run.NoAndroidBranchError(
'Android branch could not be determined for %s (ebuild empty?)' %
ebuild_path)
def _GetStatefulUpdateScript(self):
"""Returns the path to the stateful_update_bin on the target.
Returns:
<need_transfer, path>:
need_transfer is True if stateful_update_bin is found in local path,
False if we directly use stateful_update_bin on the host.
path: If need_transfer is True, it represents the local path of
stateful_update_bin, and is used for further transferring. Otherwise,
it refers to the host path.
"""
# We attempt to load the local stateful update path in 2 different
# ways. If this doesn't exist, we attempt to use the Chromium OS
# Chroot path to the installed script. If all else fails, we use the
# stateful update script on the host.
stateful_update_path = path_util.FromChrootPath(
self.STATEFUL_UPDATE_BIN)
if not os.path.exists(stateful_update_path):
logging.warning(
'Could not find chroot stateful_update script, falling '
'back on client copy.')
return False, self.REMOTE_STATEUL_UPDATE_PATH
else:
return True, stateful_update_path
def DetermineAndroidBranch(self, board):
"""Returns the Android branch in use by the active container ebuild."""
try:
android_package = self.DetermineAndroidPackage(board)
except cros_build_lib.RunCommandError:
raise NoAndroidBranchError(
'Android branch could not be determined for %s' % board)
if not android_package:
raise NoAndroidBranchError(
'Android branch could not be determined for %s (no package?)' %
board)
ebuild_path = portage_util.FindEbuildForBoardPackage(
android_package, board)
host_ebuild_path = path_util.FromChrootPath(ebuild_path)
# We assume all targets pull from the same branch and that we always
# have at least one of the following targets.
targets = constants.ANDROID_ALL_BUILD_TARGETS
ebuild_content = osutils.SourceEnvironment(host_ebuild_path, targets)
for target in targets:
if target in ebuild_content:
branch = re.search(r'(.*?)-linux-', ebuild_content[target])
if branch is not None:
return branch.group(1)
raise NoAndroidBranchError(
'Android branch could not be determined for %s (ebuild empty?)' %
board)
def CreateMakeConfUser():
"""Create default make.conf.user file in the chroot if it does not exist."""
path = '/etc/make.conf.user'
if not cros_build_lib.IsInsideChroot():
path = path_util.FromChrootPath(path)
if not os.path.exists(path):
osutils.WriteFile(path, _DEFAULT_MAKE_CONF_USER, sudo=True)
def testTempDirInChroot(self):
"""Tests the correctness of TempDirInChroot."""
rm_check_dir = ''
with chroot_util.TempDirInChroot() as tempdir:
rm_check_dir = tempdir
self.assertExists(tempdir)
chroot_tempdir = path_util.FromChrootPath('/tmp')
self.assertNotEqual(chroot_tempdir, tempdir)
self.assertStartsWith(tempdir, chroot_tempdir)
self.assertNotExists(rm_check_dir)
def main(args):
opts = ParseArgs(args)
# Build up test suites.
loader = unittest.TestLoader()
loader.suiteClass = image_test_lib.ImageTestSuite
# We use a different prefix here so that unittest DO NOT pick up the
# image tests automatically because they depend on a proper environment.
loader.testMethodPrefix = 'Test'
tests_namespace = 'chromite.cros.test.image_test'
if opts.tests:
tests = ['%s.%s' % (tests_namespace, x) for x in opts.tests]
else:
tests = (tests_namespace, )
all_tests = loader.loadTestsFromNames(tests)
# If they just want to see the lists of tests, show them now.
if opts.list:
def _WalkSuite(suite):
for test in suite:
if isinstance(test, unittest.BaseTestSuite):
for result in _WalkSuite(test):
yield result
else:
yield (test.id()[len(tests_namespace) + 1:],
test.shortDescription() or '')
test_list = list(_WalkSuite(all_tests))
maxlen = max(len(x[0]) for x in test_list)
for name, desc in test_list:
print('%-*s %s' % (maxlen, name, desc))
return
# Run them in the image directory.
runner = image_test_lib.ImageTestRunner()
runner.SetBoard(opts.board)
runner.SetResultDir(opts.test_results_root)
image_file = FindImage(opts.image)
tmp_in_chroot = path_util.FromChrootPath('/tmp')
with osutils.TempDir(base_dir=tmp_in_chroot) as temp_dir:
with image_lib.LoopbackPartitions(image_file, temp_dir) as image:
# Due to the lack of mount context, we mount the partitions
# but do not reference directly. This will be removed with the
# submission of http://crrev/c/1795578
_ = image.Mount((constants.PART_ROOT_A, ))[0]
_ = image.Mount((constants.PART_STATE, ))[0]
with osutils.ChdirContext(temp_dir):
result = runner.run(all_tests)
if result and not result.wasSuccessful():
return 1
return 0
def testTempDirInChrootWithBaseDir(self):
"""Tests the correctness of TempDirInChroot with a passed prefix."""
rm_check_dir = ''
chroot_tempdir = path_util.FromChrootPath('/tmp/some-prefix')
osutils.SafeMakedirs(chroot_tempdir)
with chroot_util.TempDirInChroot(base_dir='/tmp/some-prefix') as tempdir:
rm_check_dir = tempdir
self.assertExists(tempdir)
self.assertNotEqual(chroot_tempdir, tempdir)
self.assertStartsWith(tempdir, chroot_tempdir)
self.assertNotExists(rm_check_dir)
osutils.RmDir(chroot_tempdir)
def _RunTest(self, test, remote, log_directory_base, fail_directory_base):
"""Runs a test or a suite of tests on a given remote.
Runs a test target, whether an individual test or a suite of tests, with
'test_that'.
Args:
test: The test or suite to run.
remote: The hostname of the remote DUT.
log_directory_base: The base directory to store test logs. A sub directory
specific to this test will be created there.
fail_directory_base: The base directory to store test logs in case of a
test failure.
Returns:
test: Same as |test|. This is useful when the caller wants to correlate
results to the test name.
percent_passed: Pass rate.
test_report: Content of the test report generated by test_that.
"""
log_directory, _ = self._GetResultsDirectoryForTest(
test, log_directory_base, fail_directory_base)
log_directory_in_chroot = log_directory.rpartition('chroot')[2]
# Copy GCE key file in a temporary file inside the chroot and
# make sure to remove it before return.
with tempfile.NamedTemporaryFile(
dir=path_util.FromChrootPath('/tmp')) as gce_key_copy:
shutil.copy(self.json_key_file, gce_key_copy.name)
args = 'gce_project=%s gce_zone=%s gce_instance=%s gce_key_file=%s' % (
self.gce_context.project, self.gce_context.zone, self.instances[test],
path_util.ToChrootPath(gce_key_copy.name))
cmd = ['test_that', '-b', self.board, '--no-quickmerge',
'--results_dir=%s' % log_directory_in_chroot, remote, test,
'--args=%s' % args]
if self.ssh_private_key is not None:
cmd.append('--ssh_private_key=%s' %
path_util.ToChrootPath(self.ssh_private_key))
result = cros_build_lib.RunCommand(cmd, error_code_ok=True,
enter_chroot=True,
redirect_stdout=True,
cwd=constants.CROSUTILS_DIR)
percent_passed = self.ParseGeneratedTestOutput(result.output)
test_report = self._GetTestReport(log_directory)
# Returns the summarized test_report as it is more useful than the full
# output, plus the entire log will always be linked in the failure report.
return test, percent_passed, test_report
def _StageChrootFilesIntoDir(target_path, paths):
"""Install chroot files into a staging directory.
Args:
target_path: Path to the staging directory
paths: An iterable of absolute paths inside the chroot
"""
for path in paths:
chroot_path = path_util.FromChrootPath(path)
install_path = os.path.join(target_path, os.path.relpath(path, '/'))
install_parent = os.path.dirname(install_path)
osutils.SafeMakedirs(install_parent, 0o775)
shutil.copyfile(chroot_path, install_path)
shutil.copymode(chroot_path, install_path)
def TransferStatefulUpdate(self):
"""Transfer files for stateful update.
The stateful update bin and the corresponding payloads are copied to the
target remote device for stateful update.
Args:
payload: The payload for stateful update.
"""
stateful_update_bin = path_util.FromChrootPath(
self.STATEFUL_UPDATE_BIN)
self.device.CopyToWorkDir(stateful_update_bin)
logging.info('Copying stateful payload to device...')
payload = os.path.join(self.payload_dir, ds_wrapper.STATEFUL_FILENAME)
self.device.CopyToWorkDir(payload)
def PerformStage(self):
test_root_in_chroot = commands.CreateTestRoot(self._build_root)
test_root = path_util.FromChrootPath(test_root_in_chroot)
results_dir = os.path.join(test_root, 'results')
work_dir = os.path.join(test_root, 'workdir')
osutils.SafeMakedirsNonRoot(results_dir)
osutils.SafeMakedirsNonRoot(work_dir)
try:
self._PerformStage(work_dir, results_dir)
except:
logging.error(
_ERROR_MSG %
dict(test_name='MoblabVMTest', test_results='directory'))
raise
finally:
self._ArchiveTestResults(results_dir)
def _GetTastSspFiles(self):
"""Build out the paths to the tast SSP files.
Returns:
list[str] - The paths to the files.
"""
files = []
if cros_build_lib.IsInsideChroot():
files.extend(self._TAST_SSP_CHROOT_FILES)
else:
files.extend(path_util.FromChrootPath(x)
for x in self._TAST_SSP_CHROOT_FILES)
for filename in self._TAST_SSP_SOURCE_FILES:
files.append(os.path.join(constants.SOURCE_ROOT, filename))
return files
def _TempDirInChroot():
"""A context to create and use a tempdir inside the chroot.
Yields:
A host path (not chroot path) to a tempdir inside the chroot. This tempdir
is cleaned up when exiting the context.
"""
chroot_tempdir = cros_build_lib.RunCommand(
['mktemp', '-d'],
capture_output=True,
enter_chroot=True,
).output.strip()
tempdir = path_util.FromChrootPath(chroot_tempdir)
try:
yield tempdir
finally:
osutils.RmDir(tempdir, ignore_missing=True)
def TranslatedPathToLocalPath(translated_path, static_dir):
"""Convert the translated path to a local path to the image file.
Args:
translated_path: the translated xbuddy path
(e.g., peppy-release/R36-5760.0.0/chromiumos_image).
static_dir: The static directory used by the devserver.
Returns:
A local path to the image file.
"""
real_path = osutils.ExpandPath(os.path.join(static_dir, translated_path))
if os.path.exists(real_path):
return real_path
else:
return path_util.FromChrootPath(real_path)
def __init__(self, version_file=None, hooks_dir=None):
if version_file:
# We have one. Just here to skip the logic below since we don't need it.
default_version_file = None
elif cros_build_lib.IsInsideChroot():
# Use the absolute path since we're inside the chroot.
default_version_file = CHROOT_VERSION_FILE
else:
# Otherwise convert to the path outside the chroot.
default_version_file = path_util.FromChrootPath(
CHROOT_VERSION_FILE)
self._version_file = version_file or default_version_file
self._hooks_dir = hooks_dir or _CHROOT_VERSION_HOOKS_DIR
self._version = None
self._latest_version = None
self._hook_files = None
def TempDirInChroot(**kwargs):
"""A context to create and use a tempdir inside the chroot.
Args:
prefix: See tempfile.mkdtemp documentation.
base_dir: The directory to place the temporary directory in the chroot.
set_global: See osutils.TempDir documentation.
delete: See osutils.TempDir documentation.
sudo_rm: See osutils.TempDir documentation.
Yields:
A host path (not chroot path) to a tempdir inside the chroot. This tempdir
is cleaned up when exiting the context.
"""
base_dir = kwargs.pop('base_dir', '/tmp')
kwargs['base_dir'] = path_util.FromChrootPath(base_dir)
tempdir = osutils.TempDir(**kwargs)
yield tempdir.tempdir
def CreateAndUploadPayload(payload,
cache,
work_dir,
sign=True,
verify=True,
dry_run=False,
au_generator_uri=None):
"""Helper to create a PaygenPayloadLib instance and use it.
Args:
payload: An instance of utils.Payload describing the payload to generate.
cache: An instance of DownloadCache for retrieving files.
work_dir: A working directory that can hold scratch files. Will be cleaned
up when done, and won't interfere with other users. None for /tmp.
sign: Boolean saying if the payload should be signed (normally, you do).
verify: whether the payload should be verified (default: True)
dry_run: don't perform actual work
au_generator_uri: URI to override standard au_generator.zip rules.
"""
# We need to create a temp directory inside the chroot so be able to access
# from both inside and outside the chroot.
temp_dir = path_util.FromChrootPath(
cros_build_lib.RunCommand(['mktemp', '-d'],
capture_output=True,
enter_chroot=True).output.strip())
with osutils.TempDir(prefix='paygen_payload.',
base_dir=temp_dir) as gen_dir:
logging.info('* Starting payload generation')
start_time = datetime.datetime.now()
_PaygenPayload(payload,
cache,
gen_dir,
sign,
verify,
au_generator_uri,
dry_run=dry_run).Run()
end_time = datetime.datetime.now()
logging.info('* Finished payload generation in %s',
end_time - start_time)
def UpdateChromeEbuildAFDOFile(board, profiles):
"""Update chrome ebuild with the dictionary of {arch: afdo_file} pairs.
Modifies the Chrome ebuild to set the appropriate AFDO file for each
given architecture. Regenerates the associated Manifest file and
commits the new ebuild and Manifest.
Args:
board: board we are building Chrome for.
profiles: {arch: afdo_file} pairs to put into the ebuild.
These are profiles from selected benchmarks.
"""
# Find the Chrome ebuild file.
equery_prog = 'equery'
ebuild_prog = 'ebuild'
if board:
equery_prog += '-%s' % board
ebuild_prog += '-%s' % board
equery_cmd = [equery_prog, 'w', 'chromeos-chrome']
ebuild_file = cros_build_lib.run(equery_cmd,
enter_chroot=True,
stdout=True).output.rstrip()
# Patch the ebuild file with the names of the available afdo_files.
PatchChromeEbuildAFDOFile(ebuild_file, profiles)
# Also patch the 9999 ebuild. This is necessary because the uprev
# process starts from the 9999 ebuild file and then compares to the
# current version to see if the uprev is really necessary. We dont
# want the names of the available afdo_files to show as differences.
# It also allows developers to do USE=afdo_use when using the 9999
# ebuild.
ebuild_9999 = os.path.join(os.path.dirname(ebuild_file),
'chromeos-chrome-9999.ebuild')
PatchChromeEbuildAFDOFile(ebuild_9999, profiles)
UpdateManifest(ebuild_9999, ebuild_prog)
ebuild_dir = path_util.FromChrootPath(os.path.dirname(ebuild_file))
CommitIfChanged(ebuild_dir, 'Update profiles and manifests for Chrome.')
def _BuildTarball(self,
input_list,
tarball_output,
compressed=True,
**kwargs):
"""Tars and zips files and directories from input_list to tarball_output.
Args:
input_list: A list of files and directories to be archived.
tarball_output: Path of output tar archive file.
compressed: Whether or not the tarball should be compressed with pbzip2.
**kwargs: Keyword arguments to pass to CreateTarball.
Returns:
Return value of cros_build_lib.CreateTarball.
"""
for pathname in input_list:
if os.path.exists(os.path.join(self.archive_basedir, pathname)):
break
else:
# If any of them exist we can create an archive, but if none
# do then we need to stop. For now, since we either pass in a
# handful of directories we don't necessarily check, or actually
# search the filesystem for lots of files, this is far more
# efficient than building out a list of files that do exist.
return None
compressor = cros_build_lib.COMP_NONE
chroot = None
if compressed:
compressor = cros_build_lib.COMP_BZIP2
if not cros_build_lib.IsInsideChroot():
chroot = path_util.FromChrootPath('/')
return cros_build_lib.CreateTarball(tarball_output,
self.archive_basedir,
compression=compressor,
chroot=chroot,
inputs=input_list,
**kwargs)
def PatchChromeEbuildAFDOFile(ebuild_file, arch_profiles):
"""Patch the Chrome ebuild with the dictionary of {arch: afdo_file} pairs.
Args:
ebuild_file: path of the ebuild file within the chroot.
arch_profiles: {arch: afdo_file} pairs to put into the ebuild.
"""
original_ebuild = path_util.FromChrootPath(ebuild_file)
modified_ebuild = '%s.new' % original_ebuild
arch_patterns = {}
arch_repls = {}
arch_markers = {}
for arch in arch_profiles.keys():
arch_patterns[arch] = re.compile(CHROME_EBUILD_AFDO_EXP % arch)
arch_repls[arch] = CHROME_EBUILD_AFDO_REPL % arch_profiles[arch]
arch_markers[arch] = False
with open(original_ebuild, 'r') as original:
with open(modified_ebuild, 'w') as modified:
for line in original:
for arch in arch_profiles.keys():
matched = arch_patterns[arch].match(line)
if matched:
arch_markers[arch] = True
modified.write(arch_patterns[arch].sub(
arch_repls[arch], line))
break
else: # line without markers, just copy it.
modified.write(line)
for arch, found in arch_markers.iteritems():
if not found:
raise MissingAFDOMarkers(
'Chrome ebuild file does not have appropriate '
'AFDO markers for arch %s' % arch)
os.rename(modified_ebuild, original_ebuild)
def main(args):
opts = ParseArgs(args)
# Build up test suites.
loader = unittest.TestLoader()
loader.suiteClass = image_test_lib.ImageTestSuite
# We use a different prefix here so that unittest DO NOT pick up the
# image tests automatically because they depend on a proper environment.
loader.testMethodPrefix = 'Test'
all_tests = loader.loadTestsFromName('chromite.cros.test.image_test')
forgiving = image_test_lib.ImageTestSuite()
non_forgiving = image_test_lib.ImageTestSuite()
for suite in all_tests:
for test in suite.GetTests():
if test.IsForgiving():
forgiving.addTest(test)
else:
non_forgiving.addTest(test)
# Run them in the image directory.
runner = image_test_lib.ImageTestRunner()
runner.SetBoard(opts.board)
runner.SetResultDir(opts.test_results_root)
image_file = FindImage(opts.image_dir)
tmp_in_chroot = path_util.FromChrootPath('/tmp')
with osutils.TempDir(base_dir=tmp_in_chroot) as temp_dir:
with osutils.MountImageContext(image_file, temp_dir):
with osutils.ChdirContext(temp_dir):
# Run non-forgiving tests first so that exceptions in forgiving tests
# do not skip any required tests.
logging.info('Running NON-forgiving tests.')
result = runner.run(non_forgiving)
logging.info('Running forgiving tests.')
runner.run(forgiving)
if result and not result.wasSuccessful():
return 1
return 0
def _BuildTarball(self, input_list, tarball_output, compressed=True,
**kwargs):
"""Tars and zips files and directories from input_list to tarball_output.
Args:
input_list: A list of files and directories to be archived.
tarball_output: Path of output tar archive file.
compressed: Whether or not the tarball should be compressed with pbzip2.
**kwargs: Keyword arguments to pass to CreateTarball.
Returns:
Return value of cros_build_lib.CreateTarball.
"""
compressor = cros_build_lib.COMP_NONE
chroot = None
if compressed:
compressor = cros_build_lib.COMP_BZIP2
if not cros_build_lib.IsInsideChroot():
chroot = path_util.FromChrootPath('/')
return cros_build_lib.CreateTarball(
tarball_output, self.archive_basedir, compression=compressor,
chroot=chroot, inputs=input_list, **kwargs)
def _GetStatefulUpdateScript(self):
"""Returns the path to the stateful_update_bin on the target.
Returns:
(need_transfer, path):
need_transfer is True if stateful_update_bin is found in local path,
False if we directly use stateful_update_bin on the host.
path: If need_transfer is True, it represents the local path of
stateful_update_bin, and is used for further transferring. Otherwise,
it refers to the host path.
"""
# We attempt to load the local stateful update path in 2 different
# ways. If this doesn't exist, we attempt to use the Chromium OS
# Chroot path to the installed script. If all else fails, we use the
# stateful update script on the host.
stateful_update_path = path_util.FromChrootPath(
self.LOCAL_CHROOT_STATEFUL_UPDATE_PATH)
if not os.path.exists(stateful_update_path):
logging.warning(
'Could not find chroot stateful_update script in %s, '
'falling back to the client copy.', stateful_update_path)
stateful_update_path = os.path.join(
self._dev_dir, self.LOCAL_STATEFUL_UPDATE_FILENAME)
if os.path.exists(stateful_update_path):
logging.debug(
'Use stateful_update script in devserver path: %s',
stateful_update_path)
return True, stateful_update_path
logging.debug(
'Cannot find stateful_update script, will use the script '
'on the host')
return False, self.REMOTE_STATEFUL_UPDATE_PATH
else:
return True, stateful_update_path
def UpdateChromeEbuildAFDOFile(board, arch_profiles):
"""Update chrome ebuild with the dictionary of {arch: afdo_file} pairs.
Modifies the Chrome ebuild to set the appropriate AFDO file for each
given architecture. Regenerates the associated Manifest file and
commits the new ebuild and Manifest.
Args:
board: board we are building Chrome for.
arch_profiles: {arch: afdo_file} pairs to put into the ebuild.
"""
# Find the Chrome ebuild file.
equery_prog = 'equery'
ebuild_prog = 'ebuild'
if board:
equery_prog += '-%s' % board
ebuild_prog += '-%s' % board
equery_cmd = [equery_prog, 'w', 'chromeos-chrome']
ebuild_file = cros_build_lib.RunCommand(
equery_cmd, enter_chroot=True, redirect_stdout=True).output.rstrip()
# Patch the ebuild file with the names of the available afdo_files.
PatchChromeEbuildAFDOFile(ebuild_file, arch_profiles)
# Also patch the 9999 ebuild. This is necessary because the uprev
# process starts from the 9999 ebuild file and then compares to the
# current version to see if the uprev is really necessary. We dont
# want the names of the available afdo_files to show as differences.
# It also allows developers to do USE=afdo_use when using the 9999
# ebuild.
ebuild_9999 = os.path.join(os.path.dirname(ebuild_file),
'chromeos-chrome-9999.ebuild')
PatchChromeEbuildAFDOFile(ebuild_9999, arch_profiles)
# Regenerate the Manifest file.
ebuild_gs_dir = None
# If using the GS test location, pass this location to the
# chrome ebuild.
if AFDO_BASE_URL == AFDO_TEST_URL:
ebuild_gs_dir = {'AFDO_GS_DIRECTORY': AFDO_TEST_URL}
gen_manifest_cmd = [ebuild_prog, ebuild_file, 'manifest', '--force']
cros_build_lib.RunCommand(gen_manifest_cmd,
enter_chroot=True,
extra_env=ebuild_gs_dir,
print_cmd=True)
ebuild_dir = path_util.FromChrootPath(os.path.dirname(ebuild_file))
git.RunGit(ebuild_dir, ['add', 'Manifest'])
# Check if anything changed compared to the previous version.
mod_files = [
'Manifest',
os.path.basename(ebuild_file),
os.path.basename(ebuild_9999)
]
modifications = git.RunGit(ebuild_dir,
['status', '--porcelain', '--'] + mod_files,
capture_output=True,
print_cmd=True).output
if not modifications:
logging.info('AFDO info for the Chrome ebuild did not change. '
'Nothing to commit')
return
# If there are changes to ebuild or Manifest, commit them.
commit_msg = ('"Set {arch: afdo_file} pairs %s and updated Manifest"' %
arch_profiles)
git.RunGit(ebuild_dir, ['commit', '-m', commit_msg, '--'] + mod_files,
print_cmd=True)
import urlparse
from chromite.lib import constants
from chromite.cli import command
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import remote_access
from chromite.lib import timeout_util
DEFAULT_PORT = 8080
DEVSERVER_PKG_DIR = os.path.join(constants.SOURCE_ROOT, 'src/platform/dev')
DEFAULT_STATIC_DIR = path_util.FromChrootPath(
os.path.join(constants.SOURCE_ROOT, 'src', 'platform', 'dev', 'static'))
XBUDDY_REMOTE = 'remote'
XBUDDY_LOCAL = 'local'
ROOTFS_FILENAME = 'update.gz'
STATEFUL_FILENAME = 'stateful.tgz'
class ImagePathError(Exception):
"""Raised when the provided path can't be resolved to an image."""
def ConvertTranslatedPath(original_path, translated_path):
"""Converts a translated xbuddy path to an xbuddy path.
def __init__(self, goma_dir, goma_client_json, goma_tmp_dir=None,
stage_name=None):
"""Initializes Goma instance.
This ensures that |self.goma_log_dir| directory exists (if missing,
creates it).
Args:
goma_dir: Path to the goma directory (outside of chroot).
goma_client_json: Path to the service account json file to use goma.
On bots, this must be specified, otherwise raise a ValueError.
On local, this is optional, and can be set to None.
goma_tmp_dir: Path to the GOMA_TMP_DIR to be passed to goma programs.
If given, it is used. If not given, creates a directory under
/tmp in the chroot, expecting that the directory is removed in the
next run's clean up phase on bots.
stage_name: optional name of the currently running stage. E.g.
"build_packages" or "test_simple_chrome_workflow". If this is set
deps cache is enabled.
Raises:
ValueError if 1) |goma_dir| does not point to a directory, 2)
on bots, but |goma_client_json| is not given, 3) |goma_client_json|
is given, but it does not point to a file, or 4) if |goma_tmp_dir| is
given but it does not point to a directory.
"""
# Sanity checks of given paths.
if not os.path.isdir(goma_dir):
raise ValueError('goma_dir does not point a directory: %s' % (goma_dir,))
# If this script runs on bot, service account json file needs to be
# provided, otherwise it cannot access to goma service.
if cros_build_lib.HostIsCIBuilder() and goma_client_json is None:
raise ValueError(
'goma is enabled on bot, but goma_client_json is not provided')
# If goma_client_json file is provided, it must be an existing file.
if goma_client_json and not os.path.isfile(goma_client_json):
raise ValueError(
'Goma client json file is missing: %s' % (goma_client_json,))
# If goma_tmp_dir is provided, it must be an existing directory.
if goma_tmp_dir and not os.path.isdir(goma_tmp_dir):
raise ValueError(
'GOMA_TMP_DIR does not point a directory: %s' % (goma_tmp_dir,))
self.goma_dir = goma_dir
self.goma_client_json = goma_client_json
if stage_name:
self.goma_cache = os.path.join(goma_dir, 'goma_cache', stage_name)
osutils.SafeMakedirs(self.goma_cache)
else:
self.goma_cache = None
if goma_tmp_dir is None:
# If |goma_tmp_dir| is not given, create GOMA_TMP_DIR (goma
# compiler_proxy's working directory), and its log directory.
# Create unique directory by mkdtemp under chroot's /tmp.
# Expect this directory is removed in next run's clean up phase.
goma_tmp_dir = tempfile.mkdtemp(
prefix='goma_tmp_dir.', dir=path_util.FromChrootPath('/tmp'))
self.goma_tmp_dir = goma_tmp_dir
# Create log directory if not exist.
if not os.path.isdir(self.goma_log_dir):
os.mkdir(self.goma_log_dir)
import re
import shutil
import tempfile
from chromite.cbuildbot import constants
from chromite.lib import auto_updater
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import dev_server_wrapper as ds_wrapper
from chromite.lib import operation
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import remote_access
DEVSERVER_STATIC_DIR = path_util.FromChrootPath(
os.path.join(constants.CHROOT_SOURCE_ROOT, 'devserver', 'static'))
class UsbImagerOperation(operation.ProgressBarOperation):
"""Progress bar for flashing image to operation."""
def __init__(self, image):
super(UsbImagerOperation, self).__init__()
self._size = os.path.getsize(image)
self._transferred = 0.
self._bytes = re.compile(r'(\d+) bytes')
def _GetDDPid(self):
"""Get the Pid of dd."""
try:
pids = cros_build_lib.RunCommand(['pgrep', 'dd'],
capture_output=True,
