def testSkipCleanupGlobal(self):
"""Test that we reset global tempdir as expected even with skip."""
with osutils.TempDir(prefix=self.PREFIX, set_global=True) as tempdir:
tempdir_before = osutils.GetGlobalTempDir()
tempdir_obj = osutils.TempDir(prefix=self.PREFIX, set_global=True,
delete=False)
tempdir_inside = osutils.GetGlobalTempDir()
tempdir_obj.Cleanup()
tempdir_after = osutils.GetGlobalTempDir()
# We shouldn't leak the outer directory.
self.assertNotExists(tempdir)
self.assertEqual(tempdir_before, tempdir_after)
# This is a strict substring check.
self.assertLess(tempdir_before, tempdir_inside)
def __init__(self, opts):
"""Initialize VM.
Args:
opts: command line options.
"""
super(VM, self).__init__(opts)
self.qemu_path = opts.qemu_path
self.qemu_img_path = opts.qemu_img_path
self.qemu_bios_path = opts.qemu_bios_path
self.qemu_m = opts.qemu_m
self.qemu_cpu = opts.qemu_cpu
self.qemu_smp = opts.qemu_smp
if self.qemu_smp == 0:
self.qemu_smp = min(8, multiprocessing.cpu_count())
self.qemu_hostfwd = opts.qemu_hostfwd
self.qemu_args = opts.qemu_args
self.enable_kvm = opts.enable_kvm
self.copy_on_write = opts.copy_on_write
# We don't need sudo access for software emulation or if /dev/kvm is
# writeable.
self.use_sudo = self.enable_kvm and not os.access('/dev/kvm', os.W_OK)
self.display = opts.display
self.image_path = opts.image_path
self.image_format = opts.image_format
self.device = remote_access.LOCALHOST
self.ssh_port = opts.ssh_port
self.start = opts.start
self.stop = opts.stop
self.chroot_path = opts.chroot_path
self.cache_dir = os.path.abspath(opts.cache_dir)
assert os.path.isdir(self.cache_dir), "Cache directory doesn't exist"
self.vm_dir = opts.vm_dir
if not self.vm_dir:
self.vm_dir = os.path.join(osutils.GetGlobalTempDir(),
'cros_vm_%d' % self.ssh_port)
self._CreateVMDir()
self.pidfile = os.path.join(self.vm_dir, 'kvm.pid')
self.kvm_monitor = os.path.join(self.vm_dir, 'kvm.monitor')
self.kvm_pipe_in = '%s.in' % self.kvm_monitor # to KVM
self.kvm_pipe_out = '%s.out' % self.kvm_monitor # from KVM
self.kvm_serial = '%s.serial' % self.kvm_monitor
self.InitRemote()
def __init__(self, opts):
"""Initialize VM.
Args:
opts: command line options.
"""
self.qemu_path = opts.qemu_path
self.qemu_bios_path = opts.qemu_bios_path
self.qemu_m = opts.qemu_m
self.qemu_cpu = opts.qemu_cpu
self.qemu_smp = opts.qemu_smp
if self.qemu_smp == 0:
self.qemu_smp = min(8, multiprocessing.cpu_count)
self.enable_kvm = opts.enable_kvm
# We don't need sudo access for software emulation or if /dev/kvm is
# writeable.
self.use_sudo = self.enable_kvm and not os.access('/dev/kvm', os.W_OK)
self.display = opts.display
self.image_path = opts.image_path
self.image_format = opts.image_format
self.board = opts.board
self.ssh_port = opts.ssh_port
self.dry_run = opts.dry_run
self.start = opts.start
self.stop = opts.stop
self.cmd = opts.args[1:] if opts.cmd else None
self.cache_dir = os.path.abspath(opts.cache_dir)
assert os.path.isdir(self.cache_dir), "Cache directory doesn't exist"
self.vm_dir = opts.vm_dir
if not self.vm_dir:
self.vm_dir = os.path.join(osutils.GetGlobalTempDir(),
'cros_vm_%d' % self.ssh_port)
self._CreateVMDir()
self.pidfile = os.path.join(self.vm_dir, 'kvm.pid')
self.kvm_monitor = os.path.join(self.vm_dir, 'kvm.monitor')
self.kvm_pipe_in = '%s.in' % self.kvm_monitor # to KVM
self.kvm_pipe_out = '%s.out' % self.kvm_monitor # from KVM
self.kvm_serial = '%s.serial' % self.kvm_monitor
self.remote = remote_access.RemoteDevice(remote_access.LOCALHOST,
port=self.ssh_port)
self.device_addr = 'ssh://%s:%d' % (remote_access.LOCALHOST,
self.ssh_port)
def __init__(self, image_path=None, qemu_path=None, enable_kvm=True,
display=True, ssh_port=SSH_PORT, dry_run=False):
"""Initialize VM.
Args:
image_path: path of vm image.
qemu_path: path to qemu binary.
enable_kvm: enable kvm (kernel support for virtualization).
display: display video output.
ssh_port: ssh port to use.
dry_run: disable VM commands.
"""
self.qemu_path = qemu_path
self.enable_kvm = enable_kvm
# Software emulation doesn't need sudo access.
self.use_sudo = enable_kvm
self.display = display
self.image_path = image_path
self.ssh_port = ssh_port
self.dry_run = dry_run
self.vm_dir = os.path.join(osutils.GetGlobalTempDir(), 'cros_vm')
if os.path.exists(self.vm_dir):
# For security, ensure that vm_dir is not a symlink, and is owned by us or
# by root.
assert not os.path.islink(self.vm_dir), \
'VM state dir is misconfigured; please recreate: %s' % self.vm_dir
st_uid = os.stat(self.vm_dir).st_uid
assert st_uid == 0 or st_uid == os.getuid(), \
'VM state dir is misconfigured; please recreate: %s' % self.vm_dir
self.pidfile = os.path.join(self.vm_dir, 'kvm.pid')
self.kvm_monitor = os.path.join(self.vm_dir, 'kvm.monitor')
self.kvm_pipe_in = '%s.in' % self.kvm_monitor # to KVM
self.kvm_pipe_out = '%s.out' % self.kvm_monitor # from KVM
self.kvm_serial = '%s.serial' % self.kvm_monitor
def UploadReleaseProfiles(gs_context, run_id, merge_plan, merge_results):
"""Uploads the profiles in merge_results to our release profile bucket.
Args:
gs_context: Our GS context
run_id: A unique identifier for this run. Generally recommended to be the
number of seconds since the unix epoch, or something similarly difficult
to 'collide' with other runs. This is used in paths to guarantee
uniqueness.
merge_plan: The merge plan that generated the given |merge_results|. Only
used to write to a metadata file, so we know what went into this profile.
merge_results: A map describing the profiles to upload; you can get one
from ExecuteReleaseProfileMergePlan.
"""
gs_url_base = os.path.join(GSURL_BASE_RELEASE, run_id)
def copy_file_to_gs(local_path, remote_path):
# Note that version=0 implies that we'll never overwrite anything. If
# run_id is truly unique, this should never make a difference.
gs_context.Copy(local_path, remote_path, acl='public-read', version=0)
for version, profile in merge_results.items():
suffix = os.path.splitext(profile)[1]
assert suffix != '.afdo', 'All profiles should be compressed.'
output_path = os.path.join(gs_url_base,
'profiles/m%d.afdo%s' % (version, suffix))
copy_file_to_gs(profile, output_path)
# Write a map describing the profiles that have been uploaded. Not
# compressed, because it's expected to be <500 bytes. At the time of writing,
# no automated system relies on these; we just write them so it's easier to
# understand what 'gs://path/to/profiles/m75.afdo' actually consists of.
temp_dir = osutils.GetGlobalTempDir()
meta_file_path = os.path.join(temp_dir, 'meta.json')
osutils.WriteFile(meta_file_path, json.dumps(merge_plan))
copy_file_to_gs(meta_file_path, os.path.join(gs_url_base, 'meta.json'))
def _CreateParser():
"""Generate and return the parser with all the options."""
# Parse options
usage = 'usage: %prog [options] buildbot_config [buildbot_config ...]'
parser = CustomParser(usage=usage, caching=FindCacheDir)
# Main options
parser.add_remote_option(
'-b',
'--branch',
help='The manifest branch to test. The branch to '
'check the buildroot out to.')
parser.add_option(
'-r',
'--buildroot',
type='path',
dest='buildroot',
help='Root directory where source is checked out to, and '
'where the build occurs. For external build configs, '
"defaults to 'trybot' directory at top level of your "
'repo-managed checkout.')
parser.add_option('--bootstrap-dir',
type='path',
help='Bootstrapping cbuildbot may involve checking out '
'multiple copies of chromite. All these checkouts '
'will be contained in the directory specified here. '
'Default:%s' % osutils.GetGlobalTempDir())
parser.add_remote_option(
'--android_rev',
type='choice',
choices=constants.VALID_ANDROID_REVISIONS,
help=('Revision of Android to use, of type [%s]' %
'|'.join(constants.VALID_ANDROID_REVISIONS)))
parser.add_remote_option('--chrome_rev',
type='choice',
choices=constants.VALID_CHROME_REVISIONS,
help=('Revision of Chrome to use, of type [%s]' %
'|'.join(constants.VALID_CHROME_REVISIONS)))
parser.add_remote_option(
'--profile', help='Name of profile to sub-specify board variant.')
# TODO(crbug.com/279618): Running GOMA is under development. Following
# flags are added for development purpose due to repository dependency,
# but not officially supported yet.
parser.add_option('--goma_dir',
type='path',
api=constants.REEXEC_API_GOMA,
help='Specify a directory containing goma. When this is '
'set, GOMA is used to build Chrome.')
parser.add_option('--goma_client_json',
type='path',
api=constants.REEXEC_API_GOMA,
help='Specify a service-account-goma-client.json path. '
'The file is needed on bots to run GOMA.')
group = CustomGroup(parser, 'Deprecated Options')
parser.add_option('--local',
action='store_true',
default=False,
help='Deprecated. See cros tryjob.')
parser.add_option('--remote',
action='store_true',
default=False,
help='Deprecated. See cros tryjob.')
#
# Patch selection options.
#
group = CustomGroup(parser, 'Patch Options')
group.add_remote_option('-g',
'--gerrit-patches',
action='split_extend',
type='string',
default=[],
metavar="'Id1 *int_Id2...IdN'",
help='Space-separated list of short-form Gerrit '
"Change-Id's or change numbers to patch. "
"Please prepend '*' to internal Change-Id's")
parser.add_argument_group(group)
#
# Remote trybot options.
#
group = CustomGroup(parser, 'Options used to configure tryjob behavior.')
group.add_remote_option(
'--hwtest',
action='store_true',
default=False,
help='Run the HWTest stage (tests on real hardware)')
group.add_remote_option(
'--channel',
action='split_extend',
dest='channels',
default=[],
help='Specify a channel for a payloads trybot. Can '
'be specified multiple times. No valid for '
'non-payloads configs.')
parser.add_argument_group(group)
#
# Branch creation options.
#
group = CustomGroup(parser,
'Branch Creation Options (used with branch-util)')
group.add_remote_option('--branch-name',
help='The branch to create or delete.')
group.add_remote_option(
'--delete-branch',
action='store_true',
default=False,
help='Delete the branch specified in --branch-name.')
group.add_remote_option('--rename-to',
help='Rename a branch to the specified name.')
group.add_remote_option('--force-create',
action='store_true',
default=False,
help='Overwrites an existing branch.')
group.add_remote_option('--skip-remote-push',
action='store_true',
default=False,
help='Do not actually push to remote git repos. '
'Used for end-to-end testing branching.')
parser.add_argument_group(group)
#
# Advanced options.
#
group = CustomGroup(parser, 'Advanced Options',
'Caution: use these options at your own risk.')
group.add_remote_option(
'--bootstrap-args',
action='append',
default=[],
help='Args passed directly to the bootstrap re-exec '
'to skip verification by the bootstrap code')
group.add_remote_option('--buildbot',
action='store_true',
dest='buildbot',
default=False,
help='This is running on a buildbot. '
'This can be used to make a build operate '
'like an official builder, e.g. generate '
'new version numbers and archive official '
'artifacts and such. This should only be '
'used if you are confident in what you are '
'doing, as it will make automated commits.')
parser.add_remote_option(
'--repo-cache',
type='path',
dest='_repo_cache',
help='Present for backwards compatibility, ignored.')
group.add_remote_option('--no-buildbot-tags',
action='store_false',
dest='enable_buildbot_tags',
default=True,
help='Suppress buildbot specific tags from log '
'output. This is used to hide recursive '
'cbuilbot runs on the waterfall.')
group.add_remote_option('--buildnumber',
type='int',
default=0,
help='build number')
group.add_option('--chrome_root',
action='callback',
type='path',
callback=_CheckChromeRootOption,
help='Local checkout of Chrome to use.')
group.add_remote_option('--chrome_version',
action='callback',
type='string',
dest='chrome_version',
callback=_CheckChromeVersionOption,
help='Used with SPEC logic to force a particular '
'git revision of chrome rather than the '
'latest.')
group.add_remote_option('--clobber',
action='store_true',
default=False,
help='Clears an old checkout before syncing')
group.add_remote_option('--latest-toolchain',
action='store_true',
default=False,
help='Use the latest toolchain.')
parser.add_option('--log_dir',
dest='log_dir',
type='path',
help='Directory where logs are stored.')
group.add_remote_option('--maxarchives',
type='int',
dest='max_archive_builds',
default=3,
help='Change the local saved build count limit.')
parser.add_remote_option('--manifest-repo-url',
help='Overrides the default manifest repo url.')
group.add_remote_option('--compilecheck',
action='store_true',
default=False,
help='Only verify compilation and unit tests.')
group.add_remote_option('--noarchive',
action='store_false',
dest='archive',
default=True,
help="Don't run archive stage.")
group.add_remote_option('--nobootstrap',
action='store_false',
dest='bootstrap',
default=True,
help="Don't checkout and run from a standalone "
'chromite repo.')
group.add_remote_option('--nobuild',
action='store_false',
dest='build',
default=True,
help="Don't actually build (for cbuildbot dev)")
group.add_remote_option('--noclean',
action='store_false',
dest='clean',
default=True,
help="Don't clean the buildroot")
group.add_remote_option('--nocgroups',
action='store_false',
dest='cgroups',
default=True,
help='Disable cbuildbots usage of cgroups.')
group.add_remote_option('--nochromesdk',
action='store_false',
dest='chrome_sdk',
default=True,
help="Don't run the ChromeSDK stage which builds "
'Chrome outside of the chroot.')
group.add_remote_option('--noprebuilts',
action='store_false',
dest='prebuilts',
default=True,
help="Don't upload prebuilts.")
group.add_remote_option(
'--nopatch',
action='store_false',
dest='postsync_patch',
default=True,
help="Don't run PatchChanges stage. This does not "
'disable patching in of chromite patches '
'during BootstrapStage.')
group.add_remote_option('--nopaygen',
action='store_false',
dest='paygen',
default=True,
help="Don't generate payloads.")
group.add_remote_option(
'--noreexec',
action='store_false',
dest='postsync_reexec',
default=True,
help="Don't reexec into the buildroot after syncing.")
group.add_remote_option('--nosdk',
action='store_true',
default=False,
help='Re-create the SDK from scratch.')
group.add_remote_option('--nosync',
action='store_false',
dest='sync',
default=True,
help="Don't sync before building.")
group.add_remote_option('--notests',
action='store_false',
dest='tests',
default=True,
help='Override values from buildconfig, run no '
'tests, and build no autotest and artifacts.')
group.add_remote_option('--novmtests',
action='store_false',
dest='vmtests',
default=True,
help='Override values from buildconfig, run no '
'vmtests.')
group.add_remote_option('--noimagetests',
action='store_false',
dest='image_test',
default=True,
help='Override values from buildconfig and run no '
'image tests.')
group.add_remote_option('--nouprev',
action='store_false',
dest='uprev',
default=True,
help='Override values from buildconfig and never '
'uprev.')
group.add_option('--reference-repo',
help='Reuse git data stored in an existing repo '
'checkout. This can drastically reduce the network '
'time spent setting up the trybot checkout. By '
"default, if this option isn't given but cbuildbot "
'is invoked from a repo checkout, cbuildbot will '
'use the repo root.')
group.add_option('--resume',
action='store_true',
default=False,
help='Skip stages already successfully completed.')
group.add_remote_option('--timeout',
type='int',
default=0,
help='Specify the maximum amount of time this job '
'can run for, at which point the build will be '
'aborted. If set to zero, then there is no '
'timeout.')
group.add_remote_option(
'--version',
dest='force_version',
help='Used with manifest logic. Forces use of this '
'version rather than create or get latest. '
'Examples: 4815.0.0-rc1, 4815.1.2')
group.add_remote_option('--git-cache-dir',
type='path',
api=constants.REEXEC_API_GIT_CACHE_DIR,
help='Specify the cache directory to store the '
'project caches populated by the git-cache '
'tool. Bootstrap the projects based on the git '
'cache files instead of fetching them directly '
'from the GoB servers.')
group.add_remote_option('--sanity-check-build',
action='store_true',
default=False,
dest='sanity_check_build',
api=constants.REEXEC_API_SANITY_CHECK_BUILD,
help='Run the build as a sanity check build.')
group.add_remote_option('--debug-cidb',
action='store_true',
default=False,
help='Force Debug CIDB to be used.')
parser.add_argument_group(group)
#
# Internal options.
#
group = CustomGroup(
parser, 'Internal Chromium OS Build Team Options',
'Caution: these are for meant for the Chromium OS build team only')
group.add_remote_option('--archive-base',
type='gs_path',
help='Base GS URL (gs://<bucket_name>/<path>) to '
'upload archive artifacts to')
group.add_remote_option(
'--cq-gerrit-query',
dest='cq_gerrit_override',
help='If given, this gerrit query will be used to find what patches to '
"test, rather than the normal 'CommitQueue>=1 AND Verified=1 AND "
"CodeReview=2' query it defaults to. Use with care- note "
'additionally this setting only has an effect if the buildbot '
"target is a cq target, and we're in buildbot mode.")
group.add_option('--pass-through',
action='append',
type='string',
dest='pass_through_args',
default=[])
group.add_option('--reexec-api-version',
action='store_true',
dest='output_api_version',
default=False,
help='Used for handling forwards/backwards compatibility '
'with --resume and --bootstrap')
group.add_option(
'--remote-trybot',
action='store_true',
default=False,
help='Indicates this is running on a remote trybot machine')
group.add_option(
'--buildbucket-id',
api=constants.REEXEC_API_GOMA, # Approximate.
help='The unique ID in buildbucket of current build '
'generated by buildbucket.')
group.add_remote_option('--remote-patches',
action='split_extend',
default=[],
help='Patches uploaded by the trybot client when '
'run using the -p option')
# Note the default here needs to be hardcoded to 3; that is the last version
# that lacked this functionality.
group.add_option('--remote-version',
type='int',
default=3,
help='Deprecated and ignored.')
group.add_option('--sourceroot',
type='path',
default=constants.SOURCE_ROOT)
group.add_option('--ts-mon-task-num',
type='int',
default=0,
api=constants.REEXEC_API_TSMON_TASK_NUM,
help='The task number of this process. Defaults to 0. '
'This argument is useful for running multiple copies '
'of cbuildbot without their metrics colliding.')
group.add_remote_option('--test-bootstrap',
action='store_true',
default=False,
help='Causes cbuildbot to bootstrap itself twice, '
'in the sequence A->B->C: A(unpatched) patches '
'and bootstraps B; B patches and bootstraps C')
group.add_remote_option('--validation_pool',
help='Path to a pickled validation pool. Intended '
'for use only with the commit queue.')
group.add_remote_option('--metadata_dump',
help='Path to a json dumped metadata file. This '
'will be used as the initial metadata.')
group.add_remote_option('--master-build-id',
type='int',
api=constants.REEXEC_API_MASTER_BUILD_ID,
help='cidb build id of the master build to this '
'slave build.')
group.add_remote_option(
'--mock-tree-status',
help='Override the tree status value that would be '
'returned from the the actual tree. Example '
'values: open, closed, throttled. When used '
'in conjunction with --debug, the tree status '
'will not be ignored as it usually is in a '
'--debug run.')
# TODO(nxia): crbug.com/778838
# cbuildbot doesn't use pickle files anymore, remove this.
group.add_remote_option(
'--mock-slave-status',
metavar='MOCK_SLAVE_STATUS_PICKLE_FILE',
help='Override the result of the _FetchSlaveStatuses '
'method of MasterSlaveSyncCompletionStage, by '
'specifying a file with a pickle of the result '
'to be returned.')
group.add_option(
'--previous-build-state',
type='string',
default='',
api=constants.REEXEC_API_PREVIOUS_BUILD_STATE,
help='A base64-encoded BuildSummary object describing the '
'previous build run on the same build machine.')
parser.add_argument_group(group)
#
# Debug options
#
# Temporary hack; in place till --dry-run replaces --debug.
# pylint: disable=W0212
group = parser.debug_group
debug = [x for x in group.option_list if x._long_opts == ['--debug']][0]
debug.help += ' Currently functions as --dry-run in addition.'
debug.pass_through = True
group.add_option(
'--notee',
action='store_false',
dest='tee',
default=True,
help='Disable logging and internal tee process. Primarily '
'used for debugging cbuildbot itself.')
return parser
