def GetUpdatePayloadsFromLocalPath(path,
payload_dir,
src_image_to_delta=None,
static_dir=DEFAULT_STATIC_DIR):
"""Generates update payloads from a local image path.
This function wraps around ConvertLocalPathToXbuddy and GetUpdatePayloads,
managing the creation and destruction of the necessary temporary directories
required by this process.
Args:
path: Path to an image.
payload_dir: The directory to store the payloads. On failure, the devserver
log will be copied to |payload_dir|.
src_image_to_delta: Image used as the base to generate the delta payloads.
static_dir: Devserver static dir to use.
"""
with cros_build_lib.ContextManagerStack() as stack:
image_tempdir = stack.Add(osutils.TempDir,
base_dir=path_util.FromChrootPath('/tmp'),
prefix='dev_server_wrapper_local_image',
sudo_rm=True)
static_tempdir = stack.Add(osutils.TempDir,
base_dir=static_dir,
prefix='local_image',
sudo_rm=True)
xbuddy_path = ConvertLocalPathToXbuddyPath(path, image_tempdir,
static_tempdir, static_dir)
GetUpdatePayloads(xbuddy_path,
payload_dir,
src_image_to_delta=src_image_to_delta,
static_dir=static_dir)
def main(argv):
with cros_build_lib.ContextManagerStack() as stack:
router = router_lib.GetRouter()
opts = _ParseArgs(argv, router)
if opts.tee_log:
stack.Add(tee.Tee, opts.tee_log)
logging.info('Teeing stdout and stderr to %s', opts.tee_log)
if opts.config.log_path:
stack.Add(tee.Tee, opts.config.log_path)
logging.info('Teeing stdout and stderr to %s',
opts.config.log_path)
tee_log_env_value = os.environ.get('BUILD_API_TEE_LOG_FILE')
if tee_log_env_value:
stack.Add(tee.Tee, tee_log_env_value)
logging.info('Teeing stdout and stderr to env path %s',
tee_log_env_value)
if opts.config.mock_invalid:
# --mock-invalid handling. We print error messages, but no output is ever
# set for validation errors, so we can handle it by just giving back the
# correct return code here.
return controller.RETURN_CODE_INVALID_INPUT
input_handler, output_handlers = _get_io_handlers(opts)
try:
return router.Route(opts.service, opts.method, opts.config,
input_handler, output_handlers,
opts.config_handler)
except router_lib.Error as e:
# Handle router_lib.Error derivatives nicely, but let anything else bubble
# up.
cros_build_lib.Die(e)
def test(self):
invoked = []
counter = iter(itertools.count()).next
def _mk_kls(has_exception=None, exception_kls=None, suppress=False):
class foon(object):
marker = counter()
def __enter__(self):
return self
# pylint: disable=E0213
def __exit__(obj_self, exc_type, exc, traceback):
invoked.append(obj_self.marker)
if has_exception is not None:
self.assertTrue(all(x is not None
for x in (exc_type, exc, traceback)))
self.assertTrue(exc_type == has_exception)
if exception_kls:
raise exception_kls()
if suppress:
return True
return foon
with cros_build_lib.ContextManagerStack() as stack:
# Note... these tests are in reverse, since the exception
# winds its way up the stack.
stack.Add(_mk_kls())
stack.Add(_mk_kls(ValueError, suppress=True))
stack.Add(_mk_kls(IndexError, exception_kls=ValueError))
stack.Add(_mk_kls(IndexError))
stack.Add(_mk_kls(exception_kls=IndexError))
stack.Add(_mk_kls())
self.assertEqual(invoked, list(reversed(range(6))))
def main(argv):
parser = GetParser()
opts = parser.parse_args(argv)
opts.Freeze()
# Process list output quickly as it takes no privileges.
if opts.list:
print('\n'.join(sorted(opts.tests or FindTests((constants.CHROMITE_DIR,)))))
return
# Many of our tests require a valid chroot to run. Make sure it's created
# before we block network access.
chroot = os.path.join(constants.SOURCE_ROOT, constants.DEFAULT_CHROOT_DIR)
if (not os.path.exists(chroot) and
ChrootAvailable() and
not cros_build_lib.IsInsideChroot()):
cros_build_lib.RunCommand(['cros_sdk', '--create'])
# This is a cheesy hack to make sure gsutil is populated in the cache before
# we run tests. This is a partial workaround for crbug.com/468838.
gs.GSContext.GetDefaultGSUtilBin()
# Now let's run some tests.
_ReExecuteIfNeeded([sys.argv[0]] + argv, opts.network)
# A lot of pieces here expect to be run in the root of the chromite tree.
# Make them happy.
os.chdir(constants.CHROMITE_DIR)
tests = opts.tests or FindTests()
if opts.quick:
SPECIAL_TESTS.update(SLOW_TESTS)
global TIMING_CACHE_FILE # pylint: disable=global-statement
TIMING_CACHE_FILE = os.path.join(
path_util.GetCacheDir(), constants.COMMON_CACHE, 'run_tests.cache.json')
jobs = opts.jobs or multiprocessing.cpu_count()
with cros_build_lib.ContextManagerStack() as stack:
# If we're running outside the chroot, try to contain ourselves.
if cgroups.Cgroup.IsSupported() and not cros_build_lib.IsInsideChroot():
stack.Add(cgroups.SimpleContainChildren, 'run_tests')
# Throw all the tests into a custom tempdir so that if we do CTRL+C, we can
# quickly clean up all the files they might have left behind.
stack.Add(osutils.TempDir, prefix='chromite.run_tests.', set_global=True,
sudo_rm=True)
with cros_build_lib.TimedSection() as timer:
result = RunTests(
tests, jobs=jobs, chroot_available=ChrootAvailable(),
network=opts.network, dryrun=opts.dryrun, failfast=opts.failfast)
if result:
logging.info('All tests succeeded! (%s total)', timer.delta)
else:
return 1
if not opts.network:
logging.warning('Network tests skipped; use --network to run them')
def main(argv):
with cros_build_lib.ContextManagerStack() as stack:
router = router_lib.GetRouter()
opts = _ParseArgs(argv, router)
if opts.tee_log:
stack.Add(tee.Tee, opts.tee_log)
logging.info('Teeing stdout and stderr to %s', opts.tee_log)
if opts.mock_invalid:
# --mock-invalid handling. We print error messages, but no output is ever
# set for validation errors, so we can handle it by just giving back the
# correct return code here.
return controller.RETURN_CODE_INVALID_INPUT
try:
return router.Route(opts.service, opts.method, opts.input_json,
opts.output_json, opts.config)
except router_lib.Error as e:
# Handle router_lib.Error derivatives nicely, but let anything else bubble
# up.
cros_build_lib.Die(e)
def test(self):
invoked = []
counter = functools.partial(next, itertools.count())
def _mk_kls(has_exception=None, exception_kls=None, suppress=False):
class foon(object):
"""Simple context manager which runs checks on __exit__."""
marker = counter()
def __enter__(self):
return self
# pylint: disable=no-self-argument,bad-context-manager
def __exit__(obj_self, exc_type, exc, traceback):
invoked.append(obj_self.marker)
if has_exception is not None:
self.assertTrue(
all(x is not None
for x in (exc_type, exc, traceback)))
self.assertTrue(exc_type == has_exception)
if exception_kls:
raise exception_kls()
if suppress:
return True
return foon
with cros_build_lib.ContextManagerStack() as stack:
# Note... these tests are in reverse, since the exception
# winds its way up the stack.
stack.Add(_mk_kls())
stack.Add(_mk_kls(ValueError, suppress=True))
stack.Add(_mk_kls(IndexError, exception_kls=ValueError))
stack.Add(_mk_kls(IndexError))
stack.Add(_mk_kls(exception_kls=IndexError))
stack.Add(_mk_kls())
self.assertEqual(invoked, list(range(5, -1, -1)))
def testCLStatsEngineSummary(self):
with cros_build_lib.ContextManagerStack() as stack:
self._PopulateFakeCidbWithTestData(cq=False)
self._PopulateFakeCidbWithTestData(cq=True)
stack.Add(mock.patch.object, summarize_build_stats.CLStatsEngine,
'GatherBuildAnnotations')
cl_stats = summarize_build_stats.CLStatsEngine(self.fake_db)
cl_stats.Gather(datetime.date.today(), datetime.date.today())
cl_stats.reasons = {
1: '',
2: '',
3: constants.FAILURE_CATEGORY_BAD_CL,
4: constants.FAILURE_CATEGORY_BAD_CL
}
cl_stats.blames = {
1: '',
2: '',
3: 'crosreview.com/1',
4: 'crosreview.com/1'
}
summary = cl_stats.Summarize('cq')
expected = {
'mean_good_patch_rejections': 0.5,
'unique_patches': 7,
'unique_blames_change_count': 0,
'total_cl_actions': 28,
'good_patch_rejection_breakdown': [(0, 3), (1, 0), (2, 1)],
'good_patch_rejection_count': {
CQ: 1,
PRE_CQ: 1
},
'good_patch_rejections': 2,
'false_rejection_rate': {
CQ: 20.,
PRE_CQ: 20.,
'combined': 100. / 3
},
'long_pole_slave_counts': {},
'submitted_patches': 4,
'submit_fails': 0,
'unique_cls': 4,
'median_handling_time':
-1, # This will be ignored in comparison
'patch_handling_time':
-1, # This will be ignored in comparison
'bad_cl_candidates': {
CQ: [
metadata_lib.GerritChangeTuple(gerrit_number=2,
internal=True)
],
PRE_CQ: [
metadata_lib.GerritChangeTuple(gerrit_number=4,
internal=True),
metadata_lib.GerritChangeTuple(gerrit_number=2,
internal=True)
],
},
'rejections': 10,
'total_builds': 5,
'first_build_num': 1,
'last_build_num': 4,
'last_build_id': mock.ANY,
'cl_handling_time_50': 0.0,
'cl_handling_time_90': 0.0,
'cq_time_50': 0.0,
'cq_time_90': 0.0,
'wait_time_50': 0.0,
'wait_time_90': 0.0,
'slowest_cq_slaves': [],
'patch_flake_rejections': 1,
'bad_cl_precq_rejected': 2,
'false_rejection_total': 2,
'false_rejection_pre_cq': 1,
'false_rejection_cq': 1,
#'build_blame_counts': {},
#'patch_blame_counts': {},
}
# Ignore handling times in comparison, since these are not fully
# reproducible from run to run of the unit test.
summary['median_handling_time'] = expected['median_handling_time']
summary['patch_handling_time'] = expected['patch_handling_time']
self.maxDiff = None
self.assertDictContainsSubset(expected, summary)
def Copy(self,
src_path,
dest_path,
acl=None,
recursive=False,
skip_symlinks=True,
auto_compress=False,
**kwargs):
"""Copy to/from GS bucket.
Canned ACL permissions can be specified on the gsutil cp command line.
More info:
https://developers.google.com/storage/docs/accesscontrol#applyacls
Args:
src_path: Fully qualified local path or full gs:// path of the src file.
dest_path: Fully qualified local path or full gs:// path of the dest
file.
acl: One of the google storage canned_acls to apply.
recursive: Whether to copy recursively.
skip_symlinks: Skip symbolic links when copying recursively.
auto_compress: Automatically compress with gzip when uploading.
Returns:
The generation of the remote file.
Raises:
RunCommandError if the command failed despite retries.
"""
# -v causes gs://bucket/path#generation to be listed in output.
cmd = ['cp', '-v']
# Certain versions of gsutil (at least 4.3) assume the source of a copy is
# a directory if the -r option is used. If it's really a file, gsutil will
# look like it's uploading it but not actually do anything. We'll work
# around that problem by surpressing the -r flag if we detect the source
# is a local file.
if recursive and not os.path.isfile(src_path):
cmd.append('-r')
if skip_symlinks:
cmd.append('-e')
if auto_compress:
# Pass the suffix without the '.' as that is what gsutil wants.
suffix = os.path.splitext(src_path)[1]
if not suffix:
raise ValueError(
'src file "%s" needs an extension to compress' %
(src_path, ))
cmd += ['-z', suffix[1:]]
acl = self.acl if acl is None else acl
if acl is not None:
cmd += ['-a', acl]
with cros_build_lib.ContextManagerStack() as stack:
# Write the input into a tempfile if possible. This is needed so that
# gsutil can retry failed requests.
if src_path == '-' and kwargs.get('input') is not None:
f = stack.Add(tempfile.NamedTemporaryFile)
f.write(kwargs['input'])
f.flush()
del kwargs['input']
src_path = f.name
cmd += ['--', src_path, dest_path]
if not (PathIsGs(src_path) or PathIsGs(dest_path)):
# Don't retry on local copies.
kwargs.setdefault('retries', 0)
kwargs['capture_output'] = True
try:
result = self.DoCommand(cmd, **kwargs)
if self.dry_run:
return None
# Now we parse the output for the current generation number. Example:
# Created: gs://chromeos-throw-away-bucket/foo#1360630664537000.1
m = re.search(r'Created: .*#(\d+)([.](\d+))?$', result.error)
if m:
return int(m.group(1))
else:
return None
except GSNoSuchKey:
# If the source was a local file, the error is a quirk of gsutil 4.5
# and should be ignored. If the source was remote, there might
# legitimately be no such file. See crbug.com/393419.
if os.path.isfile(src_path):
return None
raise
def main(argv):
parser = GetParser()
opts = parser.parse_args(argv)
opts.Freeze()
# Process list output quickly as it takes no privileges.
if opts.list:
tests = set(opts.tests or FindTests((constants.CHROMITE_DIR, )))
print('\n'.join(sorted(tests)))
return
# Many of our tests require a valid chroot to run. Make sure it's created
# before we block network access.
chroot = os.path.join(constants.SOURCE_ROOT, constants.DEFAULT_CHROOT_DIR)
if (not os.path.exists(chroot) and ChrootAvailable()
and not cros_build_lib.IsInsideChroot()):
cros_build_lib.run(['cros_sdk', '--create'])
# This is a cheesy hack to make sure gsutil is populated in the cache before
# we run tests. This is a partial workaround for crbug.com/468838.
gs.GSContext.GetDefaultGSUtilBin()
# Now let's run some tests.
_ReExecuteIfNeeded([sys.argv[0]] + argv, opts.network)
# A lot of pieces here expect to be run in the root of the chromite tree.
# Make them happy.
os.chdir(constants.CHROMITE_DIR)
tests = opts.tests or FindTests()
# Clear python caches now that we're root, in the right dir, but before we
# run any tests.
ClearPythonCacheFiles()
if opts.clear_pycache:
return
# Sanity check the environment. https://crbug.com/1015450
st = os.stat('/')
if st.st_mode & 0o7777 != 0o755:
cros_build_lib.Die('The root directory has broken permissions: %o\n'
'Fix with: sudo chmod 755 /' % (st.st_mode, ))
if st.st_uid or st.st_gid:
cros_build_lib.Die('The root directory has broken ownership: %i:%i'
' (should be 0:0)\nFix with: sudo chown 0:0 /' %
(st.st_uid, st.st_gid))
# Sanity check the settings to avoid bitrot.
CheckStaleSettings()
if opts.quick:
SPECIAL_TESTS.update(SLOW_TESTS)
global TIMING_CACHE_FILE # pylint: disable=global-statement
TIMING_CACHE_FILE = os.path.join(path_util.GetCacheDir(),
constants.COMMON_CACHE,
'run_tests.cache.json')
jobs = opts.jobs or multiprocessing.cpu_count()
with cros_build_lib.ContextManagerStack() as stack:
# If we're running outside the chroot, try to contain ourselves.
if cgroups.Cgroup.IsSupported(
) and not cros_build_lib.IsInsideChroot():
stack.Add(cgroups.SimpleContainChildren, 'run_tests')
# Throw all the tests into a custom tempdir so that if we do CTRL+C, we can
# quickly clean up all the files they might have left behind.
stack.Add(osutils.TempDir,
prefix='chromite.run_tests.',
set_global=True,
sudo_rm=True)
with cros_build_lib.TimedSection() as timer:
result = RunTests(tests,
jobs=jobs,
chroot_available=ChrootAvailable(),
network=opts.network,
dryrun=opts.dryrun,
failfast=opts.failfast,
pyver=opts.pyver)
if result:
logging.info('All tests succeeded! (%s total)', timer.delta)
else:
return 1
if not opts.network:
logging.warning('Network tests skipped; use --network to run them')
def BackgroundTaskRunner(task, *args, **kwargs):
"""Run the specified task on each queued input in a pool of processes.
This context manager starts a set of workers in the background, who each
wait for input on the specified queue. For each input on the queue, these
workers run task(*args + *input, **kwargs). Note that certain kwargs will
not pass through to the task (see Args below for the list).
The output from these tasks is saved to a temporary file. When control
returns to the context manager, the background output is printed in order,
as if the tasks were run in sequence.
If exceptions occur in the steps, we join together the tracebacks and print
them after all parallel tasks have finished running. Further, a
BackgroundFailure is raised with full stack traces of all exceptions.
Example:
# This will run somefunc(1, 'small', 'cow', foo='bar') in the background
# as soon as data is added to the queue (i.e. queue.put() is called).
def somefunc(arg1, arg2, arg3, foo=None):
...
with BackgroundTaskRunner(somefunc, 1, foo='bar') as queue:
... do random stuff ...
queue.put(['small', 'cow'])
... do more random stuff while somefunc() runs ...
# Exiting the with statement will block until all calls have completed.
Args:
task: Function to run on each queued input.
queue: A queue of tasks to run. Add tasks to this queue, and they will
be run in the background. If None, one will be created on the fly.
processes: Number of processes to launch.
onexit: Function to run in each background process after all inputs are
processed.
halt_on_error: After the first exception occurs, halt any running steps, and
squelch any further output, including any exceptions that might occur.
Halts on exceptions in any of the background processes, or in the
foreground process using the BackgroundTaskRunner.
"""
queue = kwargs.pop('queue', None)
processes = kwargs.pop('processes', None)
onexit = kwargs.pop('onexit', None)
halt_on_error = kwargs.pop('halt_on_error', False)
with cros_build_lib.ContextManagerStack() as stack:
if queue is None:
manager = stack.Add(Manager)
queue = manager.Queue()
if not processes:
processes = multiprocessing.cpu_count()
child = functools.partial(_BackgroundTask.TaskRunner, queue, task,
onexit=onexit, task_args=args,
task_kwargs=kwargs)
steps = [child] * processes
with _BackgroundTask.ParallelTasks(steps, halt_on_error=halt_on_error):
try:
yield queue
finally:
for _ in xrange(processes):
queue.put(_AllTasksComplete())
def RunParallelSteps(steps, max_parallel=None, halt_on_error=False,
return_values=False):
"""Run a list of functions in parallel.
This function blocks until all steps are completed.
The output from the functions is saved to a temporary file and printed as if
they were run in sequence.
If exceptions occur in the steps, we join together the tracebacks and print
them after all parallel tasks have finished running. Further, a
BackgroundFailure is raised with full stack traces of all exceptions.
Args:
steps: A list of functions to run.
max_parallel: The maximum number of simultaneous tasks to run in parallel.
By default, run all tasks in parallel.
halt_on_error: After the first exception occurs, halt any running steps,
and squelch any further output, including any exceptions that might occur.
return_values: If set to True, RunParallelSteps returns a list containing
the return values of the steps. Defaults to False.
Returns:
If |return_values| is True, the function will return a list containing the
return values of the steps.
Example:
# This snippet will execute in parallel:
# somefunc()
# anotherfunc()
# funcfunc()
steps = [somefunc, anotherfunc, funcfunc]
RunParallelSteps(steps)
# Blocks until all calls have completed.
"""
def ReturnWrapper(queue, fn):
"""Put the return value of |fn| into |queue|."""
queue.put(fn())
full_steps = []
queues = []
with cros_build_lib.ContextManagerStack() as stack:
if return_values:
# We use a managed queue here, because the child process will wait for the
# queue(pipe) to be flushed (i.e., when items are read from the queue)
# before exiting, and with a regular queue this may result in hangs for
# large return values. But with a managed queue, the manager process will
# read the items and hold on to them until the managed queue goes out of
# scope and is cleaned up.
manager = stack.Add(Manager)
for step in steps:
queue = manager.Queue()
queues.append(queue)
full_steps.append(functools.partial(ReturnWrapper, queue, step))
else:
full_steps = steps
with _BackgroundTask.ParallelTasks(full_steps, max_parallel=max_parallel,
halt_on_error=halt_on_error):
pass
if return_values:
return [queue.get_nowait() for queue in queues]
def main(argv):
# We get false positives with the options object.
# pylint: disable=attribute-defined-outside-init
# Turn on strict sudo checks.
cros_build_lib.STRICT_SUDO = True
# Set umask to 022 so files created by buildbot are readable.
os.umask(0o22)
parser = _CreateParser()
options = ParseCommandLine(parser, argv)
# Fetch our site_config now, because we need it to do anything else.
site_config = config_lib.GetConfig()
_PostParseCheck(parser, options, site_config)
cros_build_lib.AssertOutsideChroot()
if options.enable_buildbot_tags:
logging.EnableBuildbotMarkers()
if (options.buildbot and not options.debug
and not options.build_config_name == constants.BRANCH_UTIL_CONFIG
and not cros_build_lib.HostIsCIBuilder()):
# --buildbot can only be used on a real builder, unless it's debug, or
# 'branch-util'.
cros_build_lib.Die('This host is not a supported build machine.')
# Only one config arg is allowed in this mode, which was confirmed earlier.
build_config = site_config[options.build_config_name]
# TODO: Re-enable this block when reference_repo support handles this
# properly. (see chromium:330775)
# if options.reference_repo is None:
# repo_path = os.path.join(options.sourceroot, '.repo')
# # If we're being run from a repo checkout, reuse the repo's git pool to
# # cut down on sync time.
# if os.path.exists(repo_path):
# options.reference_repo = options.sourceroot
if options.reference_repo:
if not os.path.exists(options.reference_repo):
parser.error('Reference path %s does not exist' %
(options.reference_repo, ))
elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
parser.error('Reference path %s does not look to be the base of a '
'repo checkout; no .repo exists in the root.' %
(options.reference_repo, ))
if (options.buildbot or options.remote_trybot) and not options.resume:
if not options.cgroups:
parser.error(
'Options --buildbot/--remote-trybot and --nocgroups cannot '
'be used together. Cgroup support is required for '
'buildbot/remote-trybot mode.')
if not cgroups.Cgroup.IsSupported():
parser.error(
'Option --buildbot/--remote-trybot was given, but this '
'system does not support cgroups. Failing.')
missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
if missing:
parser.error(
'Option --buildbot/--remote-trybot requires the following '
"binaries which couldn't be found in $PATH: %s" %
(', '.join(missing)))
if options.reference_repo:
options.reference_repo = os.path.abspath(options.reference_repo)
# Sanity check of buildroot- specifically that it's not pointing into the
# midst of an existing repo since git-repo doesn't support nesting.
if (not repository.IsARepoRoot(options.buildroot)
and git.FindRepoDir(options.buildroot)):
cros_build_lib.Die(
'Configured buildroot %s is a subdir of an existing repo checkout.'
% options.buildroot)
if not options.log_dir:
options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
log_file = None
if options.tee:
log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
osutils.SafeMakedirs(options.log_dir)
_BackupPreviousLog(log_file)
with cros_build_lib.ContextManagerStack() as stack:
options.preserve_paths = set()
if log_file is not None:
# We don't want the critical section to try to clean up the tee process,
# so we run Tee (forked off) outside of it. This prevents a deadlock
# because the Tee process only exits when its pipe is closed, and the
# critical section accidentally holds on to that file handle.
stack.Add(tee.Tee, log_file)
options.preserve_paths.add(_DEFAULT_LOG_DIR)
critical_section = stack.Add(cleanup.EnforcedCleanupSection)
stack.Add(sudo.SudoKeepAlive)
if not options.resume:
# If we're in resume mode, use our parents tempdir rather than
# nesting another layer.
stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True)
logging.debug('Cbuildbot tempdir is %r.', os.environ.get('TMP'))
if options.cgroups:
stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')
# Mark everything between EnforcedCleanupSection and here as having to
# be rolled back via the contextmanager cleanup handlers. This
# ensures that sudo bits cannot outlive cbuildbot, that anything
# cgroups would kill gets killed, etc.
stack.Add(critical_section.ForkWatchdog)
if options.mock_tree_status is not None:
stack.Add(_ObjectMethodPatcher,
tree_status,
'_GetStatus',
return_value=options.mock_tree_status)
if options.mock_slave_status is not None:
with open(options.mock_slave_status, 'r') as f:
mock_statuses = pickle.load(f)
for key, value in mock_statuses.iteritems():
mock_statuses[key] = builder_status_lib.BuilderStatus(
**value)
stack.Add(_ObjectMethodPatcher,
completion_stages.MasterSlaveSyncCompletionStage,
'_FetchSlaveStatuses',
return_value=mock_statuses)
stack.Add(_SetupConnections, options, build_config)
retry_stats.SetupStats()
timeout_display_message = None
# For master-slave builds: Update slave's timeout using master's published
# deadline.
if options.buildbot and options.master_build_id is not None:
slave_timeout = None
if cidb.CIDBConnectionFactory.IsCIDBSetup():
cidb_handle = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder(
)
if cidb_handle:
slave_timeout = cidb_handle.GetTimeToDeadline(
options.master_build_id)
if slave_timeout is not None:
# We artificially set a minimum slave_timeout because '0' is handled
# specially, and because we don't want to timeout while trying to set
# things up.
slave_timeout = max(slave_timeout, 20)
if options.timeout == 0 or slave_timeout < options.timeout:
logging.info(
'Updating slave build timeout to %d seconds enforced '
'by the master', slave_timeout)
options.timeout = slave_timeout
timeout_display_message = (
'This build has reached the timeout deadline set by the master. '
'Either this stage or a previous one took too long (see stage '
'timing historical summary in ReportStage) or the build failed '
'to start on time.')
else:
logging.warning(
'Could not get master deadline for master-slave build. '
'Can not set slave timeout.')
if options.timeout > 0:
stack.Add(timeout_util.FatalTimeout, options.timeout,
timeout_display_message)
try:
_RunBuildStagesWrapper(options, site_config, build_config)
except failures_lib.ExitEarlyException as ex:
# This build finished successfully. Do not re-raise ExitEarlyException.
logging.info('One stage exited early: %s', ex)
def patches(*args):
"""Context manager for a list of patch objects."""
with cros_build_lib.ContextManagerStack() as stack:
for arg in args:
stack.Add(lambda ret=arg: ret)
yield
def _ReexecuteInside(self, input_msg, output_msg, config, input_handler,
output_handlers, config_handler, service_name,
method_name):
"""Re-execute the service inside the chroot.
Args:
input_msg (Message): The parsed input message.
output_msg (Message): The empty output message instance.
config (api_config.ApiConfig): The call configs.
input_handler (MessageHandler): Input message handler.
output_handlers (list[MessageHandler]): Output message handlers.
config_handler (MessageHandler): Config message handler.
service_name (str): The name of the service to run.
method_name (str): The name of the method to run.
"""
# Parse the chroot and clear the chroot field in the input message.
chroot = field_handler.handle_chroot(input_msg)
if not chroot.exists():
raise InvalidSdkError('Chroot does not exist.')
# Use a ContextManagerStack to avoid the deep nesting this many
# context managers introduces.
with cros_build_lib.ContextManagerStack() as stack:
# TempDirs setup.
tempdir = stack.Add(chroot.tempdir).tempdir
sync_tempdir = stack.Add(chroot.tempdir).tempdir
# The copy-paths-in context manager to handle Path messages.
stack.Add(
field_handler.copy_paths_in,
input_msg,
chroot.tmp,
prefix=chroot.path)
# The sync-directories context manager to handle SyncedDir messages.
stack.Add(
field_handler.sync_dirs, input_msg, sync_tempdir, prefix=chroot.path)
# Parse goma.
chroot.goma = field_handler.handle_goma(input_msg, chroot.path)
# Build inside-chroot paths for the input, output, and config messages.
new_input = os.path.join(tempdir, self.REEXEC_INPUT_FILE)
chroot_input = '/%s' % os.path.relpath(new_input, chroot.path)
new_output = os.path.join(tempdir, self.REEXEC_OUTPUT_FILE)
chroot_output = '/%s' % os.path.relpath(new_output, chroot.path)
new_config = os.path.join(tempdir, self.REEXEC_CONFIG_FILE)
chroot_config = '/%s' % os.path.relpath(new_config, chroot.path)
# Setup the inside-chroot message files.
logging.info('Writing input message to: %s', new_input)
input_handler.write_from(input_msg, path=new_input)
osutils.Touch(new_output)
logging.info('Writing config message to: %s', new_config)
config_handler.write_from(config.get_proto(), path=new_config)
# We can use a single output to write the rest of them. Use the
# first one as the reexec output and just translate its output in
# the rest of the handlers after.
output_handler = output_handlers[0]
cmd = [
'build_api',
'%s/%s' % (service_name, method_name),
input_handler.input_arg,
chroot_input,
output_handler.output_arg,
chroot_output,
config_handler.config_arg,
chroot_config,
'--debug',
]
try:
result = cros_build_lib.run(
cmd,
enter_chroot=True,
chroot_args=chroot.get_enter_args(),
check=False,
extra_env=chroot.env)
except cros_build_lib.RunCommandError:
# A non-zero return code will not result in an error, but one
# is still thrown when the command cannot be run in the first
# place. This is known to happen at least when the PATH does
# not include the chromite bin dir.
raise CrosSdkNotRunError('Unable to enter the chroot.')
logging.info('Endpoint execution completed, return code: %d',
result.returncode)
# Transfer result files out of the chroot.
output_handler.read_into(output_msg, path=new_output)
field_handler.extract_results(input_msg, output_msg, chroot)
# Write out all of the response formats.
for handler in output_handlers:
handler.write_from(output_msg)
return result.returncode
