def CreateTunnel(self, to_local=None, to_remote=None, connect_settings=None):
"""Establishes a SSH tunnel to the remote device as a background process.
Args:
to_local: A list of PortForwardSpec objects to forward from the local
machine to the remote machine.
to_remote: A list of PortForwardSpec to forward from the remote machine
to the local machine.
connect_settings: The SSH connect settings to use.
Returns:
A Popen object. Note that it represents an already started background
process. Calling poll() on the return value can be used to check that
the tunnel is still running. To close the tunnel call terminate().
"""
ssh_cmd = self._GetSSHCmd(connect_settings=connect_settings)
if to_local is not None:
ssh_cmd.extend(
token for spec in to_local for token in ('-L',
spec.command_line_spec))
if to_remote is not None:
ssh_cmd.extend(
token for spec in to_remote for token in ('-R',
spec.command_line_spec))
ssh_cmd.append('-N')
ssh_cmd.append(self.target_ssh_url)
logging.log(self.debug_level, '%s', cros_build_lib.CmdToStr(ssh_cmd))
return RemoteAccess._mockable_popen(ssh_cmd)
def TestingSymUpload(upload_url, sym_item, _product_name):
"""A stub version of SymUpload for --testing usage"""
cmd = ['sym_upload', sym_item.sym_file, upload_url]
# Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
returncode = random.randint(1, 100) <= 80
logging.debug('would run (and return %i): %s', returncode,
cros_build_lib.CmdToStr(cmd))
if returncode:
output = 'Failed to send the symbol file.'
else:
output = 'Successfully sent the symbol file.'
result = cros_build_lib.CommandResult(cmd=cmd,
error=None,
output=output,
returncode=returncode)
if returncode:
exceptions = (
socket.error('[socket.error] forced test fail'),
httplib.BadStatusLine('[BadStatusLine] forced test fail'),
urllib2.HTTPError(upload_url, 400, '[HTTPError] forced test fail',
{}, None),
urllib2.URLError('[URLError] forced test fail'),
)
raise random.choice(exceptions)
else:
return result
def GitBisect(self, bisect_op):
"""Calls git bisect and logs result.
Args:
bisect_op: git bisect subcommand list.
Returns:
A CommandResult object.
done: True if bisect ends.
"""
command = ['bisect'] + bisect_op
command_str = cros_build_lib.CmdToStr(['git'] + command)
result = self.Git(command)
done = False
log_msg = [command_str]
if result.output:
log_msg.append('(output): %s' % result.output.strip())
if result.error:
log_msg.append('(error): %s' % result.error.strip())
if result.returncode != 0:
log_msg.append('(returncode): %d' % result.returncode)
log_msg = '\n'.join(log_msg)
if result.error or result.returncode != 0:
logging.error(log_msg)
else:
logging.info(log_msg)
if result.output:
if 'is the first bad commit' in result.output.splitlines()[0]:
done = True
return result, done
def GetGdbInitCommands(self, inferior_cmd, device=None):
"""Generate list of commands with which to initialize the gdb session."""
gdb_init_commands = []
if self.remote:
sysroot_var = self.sysroot
else:
sysroot_var = '/'
gdb_init_commands = [
'set sysroot %s' % sysroot_var,
'set prompt %s' % self.prompt,
]
if self.in_chroot:
gdb_init_commands += [
'set solib-absolute-prefix %s' % sysroot_var,
'set solib-search-path %s' % sysroot_var,
'set debug-file-directory %s/usr/lib/debug' % sysroot_var,
]
if device:
ssh_cmd = device.GetAgent().GetSSHCommand(self.ssh_settings)
ssh_cmd.extend(['--', 'gdbserver'])
if self.pid:
ssh_cmd.extend(['--attach', 'stdio', str(self.pid)])
target_type = 'remote'
elif inferior_cmd:
ssh_cmd.extend(['-', inferior_cmd])
ssh_cmd.extend(self.inf_args)
target_type = 'remote'
else:
ssh_cmd.extend(['--multi', 'stdio'])
target_type = 'extended-remote'
ssh_cmd = cros_build_lib.CmdToStr(ssh_cmd)
if self.in_chroot:
if inferior_cmd:
gdb_init_commands.append(
'file %s' %
os.path.join(sysroot_var, inferior_cmd.lstrip(os.sep)))
else:
binary = self.GetSimpleChromeBinary()
gdb_init_commands += [
'set debug-file-directory %s' % os.path.dirname(binary),
'file %s' % binary
]
gdb_init_commands.append('target %s | %s' % (target_type, ssh_cmd))
else:
if inferior_cmd:
gdb_init_commands.append('file %s ' % inferior_cmd)
gdb_init_commands.append('set args %s' %
' '.join(self.inf_args))
return gdb_init_commands
def _BuildChrome(sdk_cmd, chrome_root, out_board_dir, goma):
"""Build Chrome with SimpleChrome environment.
Args:
sdk_cmd (ChromeSDK object): sdk_cmd to run cros chrome-sdk commands.
chrome_root (string): Path to Chrome.
out_board_dir (string): Path to board directory.
goma (goma_util.Goma): Goma object
"""
# Validate fetching of the SDK and setting everything up.
sdk_cmd.Run(['true'])
sdk_cmd.Run(['gclient', 'runhooks'])
# Generate args.gn and ninja files.
gn_cmd = os.path.join(chrome_root, 'buildtools', 'linux64', 'gn')
gn_gen_cmd = '%s gen "%s" --args="$GN_ARGS"' % (gn_cmd, out_board_dir)
sdk_cmd.Run(['bash', '-c', gn_gen_cmd])
_VerifySDKEnvironment(out_board_dir)
if goma:
# If goma is enabled, start goma compiler_proxy here, and record
# several information just before building Chrome is started.
goma.Start()
extra_env = goma.GetExtraEnv()
ninja_env_path = os.path.join(goma.goma_log_dir, 'ninja_env')
sdk_cmd.Run(['env', '--null'],
run_args={
'extra_env': extra_env,
'stdout': ninja_env_path
})
osutils.WriteFile(os.path.join(goma.goma_log_dir, 'ninja_cwd'),
sdk_cmd.cwd)
osutils.WriteFile(os.path.join(goma.goma_log_dir, 'ninja_command'),
cros_build_lib.CmdToStr(sdk_cmd.GetNinjaCommand()))
else:
extra_env = None
result = None
try:
# Build chromium.
result = sdk_cmd.Ninja(run_args={'extra_env': extra_env})
finally:
# In teardown, if goma is enabled, stop the goma compiler proxy,
# and record/copy some information to log directory, which will be
# uploaded to the goma's server in a later stage.
if goma:
goma.Stop()
ninja_log_path = os.path.join(chrome_root,
sdk_cmd.GetNinjaLogPath())
if os.path.exists(ninja_log_path):
shutil.copy2(ninja_log_path,
os.path.join(goma.goma_log_dir, 'ninja_log'))
if result:
osutils.WriteFile(
os.path.join(goma.goma_log_dir, 'ninja_exit'),
str(result.returncode))
def _ShouldRetry(exc):
"""Returns True if push operation failed with a transient error."""
if (isinstance(exc, cros_build_lib.RunCommandError)
and exc.result and exc.result.error and
GIT_TRANSIENT_ERRORS_RE.search(exc.result.error)):
logging.warning('git reported transient error (cmd=%s); retrying',
cros_build_lib.CmdToStr(cmd), exc_info=True)
return True
return False
def Wrapper(command_test):
"""Wrapper for the test function."""
command = cros_build_lib.CmdToStr(command_test.BuildCommand(command_name))
logging.info('Running test for %s.', command)
try:
test_function(command_test)
logging.info('Test for %s passed.', command)
except CommandError as e:
_PrintCommandLog(command, str(e))
raise TestError('Test for %s failed.' % command)
def GetLatestSHA1ForBranch(self, project, branch):
"""Return the git hash at the tip of a branch."""
url = '%s://%s/%s' % (gob_util.GIT_PROTOCOL, self.host, project)
cmd = ['ls-remote', url, 'refs/heads/%s' % branch]
try:
result = git.RunGit('.', cmd, print_cmd=self.print_cmd)
if result:
return result.output.split()[0]
except cros_build_lib.RunCommandError:
logging.error('Command "%s" failed.', cros_build_lib.CmdToStr(cmd),
exc_info=True)
def start_rescue_process(self, update_file):
"""Run cr50-rescue in a process, so it can be killed it if it hangs."""
pty = self._servo.get_raw_cr50_pty()
rescue_cmd = [self._updater, '-v', '-i', update_file, '-d', pty]
logging.info('Starting cr50-rescue: %s',
cros_build_lib.CmdToStr(rescue_cmd))
self._rescue_process = subprocess.Popen(rescue_cmd)
self._rescue_process.communicate()
logging.info('Rescue Finished')
def _DryRunCommand(self, cmd):
"""Print a command for dry_run.
Args:
cmd: command to print.
Returns:
cros_build_lib.CommandResult object.
"""
assert self.dry_run, 'Use with --dry-run only'
logging.info('[DRY RUN] %s', cros_build_lib.CmdToStr(cmd))
return cros_build_lib.CommandResult(cmd, output='', returncode=0)
def _ReExecuteIfNeeded(argv):
"""Re-execute cros_sdk as root.
Also unshare the mount namespace so as to ensure that processes outside
the chroot can't mess with our mounts.
"""
if os.geteuid() != 0:
# Make sure to preserve the active Python executable in case the version
# we're running as is not the default one found via the (new) $PATH.
cmd = _SudoCommand() + ['--'] + [sys.executable] + argv
logging.debug('Reexecing self via sudo:\n%s',
cros_build_lib.CmdToStr(cmd))
os.execvp(cmd[0], cmd)
else:
# We must set up the cgroups mounts before we enter our own namespace.
# This way it is a shared resource in the root mount namespace.
cgroups.Cgroup.InitSystem()
def RunTests(tests,
jobs=1,
chroot_available=True,
network=False,
dryrun=False,
failfast=False):
"""Execute |paths| with |jobs| in parallel (including |network| tests).
Args:
tests: The tests to run.
jobs: How many tests to run in parallel.
chroot_available: Whether we can run tests inside the sdk.
network: Whether to run network based tests.
dryrun: Do everything but execute the test.
failfast: Stop on first failure
Returns:
True if all tests pass, else False.
"""
finished = multiprocessing.Value('i')
testsets = []
pids = []
failed = aborted = False
def WaitOne():
(pid, status) = os.wait()
pids.remove(pid)
return status
# Launch all the tests!
try:
# Build up the testsets.
testsets = BuildTestSets(tests, chroot_available, network, jobs=jobs)
# Fork each test and add it to the list.
for test, cmd, tmpfile in testsets:
if failed and failfast:
logging.error('failure detected; stopping new tests')
break
if len(pids) >= jobs:
if WaitOne():
failed = True
pid = os.fork()
if pid == 0:
proctitle.settitle(test)
ret = 1
try:
if dryrun:
logging.info('Would have run: %s',
cros_build_lib.CmdToStr(cmd))
ret = 0
else:
ret = RunTest(test, cmd, tmpfile, finished,
len(testsets))
except KeyboardInterrupt:
pass
except BaseException:
logging.error('%s failed', test, exc_info=True)
# We cannot run clean up hooks in the child because it'll break down
# things like tempdir context managers.
os._exit(ret) # pylint: disable=protected-access
pids.append(pid)
# Wait for all of them to get cleaned up.
while pids:
if WaitOne():
failed = True
except KeyboardInterrupt:
# If the user wants to stop, reap all the pending children.
logging.warning('CTRL+C received; cleaning up tests')
aborted = True
CleanupChildren(pids)
# Walk through the results.
failed_tests = []
for test, cmd, tmpfile in testsets:
tmpfile.seek(0)
output = tmpfile.read()
if output:
failed_tests.append(test)
print()
logging.error('### LOG: %s', test)
print(output.rstrip())
print()
if failed_tests:
logging.error('The following %i tests failed:\n %s',
len(failed_tests), '\n '.join(sorted(failed_tests)))
return False
elif aborted or failed:
return False
return True
def RunTestFromDut(self, remote, report_file):
"""Runs autotest from DUT.
It runs autotest from DUT directly. It can only be used after the test was
deployed/run using "test_that" from host.
Args:
remote: DUT for running test (refer lib.commandline.Device).
report_file: Benchmark report to store (host side).
Returns:
False if sanity check fails, i.e. autotest control file missing.
If autotest ran successfully, or --eval-failsafe is set, it returns
if the test report is retrieved from DUT. Otherwise, False.
"""
def RetrieveReport(dut, remote_report_file):
"""Retrieves report from DUT to local.
Args:
dut: a RemoteAccess object to access DUT.
remote_report_file: path of the report on DUT to retrieve.
Returns:
True if a report is copied from DUT to local path (report_file).
"""
logging.info('Copy report from DUT(%s:%s) to %s',
dut.remote_host, remote_report_file, report_file)
scp_result = dut.ScpToLocal(remote_report_file, report_file,
error_code_ok=True)
if scp_result.returncode != 0:
logging.error('Failed to copy report from DUT(%s:%s) to host(%s)',
dut.remote_host, remote_report_file, report_file)
return False
return True
# TODO(deanliao): Deal with the case that test control file is not the
# same as below.
test_target = os.path.join(self.AUTOTEST_BASE, 'tests', self.test_name,
'control')
remote_report_file = os.path.join(
self.AUTOTEST_BASE, 'results', 'default', self.test_name, 'results',
self.RESULT_FILENAME)
with osutils.TempDir() as temp_dir:
dut = remote_access.RemoteAccess(
remote.hostname, temp_dir, port=remote.port, username=remote.username)
run_test_command = [self.AUTOTEST_CLIENT, test_target]
logging.info('Run autotest from DUT %s: %s', dut.remote_host,
cros_build_lib.CmdToStr(run_test_command))
# Make sure that both self.AUTOTEST_CLIENT and test_target exist.
sanity_check_result = dut.RemoteSh(['ls'] + run_test_command,
error_code_ok=True, ssh_error_ok=True)
if sanity_check_result.returncode != 0:
logging.info('Failed to run autotest from DUT %s: One of %s does not '
'exist.', dut.remote_host, run_test_command)
return False
run_test_result = dut.RemoteSh(run_test_command, error_code_ok=True,
ssh_error_ok=True)
run_test_returncode = run_test_result.returncode
if run_test_returncode != 0:
logging.info('Run failed (returncode: %d)', run_test_returncode)
if self.eval_passing_only:
return False
else:
logging.info('Ran successfully.')
return RetrieveReport(dut, remote_report_file)
def DoCommand(self,
gsutil_cmd,
headers=(),
retries=None,
version=None,
parallel=False,
**kwargs):
"""Run a gsutil command, suppressing output, and setting retry/sleep.
Args:
gsutil_cmd: The (mostly) constructed gsutil subcommand to run.
headers: A list of raw headers to pass down.
parallel: Whether gsutil should enable parallel copy/update of multiple
files. NOTE: This option causes gsutil to use significantly more
memory, even if gsutil is only uploading one file.
retries: How many times to retry this command (defaults to setting given
at object creation).
version: If given, the generation; essentially the timestamp of the last
update. Note this is not the same as sequence-number; it's
monotonically increasing bucket wide rather than reset per file.
The usage of this is if we intend to replace/update only if the version
is what we expect. This is useful for distributed reasons- for example,
to ensure you don't overwrite someone else's creation, a version of
0 states "only update if no version exists".
Returns:
A RunCommandResult object.
"""
kwargs = kwargs.copy()
kwargs.setdefault('redirect_stderr', True)
cmd = [self.gsutil_bin]
cmd += self.gsutil_flags
for header in headers:
cmd += ['-h', header]
if version is not None:
cmd += ['-h', 'x-goog-if-generation-match:%d' % int(version)]
# Enable parallel copy/update of multiple files if stdin is not to
# be piped to the command. This does not split a single file into
# smaller components for upload.
if parallel and kwargs.get('input') is None:
cmd += ['-m']
cmd.extend(gsutil_cmd)
if retries is None:
retries = self.retries
extra_env = kwargs.pop('extra_env', {})
if self.boto_file and os.path.isfile(self.boto_file):
extra_env.setdefault('BOTO_CONFIG', self.boto_file)
if self.dry_run:
logging.debug("%s: would've run: %s", self.__class__.__name__,
cros_build_lib.CmdToStr(cmd))
else:
try:
return retry_stats.RetryWithStats(retry_stats.GSUTIL,
self._RetryFilter,
retries,
cros_build_lib.RunCommand,
cmd,
sleep=self._sleep_time,
extra_env=extra_env,
**kwargs)
except cros_build_lib.RunCommandError as e:
raise GSCommandError(e.msg, e.result, e.exception)
def _RunBuildStagesWrapper(options, site_config, build_config):
"""Helper function that wraps RunBuildStages()."""
logging.info('cbuildbot was executed with args %s' %
cros_build_lib.CmdToStr(sys.argv))
chrome_rev = build_config['chrome_rev']
if options.chrome_rev:
chrome_rev = options.chrome_rev
if chrome_rev == constants.CHROME_REV_TOT:
options.chrome_version = gob_util.GetTipOfTrunkRevision(
constants.CHROMIUM_GOB_URL)
options.chrome_rev = constants.CHROME_REV_SPEC
# If it's likely we'll need to build Chrome, fetch the source.
if build_config['sync_chrome'] is None:
options.managed_chrome = (chrome_rev != constants.CHROME_REV_LOCAL and
(not build_config['usepkg_build_packages']
or chrome_rev or build_config['profile']))
else:
options.managed_chrome = build_config['sync_chrome']
if options.managed_chrome:
# Tell Chrome to fetch the source locally.
internal = constants.USE_CHROME_INTERNAL in build_config['useflags']
chrome_src = 'chrome-src-internal' if internal else 'chrome-src'
target_name = 'target'
if options.branch:
# Tie the cache per branch
target_name = 'target-%s' % options.branch
options.chrome_root = os.path.join(options.cache_dir, 'distfiles',
target_name, chrome_src)
# Create directory if in need
osutils.SafeMakedirsNonRoot(options.chrome_root)
# We are done munging options values, so freeze options object now to avoid
# further abuse of it.
# TODO(mtennant): one by one identify each options value override and see if
# it can be handled another way. Try to push this freeze closer and closer
# to the start of the script (e.g. in or after _PostParseCheck).
options.Freeze()
metadata_dump_dict = {
# A detected default has been set before now if it wasn't explicit.
'branch': options.branch,
}
if options.metadata_dump:
with open(options.metadata_dump, 'r') as metadata_file:
metadata_dump_dict = json.loads(metadata_file.read())
with parallel.Manager() as manager:
builder_run = cbuildbot_run.BuilderRun(options, site_config,
build_config, manager)
if metadata_dump_dict:
builder_run.attrs.metadata.UpdateWithDict(metadata_dump_dict)
if builder_run.config.builder_class_name is None:
# TODO: This should get relocated to chromeos_config.
if _IsDistributedBuilder(options, chrome_rev, build_config):
builder_cls_name = 'simple_builders.DistributedBuilder'
else:
builder_cls_name = 'simple_builders.SimpleBuilder'
builder_cls = builders.GetBuilderClass(builder_cls_name)
builder = builder_cls(builder_run)
else:
builder = builders.Builder(builder_run)
if not builder.Run():
sys.exit(1)
def Start(self):
"""Start the VM."""
self.Stop()
logging.debug('Start VM')
self._SetQemuPath()
self._SetVMImagePath()
self._RmVMDir()
self._CreateVMDir()
# Make sure we can read these files later on by creating them as ourselves.
osutils.Touch(self.kvm_serial)
for pipe in [self.kvm_pipe_in, self.kvm_pipe_out]:
os.mkfifo(pipe, 0600)
osutils.Touch(self.pidfile)
qemu_args = [self.qemu_path]
if self.qemu_bios_path:
if not os.path.isdir(self.qemu_bios_path):
raise VMError('Invalid QEMU bios path: %s' %
self.qemu_bios_path)
qemu_args += ['-L', self.qemu_bios_path]
qemu_args += [
'-m',
self.qemu_m,
'-smp',
str(self.qemu_smp),
'-vga',
'virtio',
'-daemonize',
'-usbdevice',
'tablet',
'-pidfile',
self.pidfile,
'-chardev',
'pipe,id=control_pipe,path=%s' % self.kvm_monitor,
'-serial',
'file:%s' % self.kvm_serial,
'-mon',
'chardev=control_pipe',
# Append 'check' to warn if the requested CPU is not fully supported.
'-cpu',
self.qemu_cpu + ',check',
'-device',
'virtio-net,netdev=eth0',
'-netdev',
'user,id=eth0,net=10.0.2.0/27,hostfwd=tcp:127.0.0.1:%d-:22' %
self.ssh_port,
'-drive',
'file=%s,index=0,media=disk,cache=unsafe,format=%s' %
(self.image_path, self.image_format),
]
if self.enable_kvm:
qemu_args.append('-enable-kvm')
if not self.display:
qemu_args.extend(['-display', 'none'])
logging.info('Pid file: %s', self.pidfile)
if not self.dry_run:
self._RunCommand(qemu_args)
else:
logging.info(cros_build_lib.CmdToStr(qemu_args))
