def RunCheck(self, check, pass_config):
"""Run the given check.
Args:
check: str - A config.vboot_dir/ensure_|check|.sh check name.
pass_config: bool - Whether the check has a corresponding
`ensure_|check|.config` file to pass.
Returns:
bool - True on success, False on failure.
Raises:
SecurityConfigDirectoryError if the directory does not exist.
VbootCheckoutError if the vboot reference repo could not be cloned or the
vboot_hash could not be checked out.
"""
self._VbootCheckout()
cmd = [
os.path.join(self._checks_dir, 'ensure_%s.sh' % check), self.image
]
if pass_config:
cmd.append(os.path.join(self.baselines,
'ensure_%s.config' % check))
try:
self._RunCommand(cmd)
except cros_build_lib.RunCommandError as e:
logging.error('%s test failed: %s', check, e)
return False
else:
return True
def _RunGeneratorCmd(self, cmd):
"""Wrapper for run in chroot.
Run the given command inside the chroot. It will automatically log the
command output. Note that the command's stdout and stderr are combined into
a single string.
Args:
cmd: Program and argument list in a list. ['delta_generator', '--help']
Raises:
cros_build_lib.RunCommandError if the command exited with a nonzero code.
"""
try:
# Run the command.
result = cros_build_lib.run(cmd,
redirect_stdout=True,
enter_chroot=True,
combine_stdout_stderr=True)
except cros_build_lib.RunCommandError as e:
# Dump error output and re-raise the exception.
logging.error(
'Nonzero exit code (%d), dumping command output:\n%s',
e.result.returncode, e.result.output)
raise
self._StoreLog('Output of command: ' + result.cmdstr)
self._StoreLog(result.output.decode('utf-8', 'replace'))
def ShouldWait(self):
"""Decides if we should continue to wait for the builders to finish.
This will be the retry function for timeout_util.WaitForSuccess, basically
this function will return False if all builders finished or we see a
problem with the builders. Otherwise we'll return True to continue polling
for the builders statuses.
Returns:
A bool of True if we should continue to wait and False if we should not.
"""
# Check if all builders completed.
if self.Completed():
return False
current_time = datetime.datetime.now()
# Guess there are some builders building, check if there is a problem.
if self.ShouldFailForBuilderStartTimeout(current_time):
logging.error('Ending build since at least one builder has not started '
'within 5 mins.')
return False
# We got here which means no problems, we should still wait.
logging.info('Still waiting for the following builds to complete: %r',
sorted(set(self.builders_array).difference(
self.GetCompleted())))
return True
def _CheckConnection(self):
try:
logging.info('Testing connection to the device...')
self.device.RunCommand('true')
except cros_build_lib.RunCommandError as ex:
logging.error('Error connecting to the test device.')
raise DeployFailure(ex)
def _RecordDurationMetric(batches):
"""Records a span duration metric for each span.
Args:
batches: A sequence of span batches (lists)
Yields:
Re-yields the same batches
"""
m = metrics.CumulativeSecondsDistribution(
_SPAN_DURATION_METRIC,
description="The durations of Spans consumed by export_to_cloud_trace",
field_spec=[ts_mon.StringField('name')])
for batch in batches:
batch = tuple(batch) # Needed because we will consume the iterator.
for span in batch:
try:
time_delta = (_ParseDatetime(span['endTime']) -
_ParseDatetime(span['startTime']))
m.add(time_delta.total_seconds(),
fields={'name': span['name']})
except KeyError:
log.error(
"Span %s did not have required fields 'endTime', "
"'startTime', and 'name'.", json.dumps(span))
yield batch
def GetAutotestMetricValue(self, report_file):
"""Gets metric value from autotest benchmark report.
Report example:
{"avg_fps_1000_fishes": {
"summary": {
"units": "fps",
"type": "scalar",
"value": 56.733810392225671,
"improvement_direction": "up"
},
...,
},
...,
}
self.metric = "avg_fps_1000_fishes/summary/value"
Args:
report_file: Path to benchmark report.
Returns:
Metric value in benchmark report.
None if self.metric is undefined or metric does not exist in the report.
"""
if not self.metric:
return None
report = json_lib.ParseJsonFileWithComments(report_file)
metric_value = json_lib.GetNestedDictValue(report, self.metric.split('/'))
if metric_value is None:
logging.error('Cannot get metric %s from %s', self.metric, report_file)
return None
if self.metric_take_average:
return float(sum(metric_value)) / len(metric_value)
return metric_value
def post(self, type_str, metadata, log_time_recorded=True, **kwargs):
"""Wraps call of send_data, inserts entry into elasticsearch.
Args:
type_str: Sets the _type field in elasticsearch db.
metadata: Dictionary object containing metadata
log_time_recorded: Whether to automatically record the time
this metadata is recorded. Default is True.
kwargs: Additional metadata fields
"""
if not metadata:
return
metadata = metadata.copy()
metadata.update(kwargs)
# metadata should not contain anything with key '_type'
if '_type' in metadata:
type_str = metadata['_type']
del metadata['_type']
if log_time_recorded:
metadata['time_recorded'] = time.time()
try:
if self.use_http:
self._send_data_http(type_str, metadata)
else:
self._send_data_udp(type_str, metadata)
except elasticsearch.ElasticsearchException as e:
logging.error(e)
def ResolveChromeBisectRangeFromCrosVersion(self):
"""Resolves Chrome bisect range given good and bad CrOS versions.
It sets up self.good_commit and self.bad_commit, which are derived from
self.good_cros_version and self.bad_cros_version, respectively.
Returns:
False if either good_commit or bad_commit failed to resolve. Otherwise,
True.
"""
self.good_commit = self.CrosVersionToChromeCommit(
self.good_cros_version)
if self.good_commit:
logging.info('Latest Chrome commit of good CrOS version %s: %s',
self.good_cros_version, self.good_commit)
else:
logging.error('Cannot find metadata for CrOS version: %s',
self.good_cros_version)
return False
self.bad_commit = self.CrosVersionToChromeCommit(self.bad_cros_version)
if self.bad_commit:
logging.info('Latest Chrome commit of bad CrOS version %s: %s',
self.bad_cros_version, self.bad_commit)
else:
logging.error('Cannot find metadata for CrOS version: %s',
self.bad_cros_version)
return False
return True
def main(argv):
"""Main function."""
# This is not meant to be a user-friendly script. It takes one and
# only one argument, which is a build stats file to be uploaded
epilog = (
'This script is not intended to be run manually. It is used as'
' part of the build command statistics project.'
)
in_golo = cros_build_lib.GetHostDomain().endswith(constants.GOLO_DOMAIN)
debug_level = commandline.ArgumentParser.DEFAULT_LOG_LEVEL
if in_golo:
debug_level = 'debug'
parser = commandline.ArgumentParser(
epilog=epilog, default_log_level=debug_level)
parser.add_argument(
'build_stats_file', nargs=1, default=None)
options = parser.parse_args(argv)
try:
cmd_stats = StatsLoader.LoadFile(options.build_stats_file[0])
except LoadError:
logging.error(FILE_LOAD_ERROR, options.build_stats_file[0],
exc_info=True)
sys.exit(1)
try:
stats.StatsUploader.Upload(cmd_stats)
except Exception:
logging.error(UNCAUGHT_ERROR, exc_info=True)
sys.exit(1)
def run(self):
"""Launches a devserver process on the device."""
self._RunCommand(["cat", "/dev/null", ">|", self.log_file])
port = self.port if self.port else 0
cmd = [
"python2",
self.devserver_bin,
"--logfile=%s" % self.log_file,
"--pidfile",
self._pid_file,
"--port=%d" % port,
]
if not self.port:
cmd.append("--portfile=%s" % self.port_file)
if self.static_dir:
cmd.append("--static_dir=%s" % self.static_dir)
logging.info("Starting devserver on %s", self.hostname)
result = self._RunCommand(cmd, error_code_ok=True, redirect_stdout=True, combine_stdout_stderr=True)
if result.returncode != 0:
msg = (
"Remote devserver failed to start!\n"
"--- Start output from the devserver startup command ---\n"
"%s"
"--- End output from the devserver startup command ---"
) % (result.output)
logging.error(msg)
if "ImportError: No module named cherrypy" in result.output:
logging.error(self.CHERRYPY_ERROR_MSG)
def CheckAndGetCIDBCreds(force_update=False, folder=None):
"""Check if CIDB creds exist, download creds if necessary."""
cache_dir = path_util.GetCacheDir()
dir_name = folder if folder is not None else 'cidb_creds'
cidb_dir = os.path.join(cache_dir, dir_name)
cidb_dir_lock = cidb_dir + '.lock'
with locking.FileLock(cidb_dir_lock).write_lock():
if os.path.exists(cidb_dir):
if force_update:
shutil.rmtree(cidb_dir, ignore_errors=True)
logging.debug('Force updating CIDB creds. Deleted %s.', cidb_dir)
else:
logging.debug('Using cached credentials %s', cidb_dir)
return cidb_dir
os.mkdir(cidb_dir)
try:
GetCIDBCreds(cidb_dir)
return cidb_dir
except Exception as e:
if isinstance(e, gs.GSCommandError):
logging.warning('Please check if the GS credentials is configured '
'correctly. Please note the permissions to fetch '
'these credentials are for Googlers only,')
logging.error('Failed to get CIDB credentials. Deleting %s', cidb_dir)
shutil.rmtree(cidb_dir, ignore_errors=True)
raise
def _DownloadCIPD(instance_id):
"""Finds the CIPD download link and requests the binary.
The 'client' endpoit of the chrome infra packages API responds with a sha1 and
a Google Storage link. After downloading the binary, we validate that the sha1
of the response and return it.
Args:
instance_id: The version of CIPD to download.
Returns:
the CIPD binary as a string.
"""
args = {'instance_id': instance_id, 'package_name': CIPD_PACKAGE}
_, body = _ChromeInfraRequest('client', request_args=args)
if 'client_binary' not in body:
log.error(
'Error requesting the link to download CIPD from. Got:\n%s',
pprint.pformat(body))
return
http = httplib2.Http(cache=None)
response, binary = http.request(uri=body['client_binary']['fetch_url'])
assert response['status'] == '200', (
'Got a %s response from Google Storage.' % response['status'])
digest = unicode(hashlib.sha1(binary).hexdigest())
assert digest == body['client_binary']['sha1'], (
'The binary downloaded does not match the expected SHA1.')
return binary
def Run(self):
"""Perfrom the cros flash command."""
self.options.Freeze()
try:
flash.Flash(
self.options.device,
self.options.image,
board=self.options.board,
install=self.options.install,
src_image_to_delta=self.options.src_image_to_delta,
rootfs_update=self.options.rootfs_update,
stateful_update=self.options.stateful_update,
clobber_stateful=self.options.clobber_stateful,
reboot=self.options.reboot,
wipe=self.options.wipe,
ping=self.options.ping,
disable_rootfs_verification=self.options.disable_rootfs_verification,
clear_cache=self.options.clear_cache,
yes=self.options.yes,
force=self.options.force,
debug=self.options.debug,
)
logging.notice("cros flash completed successfully.")
except dev_server_wrapper.ImagePathError:
logging.error(
"To get the latest remote image, please run:\n" "cros flash --board=%s %s remote/latest",
self.options.board,
self.options.device.raw,
)
raise
def RegisterBinfmt(self):
"""Make sure qemu has been registered as a format handler
Prep the binfmt handler. First mount if needed, then unregister any bad
mappings, and then register our mapping.
There may still be some race conditions here where one script
de-registers and another script starts executing before it gets
re-registered, however it should be rare.
"""
if not os.path.exists(self._BINFMT_REGISTER_PATH):
osutils.Mount('binfmt_misc', self._BINFMT_PATH, 'binfmt_misc', 0)
if os.path.exists(self.binfmt_path):
interp = 'interpreter %s\n' % self.build_path
for line in osutils.ReadFile(self.binfmt_path):
if line == interp:
break
else:
osutils.WriteFile(self.binfmt_path, '-1')
if not os.path.exists(self.binfmt_path):
register = self.GetRegisterBinfmtStr(self.arch, self.name,
self.build_path)
try:
osutils.WriteFile(self._BINFMT_REGISTER_PATH, register)
except IOError:
logging.error('error: attempted to register: (len:%i) %s',
len(register), register)
def DetermineHealthcheckStatus(hcname, healthcheck):
"""Determine the healthcheck status.
Args:
hcname: A string. The name of the health check.
healthcheck: A healthcheck object.
Returns:
A HEALTHCHECK_STATUS named tuple.
"""
try:
# Run the health check condition.
result = healthcheck.Check()
# Determine the healthcheck's status.
health = result >= HCSTATUS_HEALTHY
if result == HCSTATUS_HEALTHY:
return HEALTHCHECK_STATUS(hcname, health, NULL_DESCRIPTION,
EMPTY_ACTIONS)
description, actions = healthcheck.Diagnose(result)
return HEALTHCHECK_STATUS(hcname, health, description, actions)
except Exception as e:
# Checkfiles may contain all kinds of errors! We do not want the
# Mob* Monitor to fail, so catch generic exceptions.
logging.error('Failed to execute health check %s: %s', hcname, e)
return HEALTHCHECK_STATUS(hcname, False,
'Failed to execute the health check.'
' Please review the health check file.',
EMPTY_ACTIONS)
def _flash(dut_cmd_on, dut_cmd_off, flash_cmd):
"""Runs subprocesses for setting dut controls and flashing the AP fw.
Args:
dut_cmd_on ([[str]]): 2d array of dut-control commands
in the form [['dut-control', 'cmd1', 'cmd2'...],
['dut-control', 'cmd3'...]]
that get executed before the flashing.
dut_cmd_off ([[str]]): 2d array of dut-control commands
in the same form that get executed after flashing.
flash_cmd ([str]): array containing all arguments for
the flash command.
Returns:
bool: True if flash was successful, otherwise False.
"""
try:
for cmd in dut_cmd_on:
subprocess.run(cmd, check=True)
subprocess.run(flash_cmd, check=True)
for cmd in dut_cmd_off:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
logging.error('Flashing failed with output:\n%s', e.output)
return False
return True
def GetStagesToIgnoreForChange(build_root, change):
"""Get a list of stages that the CQ should ignore for a given |change|.
The list of stage name prefixes to ignore for each project is specified in a
config file inside the project, named COMMIT-QUEUE.ini. The file would look
like this:
[GENERAL]
ignored-stages: HWTest VMTest
The CQ will submit changes to the given project even if the listed stages
failed. These strings are stage name prefixes, meaning that "HWTest" would
match any HWTest stage (e.g. "HWTest [bvt]" or "HWTest [foo]")
Args:
build_root: The root of the checkout.
change: Change to examine, as a PatchQuery object.
Returns:
A list of stages to ignore for the given |change|.
"""
result = None
try:
result = GetOptionForChange(build_root, change, 'GENERAL',
'ignored-stages')
except ConfigParser.Error:
logging.error('%s has malformed config file', change, exc_info=True)
return result.split() if result else []
def LabelBuild(self, score):
"""Determines if a build is good.
Args:
score: evaluation score of the build.
Returns:
'good' if the score is closer to given good one; otherwise (including
score is None), 'bad'.
Note that if self.skip_failed_commit is set, returns 'skip' for empty
score.
"""
label = 'bad'
if not score:
if self.skip_failed_commit:
label = 'skip'
logging.error('No score. Marked as %s', label)
return label
good_score = self.good_commit_info.score
bad_score = self.bad_commit_info.score
ref_score_min = min(good_score.mean, bad_score.mean)
ref_score_max = max(good_score.mean, bad_score.mean)
if ref_score_min < score.mean < ref_score_max:
# Within (good_score, bad_score)
if abs(good_score.mean - score.mean) <= self.threshold:
label = 'good'
else:
dist_good = abs(good_score.mean - score.mean)
dist_bad = abs(bad_score.mean - score.mean)
if dist_good < dist_bad:
label = 'good'
logging.info('Score %.3f marked as %s', score.mean, label)
return label
def run(self):
"""Launches a devserver process on the device."""
self._RunCommand(['cat', '/dev/null', '>|', self.log_file])
port = self.port if self.port else 0
cmd = ['python2', self.devserver_bin,
'--logfile=%s' % self.log_file,
'--pidfile', self._pid_file,
'--port=%d' % port,
'--critical_update']
if not self.port:
cmd.append('--portfile=%s' % self.port_file)
if self.static_dir:
cmd.append('--static_dir=%s' % self.static_dir)
logging.info('Starting devserver on %s', self.hostname)
result = self._RunCommand(cmd, error_code_ok=True, redirect_stdout=True,
combine_stdout_stderr=True)
if result.returncode != 0:
msg = (('Remote devserver failed to start!\n'
'--- Start output from the devserver startup command ---\n'
'%s'
'--- End output from the devserver startup command ---') %
(result.output))
logging.error(msg)
if 'ImportError: No module named cherrypy' in result.output:
logging.error(self.CHERRYPY_ERROR_MSG)
def LookupReportFile(self):
"""Looks up autotest report file.
It looks up results-chart.json under chroot's /tmp/test_that_latest.
Returns:
Path to report file. None if not found.
"""
# Default result dir: /tmp/test_that_latest
results_dir = self.ResolvePathFromChroot(os.path.join(
'/tmp', 'test_that_latest', 'results-1-%s' % self.test_name))
# Invoking "find" command is faster than using os.walkdir().
try:
command_result = cros_build_lib.RunCommand(
['find', '.', '-name', self.RESULT_FILENAME],
cwd=results_dir, capture_output=True)
except cros_build_lib.RunCommandError as e:
logging.error('Failed to look up %s under %s: %s', self.RESULT_FILENAME,
results_dir, e)
return None
if not command_result.output:
logging.error('Failed to look up %s under %s', self.RESULT_FILENAME,
results_dir)
return None
report_file_under_results_dir = (
command_result.output.splitlines()[0].strip())
return os.path.normpath(
os.path.join(results_dir, report_file_under_results_dir))
def Load(self):
"""Load the charsets from gconv-modules."""
for line in open(self._filename):
line = line.split('#', 1)[0].strip()
if not line: # Comment
continue
lst = line.split()
if lst[0] == 'module':
_, fromset, toset, filename = lst[:4]
for charset in (fromset, toset):
charset = charset.rstrip('/')
mods = self._modules.get(charset, set())
mods.add(filename)
self._modules[charset] = mods
elif lst[0] == 'alias':
_, fromset, toset = lst
fromset = fromset.rstrip('/')
toset = toset.rstrip('/')
# Warn if the same charset is defined as two different aliases.
if self._alias.get(fromset, toset) != toset:
logging.error('charset "%s" already defined as "%s".', fromset,
self._alias[fromset])
self._alias[fromset] = toset
else:
cros_build_lib.Die('Unknown line: %s', line)
logging.debug('Found %d modules and %d alias in %s', len(self._modules),
len(self._alias), self._filename)
charsets = sorted(self._alias.keys() + self._modules.keys())
# Remove the 'INTERNAL' charset from the list, since it is not a charset
# but an internal representation used to convert to and from other charsets.
if 'INTERNAL' in charsets:
charsets.remove('INTERNAL')
return charsets
def ShouldFailForBuilderStartTimeout(self, current_time):
"""Decides if we should fail if a builder hasn't started within 5 mins.
If a builder hasn't started within BUILDER_START_TIMEOUT and the rest of the
builders have finished, let the caller know that we should fail.
Args:
current_time: A datetime.datetime object letting us know the current time.
Returns:
A bool saying True that we should fail, False otherwise.
"""
# Check that we're at least past the start timeout.
builder_start_deadline = datetime.timedelta(
minutes=self.BUILDER_START_TIMEOUT)
past_deadline = current_time - self.start_time > builder_start_deadline
# Check that aside from the missing builders the rest have completed.
other_builders_completed = (
(len(self.GetMissing()) + len(self.GetCompleted())) ==
len(self.builders_array))
# Check that we have missing builders and logging who they are.
builders_are_missing = False
for builder in self.GetMissing():
logging.error('No status found for build config %s.', builder)
builders_are_missing = True
return past_deadline and other_builders_completed and builders_are_missing
def Run(self):
files = self.options.files
if not files:
# Running with no arguments is allowed to make the repo upload hook
# simple, but print a warning so that if someone runs this manually
# they are aware that nothing was linted.
logging.warning('No files provided to lint. Doing nothing.')
errors = multiprocessing.Value('i')
linter_map = _BreakoutFilesByLinter(files)
dispatcher = functools.partial(_Dispatcher, errors,
self.options.output, self.options.debug)
# Special case one file as it's common -- faster to avoid parallel startup.
if sum([len(x) for _, x in linter_map.iteritems()]) == 1:
linter, files = linter_map.items()[0]
dispatcher(linter, files[0])
else:
# Run the linter in parallel on the files.
with parallel.BackgroundTaskRunner(dispatcher) as q:
for linter, files in linter_map.iteritems():
for path in files:
q.put([linter, path])
if errors.value:
logging.error('linter found errors in %i files', errors.value)
sys.exit(1)
def _DisableRootfsVerification(self):
if not self.options.force:
logging.error('Detected that the device has rootfs verification enabled.')
logging.info('This script can automatically remove the rootfs '
'verification, which requires that it reboot the device.')
logging.info('Make sure the device is in developer mode!')
logging.info('Skip this prompt by specifying --force.')
if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
# Since we stopped Chrome earlier, it's good form to start it up again.
if self.options.startui:
logging.info('Starting Chrome...')
self.device.RunCommand('start ui')
raise DeployFailure('Need rootfs verification to be disabled. '
'Aborting.')
logging.info('Removing rootfs verification from %s', self.options.to)
# Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
# Use --force to bypass the checks.
cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
'--remove_rootfs_verification --force')
for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
self.device.RunCommand(cmd % partition, error_code_ok=True)
# A reboot in developer mode takes a while (and has delays), so the user
# will have time to read and act on the USB boot instructions below.
logging.info('Please remember to press Ctrl-U if you are booting from USB.')
self.device.Reboot()
# Now that the machine has been rebooted, we need to kill Chrome again.
self._KillProcsIfNeeded()
# Make sure the rootfs is writable now.
self._MountRootfsAsWritable(error_code_ok=False)
def Run(self):
"""Perfrom the cros flash command."""
self.options.Freeze()
try:
flash.Flash(
self.options.device,
self.options.image,
project_sdk_image=self.options.project_sdk is not None,
sdk_version=self.options.project_sdk or None,
board=self.options.board,
brick_name=self.options.brick or self.curr_brick_locator,
blueprint_name=self.options.blueprint,
install=self.options.install,
src_image_to_delta=self.options.src_image_to_delta,
rootfs_update=self.options.rootfs_update,
stateful_update=self.options.stateful_update,
clobber_stateful=self.options.clobber_stateful,
reboot=self.options.reboot,
wipe=self.options.wipe,
ping=self.options.ping,
disable_rootfs_verification=self.options.disable_rootfs_verification,
clear_cache=self.options.clear_cache,
yes=self.options.yes,
force=self.options.force,
debug=self.options.debug)
except dev_server_wrapper.ImagePathError as e:
logging.error('To get the latest remote image, please run:\n'
'cros flash --board=%s %s remote/latest',
self.options.board, self.options.device.raw)
self._HandleException(e)
except Exception as e:
self._HandleException(e)
else:
logging.notice('cros flash completed successfully.')
def Run(self):
"""Run the tests."""
self.options.Freeze()
commandline.RunInsideChroot(self)
packages, non_matching = self._GetMatchingPackages(self.options.packages)
if non_matching:
cros_build_lib.Die('No packages matching: %s', ' '.join(non_matching))
packages_with_tests = portage_util.PackagesWithTest(self.sysroot, packages)
packages_without_tests = packages - packages_with_tests
if packages_without_tests:
logging.warning('Ignored the following packages because they were '
'missing tests:')
for p in packages_without_tests:
logging.warning(p)
if not packages_with_tests:
logging.error('Nothing to test.')
return
try:
chroot_util.RunUnittests(self.sysroot, packages_with_tests,
verbose=self.options.verbose, retries=0)
except cros_build_lib.RunCommandError as e:
cros_build_lib.Die('Unit tests failed: %s' % e)
def _AddLKGMToManifest(self, manifest):
"""Write the last known good version string to the manifest.
Args:
manifest: Path to the manifest.
"""
# Get the last known good version string.
try:
lkgm_filename = os.path.basename(os.readlink(self.lkgm_path))
lkgm_version, _ = os.path.splitext(lkgm_filename)
except OSError:
return
# Write the last known good version string to the manifest.
try:
manifest_dom = minidom.parse(manifest)
except expat.ExpatError:
logging.error('Got parsing error for: %s', manifest)
logging.error('Bad XML:\n%s', osutils.ReadFile(manifest))
raise
lkgm_element = manifest_dom.createElement(LKGM_ELEMENT)
lkgm_element.setAttribute(LKGM_VERSION_ATTR, lkgm_version)
manifest_dom.documentElement.appendChild(lkgm_element)
self._WriteXml(manifest_dom, manifest)
def _AddLKGMToManifest(self, manifest):
"""Write the last known good version string to the manifest.
Args:
manifest: Path to the manifest.
"""
# Get the last known good version string.
try:
lkgm_filename = os.path.basename(os.readlink(self.lkgm_path))
lkgm_version, _ = os.path.splitext(lkgm_filename)
except OSError:
return
# Write the last known good version string to the manifest.
try:
manifest_dom = minidom.parse(manifest)
except expat.ExpatError:
logging.error('Got parsing error for: %s', manifest)
logging.error('Bad XML:\n%s', osutils.ReadFile(manifest))
raise
lkgm_element = manifest_dom.createElement(LKGM_ELEMENT)
lkgm_element.setAttribute(LKGM_VERSION_ATTR, lkgm_version)
manifest_dom.documentElement.appendChild(lkgm_element)
self._WriteXml(manifest_dom, manifest)
def GetGsutilVersion():
"""Return the version string for the installed gsutil utility.
Returns:
The version string.
Raises:
GsutilMissingError if gsutil cannot be found.
GSLibError for any other error.
"""
args = ['version']
# As of version 3.26, a quirk of 'gsutil version' is that if gsutil is
# outdated it will ask if you want to update (Y/n) before proceeding... but
# do it only the first time (for a particular update? I'm not exactly sure).
# Prepare a 'n' answer just in case.
user_input = 'n\n'
result = RunGsutilCommand(args, error_ok=False, input=user_input)
output = '\n'.join(o for o in [result.output, result.error] if o)
if output:
match = re.search(r'^\s*gsutil\s+version\s+([\d\.]+)', output,
re.IGNORECASE)
if match:
return match.group(1)
else:
logging.error('Unexpected output format from %r:\n%s',
result.cmdstr, output)
raise GSLibError('Unexpected output format from %r.' % result.cmdstr)
else:
logging.error('No stdout output from %r.', result.cmdstr)
raise GSLibError('No stdout output from %r.', result.cmdstr)
def _StageOnMoblab(self, tempdir):
"""Stage the generated payloads and test bits on a moblab device.
Args:
tempdir: Temporary Directory that contains the generated payloads and
test bits.
"""
with remote_access.ChromiumOSDeviceHandler(self.options.remote) as device:
device.RunCommand(['mkdir', '-p', self.stage_directory])
for f in os.listdir(tempdir):
device.CopyToDevice(os.path.join(tempdir, f), self.stage_directory)
device.RunCommand(['chown', '-R', 'moblab:moblab',
MOBLAB_TMP_DIR])
# Delete this image from the Devserver in case it was previously staged.
device.RunCommand(['rm', '-rf', os.path.join(MOBLAB_STATIC_DIR,
self.staged_image_name)])
try:
stage_url = DEVSERVER_STAGE_URL % dict(moblab=self.options.remote,
staged_dir=self.stage_directory)
res = urllib2.urlopen(stage_url).read()
except (urllib2.HTTPError, httplib.HTTPException, urllib2.URLError) as e:
logging.error('Unable to stage artifacts on moblab. Error: %s', e)
else:
if res == 'Success':
logging.info('\n\nStaging Completed!')
logging.info('Image is staged on Moblab as %s',
self.staged_image_name)
else:
logging.info('Staging failed. Error Message: %s', res)
finally:
device.RunCommand(['rm', '-rf', self.stage_directory])
def _Emerge(device, pkg_path, root, extra_args=None):
"""Copies |pkg| to |device| and emerges it.
Args:
device: A ChromiumOSDevice object.
pkg_path: A path to a binary package.
root: Package installation root path.
extra_args: Extra arguments to pass to emerge.
Raises:
DeployError: Unrecoverable error during emerge.
"""
pkgroot = os.path.join(device.work_dir, 'packages')
pkg_name = os.path.basename(pkg_path)
pkg_dirname = os.path.basename(os.path.dirname(pkg_path))
pkg_dir = os.path.join(pkgroot, pkg_dirname)
portage_tmpdir = os.path.join(device.work_dir, 'portage-tmp')
# Clean out the dirs first if we had a previous emerge on the device so as to
# free up space for this emerge. The last emerge gets implicitly cleaned up
# when the device connection deletes its work_dir.
device.RunCommand(
['rm', '-rf', pkg_dir, portage_tmpdir, '&&',
'mkdir', '-p', pkg_dir, portage_tmpdir], remote_sudo=True)
# This message is read by BrilloDeployOperation.
logging.notice('Copying %s to device.', pkg_name)
device.CopyToDevice(pkg_path, pkg_dir, remote_sudo=True)
logging.info('Use portage temp dir %s', portage_tmpdir)
# This message is read by BrilloDeployOperation.
logging.notice('Installing %s.', pkg_name)
pkg_path = os.path.join(pkg_dir, pkg_name)
# We set PORTAGE_CONFIGROOT to '/usr/local' because by default all
# chromeos-base packages will be skipped due to the configuration
# in /etc/protage/make.profile/package.provided. However, there is
# a known bug that /usr/local/etc/portage is not setup properly
# (crbug.com/312041). This does not affect `cros deploy` because
# we do not use the preset PKGDIR.
extra_env = {
'FEATURES': '-sandbox',
'PKGDIR': pkgroot,
'PORTAGE_CONFIGROOT': '/usr/local',
'PORTAGE_TMPDIR': portage_tmpdir,
'PORTDIR': device.work_dir,
'CONFIG_PROTECT': '-*',
}
cmd = ['emerge', '--usepkg', pkg_path, '--root=%s' % root]
if extra_args:
cmd.append(extra_args)
try:
device.RunCommand(cmd, extra_env=extra_env, remote_sudo=True,
capture_output=False, debug_level=logging.INFO)
except Exception:
logging.error('Failed to emerge package %s', pkg_name)
raise
else:
logging.notice('%s has been installed.', pkg_name)
def _RunParallelStages(stage_objs):
"""Run the specified stages in parallel.
Args:
stage_objs: BuilderStage objects.
"""
steps = [stage.Run for stage in stage_objs]
try:
parallel.RunParallelSteps(steps)
except BaseException as ex:
logging.error('BaseException in _RunParallelStages %s' % ex,
exc_info=True)
# If a stage threw an exception, it might not have correctly reported
# results (e.g. because it was killed before it could report the
# results.) In this case, attribute the exception to any stages that
# didn't report back correctly (if any).
for stage in stage_objs:
for name in stage.GetStageNames():
if not results_lib.Results.StageHasResults(name):
results_lib.Results.Record(name, ex, str(ex))
if cidb.CIDBConnectionFactory.IsCIDBSetup():
db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()
if db:
for stage in stage_objs:
for build_stage_id in stage.GetBuildStageIDs():
if not db.HasBuildStageFailed(build_stage_id):
failures_lib.ReportStageFailureToCIDB(
db, build_stage_id, ex)
raise
def _Unmerge(device, pkg, root):
"""Unmerges |pkg| on |device|.
Args:
device: A RemoteDevice object.
pkg: A package name.
root: Package installation root path.
"""
pkg_name = os.path.basename(pkg)
# This message is read by BrilloDeployOperation.
logging.notice('Unmerging %s.', pkg_name)
cmd = ['qmerge', '--yes']
# Check if qmerge is available on the device. If not, use emerge.
if device.RunCommand(
['qmerge', '--version'], error_code_ok=True).returncode != 0:
cmd = ['emerge']
cmd.extend(['--unmerge', pkg, '--root=%s' % root])
try:
# Always showing the emerge output for clarity.
device.RunCommand(cmd, capture_output=False, remote_sudo=True,
debug_level=logging.INFO)
except Exception:
logging.error('Failed to unmerge package %s', pkg_name)
raise
else:
logging.notice('%s has been uninstalled.', pkg_name)
def _GetArcBasename(build, basename):
"""Tweaks filenames between Android bucket and ARC++ bucket.
Android builders create build artifacts with the same name for -user and
-userdebug builds, which breaks the android-container ebuild (b/33072485).
When copying the artifacts from the Android bucket to the ARC++ bucket some
artifacts will be renamed from the usual pattern
*cheets_${ARCH}-target_files-S{VERSION}.zip to
cheets_${BUILD_NAME}-target_files-S{VERSION}.zip which will typically look
like cheets_(${LABEL})*${ARCH}_userdebug-target_files-S{VERSION}.zip.
Args:
build: the build being mirrored, e.g. 'X86', 'ARM', 'X86_USERDEBUG'.
basename: the basename of the artifact to copy.
Returns:
The basename of the destination.
"""
if build not in constants.ARC_BUILDS_NEED_ARTIFACTS_RENAMED:
return basename
if basename in constants.ARC_ARTIFACTS_RENAME_NOT_NEEDED:
return basename
to_discard, sep, to_keep = basename.partition('-')
if not sep:
logging.error(('Build %s: Could not find separator "-" in artifact'
' basename %s'), build, basename)
return basename
if 'cheets_' in to_discard:
return 'cheets_%s-%s' % (build.lower(), to_keep)
elif 'bertha_' in to_discard:
return 'bertha_%s-%s' % (build.lower(), to_keep)
logging.error('Build %s: Unexpected artifact basename %s', build, basename)
return basename
def Run(self):
"""Runs `cros shell`."""
self.options.Freeze()
self._ReadOptions()
# Nested try blocks so the inner can raise to the outer, which handles
# overall failures.
try:
try:
return self._StartSsh()
except remote_access.SSHConnectionError as e:
# Handle a mismatched host key; mismatched keys are a bit of a pain to
# fix manually since `ssh-keygen -R` doesn't work within the chroot.
if e.IsKnownHostsMismatch():
# The full SSH error message has extra info for the user.
logging.warning('\n%s', e)
if self._UserConfirmKeyChange():
remote_access.RemoveKnownHost(self.ssh_hostname)
# The user already OK'd so we can skip the additional SSH check.
self.host_key_checking = 'no'
return self._StartSsh()
else:
raise
else:
raise
except (Exception, KeyboardInterrupt) as e:
logging.error('\n%s', e)
logging.error('`cros shell` failed.')
if self.options.debug:
raise
def CheckAndGetCIDBCreds(force_update=False, folder=None):
"""Check if CIDB creds exist, download creds if necessary."""
cache_dir = path_util.GetCacheDir()
dir_name = folder if folder is not None else 'cidb_creds'
cidb_dir = os.path.join(cache_dir, dir_name)
cidb_dir_lock = cidb_dir + '.lock'
with locking.FileLock(cidb_dir_lock).write_lock():
if os.path.exists(cidb_dir):
if force_update:
shutil.rmtree(cidb_dir, ignore_errors=True)
logging.debug('Force updating CIDB creds. Deleted %s.',
cidb_dir)
else:
logging.debug('Using cached credentials %s', cidb_dir)
return cidb_dir
os.mkdir(cidb_dir)
try:
GetCIDBCreds(cidb_dir)
return cidb_dir
except Exception as e:
if isinstance(e, gs.GSCommandError):
logging.warning(
'Please check if the GS credentials is configured '
'correctly. Please note the permissions to fetch '
'these credentials are for Googlers only,')
logging.error('Failed to get CIDB credentials. Deleting %s',
cidb_dir)
shutil.rmtree(cidb_dir, ignore_errors=True)
raise
def Run(self):
"""Run cros deploy."""
commandline.RunInsideChroot(self, auto_detect_brick=True)
self.options.Freeze()
try:
deploy.Deploy(
self.options.device,
self.options.packages,
board=self.options.board,
brick_name=self.options.brick or self.curr_brick_locator,
blueprint=self.options.blueprint,
emerge=self.options.emerge,
update=self.options.update,
deep=self.options.deep,
deep_rev=self.options.deep_rev,
clean_binpkg=self.options.clean_binpkg,
root=self.options.root,
strip=self.options.strip,
emerge_args=self.options.emerge_args,
ssh_private_key=self.options.private_key,
ping=self.options.ping,
reflash=self.options.reflash,
force=self.options.force,
dry_run=self.options.dry_run)
except Exception as e:
logging.error(e)
logging.error('cros deploy terminated before completing.')
if self.options.debug:
raise
else:
raise SystemExit(1)
else:
logging.info('cros deploy completed successfully.')
def GitBisect(self, bisect_op):
"""Calls git bisect and logs result.
Args:
bisect_op: git bisect subcommand list.
Returns:
A CommandResult object.
done: True if bisect ends.
"""
command = ['bisect'] + bisect_op
command_str = cros_build_lib.CmdToStr(['git'] + command)
result = self.Git(command)
done = False
log_msg = [command_str]
if result.output:
log_msg.append('(output): %s' % result.output.strip())
if result.error:
log_msg.append('(error): %s' % result.error.strip())
if result.returncode != 0:
log_msg.append('(returncode): %d' % result.returncode)
log_msg = '\n'.join(log_msg)
if result.error or result.returncode != 0:
logging.error(log_msg)
else:
logging.info(log_msg)
if result.output:
if 'is the first bad commit' in result.output.splitlines()[0]:
done = True
return result, done
def ProcessPackageLicenses(self):
"""Iterate through all packages provided and gather their licenses.
GetLicenses will scrape licenses from the code and/or gather stock license
names. We gather the list of stock and custom ones for later processing.
Do not call this after adding virtual packages with AddExtraPkg.
"""
for package_name in self.packages:
pkg = self.packages[package_name]
if pkg.skip:
logging.debug('Package %s is in skip list', package_name)
continue
# Other skipped packages get dumped with incomplete info and the skip flag
if not os.path.exists(pkg.license_dump_path):
if not self.gen_licenses:
raise PackageLicenseError('License for %s is missing' % package_name)
logging.error('>>> License for %s is missing, creating now <<<',
package_name)
build_info_path = os.path.join(
cros_build_lib.GetSysroot(pkg.board),
PER_PKG_LICENSE_DIR, pkg.fullnamerev)
pkg.GetLicenses(build_info_path, None)
# We dump packages where licensing failed too.
pkg.SaveLicenseDump(pkg.license_dump_path)
# Load the pre-cached version, if the in-memory version is incomplete.
if not pkg.license_names:
logging.debug('loading dump for %s', pkg.fullnamerev)
self._LoadLicenseDump(pkg)
def PerformStage(self):
"""Performs the stage."""
if self._android_rev and self._run.config.master:
self._android_version = self.GetLatestAndroidVersion()
logging.info('Latest Android version is: %s', self._android_version)
if (self._chrome_rev == constants.CHROME_REV_LATEST and
self._run.config.master):
# PFQ master needs to determine what version of Chrome to build
# for all slaves.
logging.info('I am a master running with CHROME_REV_LATEST, '
'therefore getting latest chrome version.')
self._chrome_version = self.GetLatestChromeVersion()
logging.info('Latest chrome version is: %s', self._chrome_version)
ManifestVersionedSyncStage.PerformStage(self)
# Generate blamelist
cros_version = self.GetLastChromeOSVersion()
if cros_version:
old_filename = self.manifest_manager.GetBuildSpecFilePath(
cros_version.milestone, cros_version.platform)
if not os.path.exists(old_filename):
logging.error(
'Could not generate blamelist, '
'manifest file does not exist: %s', old_filename)
else:
logging.debug('Generate blamelist against: %s', old_filename)
lkgm_manager.GenerateBlameList(self.repo, old_filename)
def UpdateStateful(self, device, payload, clobber=False):
"""Update the stateful partition of the device.
Args:
device: The ChromiumOSDevice object to update.
payload: The path to the update payload.
clobber: Clobber stateful partition (defaults to False).
"""
# Copy latest stateful_update to device.
stateful_update_bin = path_util.FromChrootPath(
self.STATEFUL_UPDATE_BIN, workspace_path=self.workspace_path)
device.CopyToWorkDir(stateful_update_bin)
msg = 'Updating stateful partition'
logging.info('Copying stateful payload to device...')
device.CopyToWorkDir(payload)
cmd = ['sh',
os.path.join(device.work_dir,
os.path.basename(self.STATEFUL_UPDATE_BIN)),
os.path.join(device.work_dir, os.path.basename(payload))]
if clobber:
cmd.append('--stateful_change=clean')
msg += ' with clobber enabled'
logging.info('%s...', msg)
try:
device.RunCommand(cmd)
except cros_build_lib.RunCommandError:
logging.error('Faild to perform stateful partition update.')
def MaySetupBoard(self):
"""Checks if /build/${board} exists. Sets it up if not.
Returns:
False if setup_board or build_package failed. True otherwise.
"""
if not os.path.isdir(self.cros_dir):
logging.notice('ChromeOS source: %s does not exist, set it up',
self.cros_dir)
self.SetupCrosRepo()
board_path = self.ResolvePathFromChroot(os.path.join('/build', self.board))
if os.path.isdir(board_path):
return True
try:
self.RunCommandInsideCrosSdk(['./setup_board', '--board', self.board])
except cros_build_lib.RunCommandError as e:
logging.error('Failed to setup_board for %s: %s', self.board, e)
return False
try:
self.RunCommandInsideCrosSdk(['./build_packages', '--board', self.board])
except cros_build_lib.RunCommandError as e:
logging.error('Failed to build_package for %s: %s', self.board, e)
return False
return True
def _ProcessDeps(cl, deps, required):
"""Yields matching dependencies for a patch"""
# We need to query the change to guarantee that we have a .gerrit_number
for dep in deps:
if not dep.remote in opts.gerrit:
opts.gerrit[dep.remote] = gerrit.GetGerritHelper(
remote=dep.remote, print_cmd=opts.debug)
helper = opts.gerrit[dep.remote]
# TODO(phobbs) this should maybe catch network errors.
changes = _QueryChange(dep.ToGerritQueryText(), helper=helper)
# Handle empty results. If we found a commit that was pushed directly
# (e.g. a bot commit), then gerrit won't know about it.
if not changes:
if required:
logging.error('CL %s depends on %s which cannot be found',
cl, dep.ToGerritQueryText())
continue
# Our query might have matched more than one result. This can come up
# when CQ-DEPEND uses a Gerrit Change-Id, but that Change-Id shows up
# across multiple repos/branches. We blindly check all of them in the
# hopes that all open ones are what the user wants, but then again the
# CQ-DEPEND syntax itself is unable to differeniate. *shrug*
if len(changes) > 1:
logging.warning('CL %s has an ambiguous CQ dependency %s', cl,
dep.ToGerritQueryText())
for change in changes:
if change.status == 'NEW':
yield change
def GetSuites(milo_client, waterfall, builder_name, build_number):
"""Gets a list of suites ids for a given build from Milo.
Args:
milo_client: MiloClient object.
waterfall: Buildbot waterfall.
builder_name: Buildbot builder name.
build_number: Buidlbot build number.
Returns:
A set of suite ids.
"""
buildinfo = milo_client.BuildInfoGetBuildbot(waterfall, builder_name,
build_number)
suite_ids = set()
for step in buildinfo['steps']:
for link in buildinfo['steps'][step].get('otherLinks', []):
if link.get('label') == 'Link to suite':
url = link.get('url')
m = SUITE_RE.search(url)
if m:
suite_ids.add(m.group(1))
else:
logging.error('Unable to parse suite link for %s: %s' %
(buildinfo['steps'][step]['name'], url))
return suite_ids
def RegisterBinfmt(self):
"""Make sure qemu has been registered as a format handler
Prep the binfmt handler. First mount if needed, then unregister any bad
mappings, and then register our mapping.
There may still be some race conditions here where one script
de-registers and another script starts executing before it gets
re-registered, however it should be rare.
"""
if not os.path.exists(self._BINFMT_REGISTER_PATH):
osutils.Mount("binfmt_misc", self._BINFMT_PATH, "binfmt_misc", 0)
if os.path.exists(self.binfmt_path):
interp = "interpreter %s\n" % self.build_path
for line in osutils.ReadFile(self.binfmt_path):
if line == interp:
break
else:
osutils.WriteFile(self.binfmt_path, "-1")
if not os.path.exists(self.binfmt_path):
register = self.GetRegisterBinfmtStr(self.arch, self.name, self.build_path)
try:
osutils.WriteFile(self._BINFMT_REGISTER_PATH, register)
except IOError:
logging.error("error: attempted to register: (len:%i) %s", len(register), register)
def Run(self):
"""Perform the cros flash command."""
self.options.Freeze()
try:
flash.Flash(
self.options.device,
self.options.image,
board=self.options.board,
version=self._GetDefaultVersion(),
install=self.options.install,
src_image_to_delta=self.options.src_image_to_delta,
rootfs_update=self.options.rootfs_update,
stateful_update=self.options.stateful_update,
clobber_stateful=self.options.clobber_stateful,
reboot=self.options.reboot,
wipe=self.options.wipe,
ssh_private_key=self.options.private_key,
ping=self.options.ping,
disable_rootfs_verification=self.options.
disable_rootfs_verification,
clear_cache=self.options.clear_cache,
yes=self.options.yes,
force=self.options.force,
debug=self.options.debug,
send_payload_in_parallel=self.options.send_payload_in_parallel)
logging.notice('cros flash completed successfully.')
except dev_server_wrapper.ImagePathError:
logging.error(
'To get the latest remote image, please run:\n'
'cros flash --board=%s %s remote/latest', self.options.board,
self.options.device.raw)
raise
def SendPerfValues(self, test_results_dir):
"""Gather all perf values in |test_results_dir| and send them to chromeperf.
The uploading will be retried 3 times for each file.
Args:
test_results_dir: A path to the directory with perf files.
"""
# A dict of list of perf values, keyed by test name.
perf_entries = collections.defaultdict(list)
for root, _, filenames in os.walk(test_results_dir):
for relative_name in filenames:
if not image_test_lib.IsPerfFile(relative_name):
continue
full_name = os.path.join(root, relative_name)
entries = perf_uploader.LoadPerfValues(full_name)
test_name = image_test_lib.ImageTestCase.GetTestName(
relative_name)
perf_entries[test_name].extend(entries)
platform_name = self._run.bot_id
try:
cros_ver = self._run.GetVersionInfo().VersionString()
except cbuildbot_run.VersionNotSetError:
logging.error('Could not obtain version info. '
'Failed to upload perf results.')
return
chrome_ver = self._run.DetermineChromeVersion()
for test_name, perf_values in perf_entries.items():
self._UploadPerfValues(perf_values,
platform_name,
test_name,
cros_version=cros_ver,
chrome_version=chrome_ver)
def _CheckFile(self, basename):
"""Validates the passwd or group file."""
match_func = getattr(self, '_match_%s' % basename)
validate_func = getattr(self, '_validate_%s' % basename)
expected_entries = self._LoadBaseline(basename)
actual_entries = self._LoadPath(
os.path.join(image_test_lib.ROOT_A, 'etc', basename))
success = True
for entry, details in actual_entries.items():
if entry not in expected_entries:
is_valid = validate_func(details)
if not is_valid:
logging.error('Unexpected %s entry for "%s".', basename,
entry)
success = success and is_valid
continue
expected = expected_entries[entry]
match_res = match_func(expected, details)
success = success and match_res
missing = set(expected_entries.keys()) - set(actual_entries.keys())
for m in missing:
logging.info('Ignoring missing %s entry for "%s".', basename, m)
self.assertTrue(success)
def run(self):
"""Launches a nebraska process on the device.
Starts a background nebraska and waits for it to finish.
"""
logging.info('Starting nebraska on %s', self._hostname)
if not self._update_metadata_dir:
raise NebraskaStartupError(
'Update metadata directory location is not passed.')
cmd = [
'python',
self._nebraska_bin,
'--update-metadata',
self._update_metadata_dir,
]
if self._update_payloads_address:
cmd += ['--update-payloads-address', self._update_payloads_address]
if self._install_metadata_dir:
cmd += ['--install-metadata', self._install_metadata_dir]
if self._install_payloads_address:
cmd += [
'--install-payloads-address', self._install_payloads_address
]
try:
self._RemoteCommand(cmd,
redirect_stdout=True,
combine_stdout_stderr=True)
except cros_build_lib.RunCommandError as err:
msg = 'Remote nebraska failed (to start): %s' % str(err)
logging.error(msg)
raise NebraskaStartupError(msg)
def _CheckConnection(self):
try:
logging.info('Testing connection to the device...')
self.device.RunCommand('true')
except cros_build_lib.RunCommandError as ex:
logging.error('Error connecting to the test device.')
raise DeployFailure(ex)
def PerformStage(self):
# These directories are used later to archive test artifacts.
test_results_root = commands.CreateTestRoot(self._build_root)
test_basename = _GCE_TEST_RESULTS % dict(attempt=self._attempt)
if self._test_basename:
test_basename = self._test_basename
try:
if not self._gce_tests:
self._gce_tests = self._run.config.gce_tests
for gce_test in self._gce_tests:
logging.info('Running GCE test %s.', gce_test.test_type)
if gce_test.test_type == constants.GCE_SUITE_TEST_TYPE:
per_test_results_dir = os.path.join(
test_results_root, gce_test.test_suite)
else:
per_test_results_dir = os.path.join(
test_results_root, gce_test.test_type)
with cgroups.SimpleContainChildren('GCETest'):
r = ' Reached GCETestStage test run timeout.'
with timeout_util.Timeout(self.TEST_TIMEOUT,
reason_message=r):
self._RunTest(gce_test, per_test_results_dir)
except Exception:
# pylint: disable=logging-not-lazy
logging.error(
_ERROR_MSG %
dict(test_name='GCETests', test_results=test_basename))
raise
finally:
self._ArchiveTestResults(test_results_root, test_basename)
def GetRunningPids(self, exe, full_path=True):
"""Get all the running pids on the device with the executable path.
Args:
exe: The executable path to get pids for.
full_path: Whether |exe| is a full executable path.
Raises:
RunningPidsError when failing to parse out pids from command output.
SSHConnectionError when error occurs during SSH connection.
"""
try:
cmd = ['pgrep', exe]
if full_path:
cmd.append('-f')
result = self.GetAgent().RemoteSh(cmd,
error_code_ok=True,
capture_output=True)
try:
return [int(pid) for pid in result.output.splitlines()]
except ValueError:
logging.error('Parsing output failed:\n%s', result.output)
raise RunningPidsError('Unable to get running pids of %s' %
exe)
except SSHConnectionError:
logging.error('Error connecting to device %s', self.hostname)
raise
def _DisableRootfsVerification(self):
if not self.options.force:
logging.error(
'Detected that the device has rootfs verification enabled.')
logging.info(
'This script can automatically remove the rootfs '
'verification, which requires that it reboot the device.')
logging.info('Make sure the device is in developer mode!')
logging.info('Skip this prompt by specifying --force.')
if not cros_build_lib.BooleanPrompt('Remove rootfs verification?',
False):
return False
logging.info('Removing rootfs verification from %s', self.options.to)
# Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
# Use --force to bypass the checks.
cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
'--remove_rootfs_verification --force')
for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
self.device.RunCommand(cmd % partition, error_code_ok=True)
self._Reboot()
# Now that the machine has been rebooted, we need to kill Chrome again.
self._KillProcsIfNeeded()
# Make sure the rootfs is writable now.
self._MountRootfsAsWritable(error_code_ok=False)
return True
def __call__(self, value):
"""Parses a device input and enforces constraints.
DeviceParser is an object so that a set of valid schemes can be specified,
but argparse expects a parsing function, so we overload __call__() for
argparse to use.
Args:
value: String representing a device target. See class comments for
valid device input formats.
Returns:
A Device object.
Raises:
ValueError: |value| is not a valid device specifier or doesn't
match the supported list of schemes.
"""
try:
device = self._ParseDevice(value)
self._EnforceConstraints(device, value)
return device
except ValueError as e:
# argparse ignores exception messages, so print the message manually.
logging.error(e)
raise
except Exception as e:
logging.error('Internal error while parsing device input: %s', e)
raise
def lsb_release(self):
"""The /etc/lsb-release content on the device.
Returns a dict of entries in /etc/lsb-release file. If multiple entries
have the same key, only the first entry is recorded. Returns an empty dict
if the reading command failed or the file is corrupted (i.e., does not have
the format of <key>=<value> for every line).
"""
if not self._lsb_release:
try:
content = self.CatFile(constants.LSB_RELEASE_PATH,
max_size=None)
except CatFileError as e:
logging.debug('Failed to read "%s" on the device: %s',
constants.LSB_RELEASE_PATH, e)
else:
try:
self._lsb_release = dict(
e.split('=', 1)
for e in reversed(content.splitlines()))
except ValueError:
logging.error('File "%s" on the device is mal-formatted.',
constants.LSB_RELEASE_PATH)
return self._lsb_release
def PerformStage(self):
template_gs_paths = self._ListTemplates()
if not template_gs_paths:
logging.info('No template files found. No need to update configs.')
return
chromite_dir = CreateTmpGitRepo('chromite', constants.CHROMITE_URL)
successful = True
failed_templates = []
for template_gs_path in template_gs_paths:
try:
branch = GetBranchName(os.path.basename(template_gs_path))
if branch:
UpdateConfigStage(self._run,
template_gs_path,
branch,
chromite_dir,
suffix='_' + branch).Run()
except Exception as e:
successful = False
failed_templates.append(template_gs_path)
logging.error('Failed to update configs for %s: %s',
template_gs_path, e)
traceback.print_exc()
# If UpdateConfigStage failures happened, raise a exception
if not successful:
raise UpdateConfigException('Failed to update config for %s' %
failed_templates)
def run(self):
"""Launches a devserver process on the device."""
self._RunCommand(['cat', '/dev/null', '>|', self.log_file])
port = self.port if self.port else 0
cmd = [
'python2', self.devserver_bin,
'--logfile=%s' % self.log_file, '--pidfile', self._pid_file,
'--port=%d' % port, '--critical_update'
]
if not self.port:
cmd.append('--portfile=%s' % self.port_file)
if self.static_dir:
cmd.append('--static_dir=%s' % self.static_dir)
logging.info('Starting devserver on %s', self.hostname)
result = self._RunCommand(cmd,
error_code_ok=True,
redirect_stdout=True,
combine_stdout_stderr=True)
if result.returncode != 0:
msg = (('Remote devserver failed to start!\n'
'--- Start output from the devserver startup command ---\n'
'%s'
'--- End output from the devserver startup command ---') %
(result.output))
logging.error(msg)
if 'ImportError: No module named cherrypy' in result.output:
logging.error(self.CHERRYPY_ERROR_MSG)
def GetRunningPids(self, exe, full_path=True):
"""Get all the running pids on the device with the executable path.
Args:
exe: The executable path to get pids for.
full_path: Whether |exe| is a full executable path.
Raises:
RunningPidsError when failing to parse out pids from command output.
SSHConnectionError when error occurs during SSH connection.
"""
try:
cmd = ['pgrep', exe]
if full_path:
cmd.append('-f')
result = self.GetAgent().RemoteSh(cmd, error_code_ok=True,
capture_output=True)
try:
return [int(pid) for pid in result.output.splitlines()]
except ValueError:
logging.error('Parsing output failed:\n%s', result.output)
raise RunningPidsError('Unable to get running pids of %s' % exe)
except SSHConnectionError:
logging.error('Error connecting to device %s', self.hostname)
raise
def Run(self):
files = self.options.files
if not files:
# Running with no arguments is allowed to make the repo upload hook
# simple, but print a warning so that if someone runs this manually
# they are aware that nothing was linted.
logging.warning('No files provided to lint. Doing nothing.')
errors = multiprocessing.Value('i')
linter_map = _BreakoutFilesByLinter(files)
dispatcher = functools.partial(_Dispatcher, errors,
self.options.output, self.options.debug)
# Special case one file as it's common -- faster to avoid parallel startup.
if sum([len(x) for _, x in linter_map.iteritems()]) == 1:
linter, files = linter_map.items()[0]
dispatcher(linter, files[0])
else:
# Run the linter in parallel on the files.
with parallel.BackgroundTaskRunner(dispatcher) as q:
for linter, files in linter_map.iteritems():
for path in files:
q.put([linter, path])
if errors.value:
logging.error('linter found errors in %i files', errors.value)
sys.exit(1)
def ShouldRetryCommandCommon(exc):
"""Returns whether any RunCommand should retry on a given exception."""
if not isinstance(exc, cros_build_lib.RunCommandError):
return False
if exc.result.returncode is None:
logging.error("Child process failed to launch; not retrying:\n" "command: %s", exc.result.cmdstr)
return False
return True