def AdjustSymbolFileSize(symbol, tempdir, file_limit):
"""Examine symbols files for size problems, and reduce if needed.
If the symbols size is too big, strip out the call frame info. The CFI
is unnecessary for 32bit x86 targets where the frame pointer is used (as
all of ours have) and it accounts for over half the size of the symbols
uploaded.
Stripped files will be created inside tempdir, and will be the callers
responsibility to clean up.
We also warn, if a symbols file is still too large after stripping.
Args:
symbol: SymbolFile instance to be examined and modified as needed..
tempdir: A temporary directory we can create files in that the caller will
clean up.
file_limit: We only strip files which are larger than this limit.
Returns:
SymbolFile instance (original or modified as needed)
"""
file_size = symbol.FileSize()
if file_limit and symbol.FileSize() > file_limit:
with cros_build_lib.UnbufferedNamedTemporaryFile(
prefix='upload_symbols',
dir=tempdir, delete=False) as temp_sym_file:
with open(symbol.file_name, 'rb') as fp:
temp_sym_file.writelines(
x for x in fp.readlines() if not x.startswith(b'STACK CFI')
)
original_file_size = file_size
symbol.file_name = temp_sym_file.name
file_size = symbol.FileSize()
logging.warning('stripped CFI for %s reducing size %s > %s',
symbol.display_name, original_file_size, file_size)
# Hopefully the crash server will let it through. But it probably won't.
# Not sure what the best answer is in this case.
if file_size >= CRASH_SERVER_FILE_LIMIT:
logging.PrintBuildbotStepWarnings()
logging.warning('upload file %s is awfully large, risking rejection by '
'the symbol server (%s > %s)', symbol.display_path,
file_size, CRASH_SERVER_FILE_LIMIT)
return symbol
def start(self):
"""Invoke multiprocessing.Process.start after flushing output/err."""
if self.SILENT_TIMEOUT < self.MINIMUM_SILENT_TIMEOUT:
raise AssertionError('Maximum recursion depth exceeded in %r' %
self)
sys.stdout.flush()
sys.stderr.flush()
tmp_dir = '/tmp/chromite.parallel.%d' % os.geteuid()
osutils.SafeMakedirs(tmp_dir, mode=0o1777)
self._output = cros_build_lib.UnbufferedNamedTemporaryFile(
delete=False, dir=tmp_dir, prefix='chromite-parallel-')
self._parent_pid = os.getpid()
return multiprocessing.Process.start(self)
def GenerateBreakpadSymbol(elf_file, debug_file=None, breakpad_dir=None,
strip_cfi=False, num_errors=None,
dump_syms_cmd='dump_syms'):
"""Generate the symbols for |elf_file| using |debug_file|
Args:
elf_file: The file to dump symbols for
debug_file: Split debug file to use for symbol information
breakpad_dir: The dir to store the output symbol file in
strip_cfi: Do not generate CFI data
num_errors: An object to update with the error count (needs a .value member)
dump_syms_cmd: Command to use for dumping symbols.
Returns:
The name of symbol file written out.
"""
assert breakpad_dir
if num_errors is None:
num_errors = ctypes.c_int()
cmd_base = [dump_syms_cmd, '-v']
if strip_cfi:
cmd_base += ['-c']
# Some files will not be readable by non-root (e.g. set*id /bin/su).
needs_sudo = not os.access(elf_file, os.R_OK)
def _DumpIt(cmd_args):
if needs_sudo:
run_command = cros_build_lib.sudo_run
else:
run_command = cros_build_lib.run
return run_command(
cmd_base + cmd_args, stderr=True, stdout=temp.name,
check=False, debug_level=logging.DEBUG)
def _CrashCheck(ret, msg):
if ret < 0:
logging.PrintBuildbotStepWarnings()
logging.warning('dump_syms crashed with %s; %s',
signals.StrSignal(-ret), msg)
osutils.SafeMakedirs(breakpad_dir)
with cros_build_lib.UnbufferedNamedTemporaryFile(
dir=breakpad_dir, delete=False) as temp:
if debug_file:
# Try to dump the symbols using the debug file like normal.
cmd_args = [elf_file, os.path.dirname(debug_file)]
result = _DumpIt(cmd_args)
if result.returncode:
# Sometimes dump_syms can crash because there's too much info.
# Try dumping and stripping the extended stuff out. At least
# this way we'll get the extended symbols. https://crbug.com/266064
_CrashCheck(result.returncode, 'retrying w/out CFI')
cmd_args = ['-c', '-r'] + cmd_args
result = _DumpIt(cmd_args)
_CrashCheck(result.returncode, 'retrying w/out debug')
basic_dump = result.returncode
else:
basic_dump = True
if basic_dump:
# If that didn't work (no debug, or dump_syms still failed), try
# dumping just the file itself directly.
result = _DumpIt([elf_file])
if result.returncode:
# A lot of files (like kernel files) contain no debug information,
# do not consider such occurrences as errors.
logging.PrintBuildbotStepWarnings()
_CrashCheck(result.returncode, 'giving up entirely')
if b'file contains no debugging information' in result.stderr:
logging.warning('no symbols found for %s', elf_file)
else:
num_errors.value += 1
logging.error('dumping symbols for %s failed:\n%s', elf_file,
result.error)
os.unlink(temp.name)
return num_errors.value
# Move the dumped symbol file to the right place:
# /build/$BOARD/usr/lib/debug/breakpad/<module-name>/<id>/<module-name>.sym
header = ReadSymsHeader(temp)
logging.info('Dumped %s as %s : %s', elf_file, header.name, header.id)
sym_file = os.path.join(breakpad_dir, header.name, header.id,
header.name + '.sym')
osutils.SafeMakedirs(os.path.dirname(sym_file))
os.rename(temp.name, sym_file)
os.chmod(sym_file, 0o644)
return sym_file
def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
sign_types=None, dry_run=False, mock=False, force_keysets=(),
force_channels=None, buildroot=constants.SOURCE_ROOT):
"""Push the image from the archive bucket to the release bucket.
Args:
src_path: Where to copy the files from; can be a local path or gs:// URL.
Should be a full path to the artifacts in either case.
board: The board we're uploading artifacts for (e.g. $BOARD).
versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
profile: The board profile in use (e.g. "asan").
priority: Set the signing priority (lower == higher prio).
sign_types: If set, a set of types which we'll restrict ourselves to
signing. See the --sign-types option for more details.
dry_run: Show what would be done, but do not upload anything.
mock: Upload to a testing bucket rather than the real one.
force_keysets: Set of keysets to use rather than what the inputs say.
force_channels: Set of channels to use rather than what the inputs say.
buildroot: Buildroot in which to look for signing instructions.
Returns:
A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
'gs://signer_instruction_uri2',
...]
"""
# Whether we hit an unknown error. If so, we'll throw an error, but only
# at the end (so that we still upload as many files as possible).
# It's implemented using a list to deal with variable scopes in nested
# functions below.
unknown_error = [False]
if versionrev is None:
# Extract milestone/version from the directory name.
versionrev = os.path.basename(src_path)
# We only support the latest format here. Older releases can use pushimage
# from the respective branch which deals with legacy cruft.
m = re.match(VERSION_REGEX, versionrev)
if not m:
raise ValueError('version %s does not match %s' %
(versionrev, VERSION_REGEX))
milestone = m.group(1)
version = m.group(2)
# Normalize board to always use dashes not underscores. This is mostly a
# historical artifact at this point, but we can't really break it since the
# value is used in URLs.
boardpath = board.replace('_', '-')
if profile is not None:
boardpath += '-%s' % profile.replace('_', '-')
ctx = gs.GSContext(dry_run=dry_run)
try:
input_insns = InputInsns(board, buildroot=buildroot)
except MissingBoardInstructions as e:
logging.warning('Missing base instruction file: %s', e)
logging.warning('not uploading anything for signing')
return
if force_channels is None:
channels = input_insns.GetChannels()
else:
# Filter out duplicates.
channels = sorted(set(force_channels))
# We want force_keysets as a set.
force_keysets = set(force_keysets)
if mock:
logging.info('Upload mode: mock; signers will not process anything')
tbs_base = gs_base = os.path.join(constants.TRASH_BUCKET, 'pushimage-tests',
getpass.getuser())
elif set(['%s-%s' % (TEST_KEYSET_PREFIX, x)
for x in TEST_KEYSETS]) & force_keysets:
logging.info('Upload mode: test; signers will process test keys')
# We need the tbs_base to be in the place the signer will actually scan.
tbs_base = TEST_SIGN_BUCKET_BASE
gs_base = os.path.join(tbs_base, getpass.getuser())
else:
logging.info('Upload mode: normal; signers will process the images')
tbs_base = gs_base = constants.RELEASE_BUCKET
sect_general = {
'config_board': board,
'board': boardpath,
'version': version,
'versionrev': versionrev,
'milestone': milestone,
}
sect_insns = {}
if dry_run:
logging.info('DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED')
logging.info('Signing for channels: %s', ' '.join(channels))
instruction_urls = {}
def _ImageNameBase(image_type=None):
lmid = ('%s-' % image_type) if image_type else ''
return 'ChromeOS-%s%s-%s' % (lmid, versionrev, boardpath)
# These variables are defined outside the loop so that the nested functions
# below can access them without 'cell-var-from-loop' linter warning.
dst_path = ''
files_to_sign = []
for channel in channels:
logging.debug('\n\n#### CHANNEL: %s ####\n', channel)
sect_insns['channel'] = channel
sub_path = '%s-channel/%s/%s' % (channel, boardpath, version)
dst_path = '%s/%s' % (gs_base, sub_path)
logging.info('Copying images to %s', dst_path)
recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
nv_lp0_firmware_basename = _ImageNameBase(
constants.IMAGE_TYPE_NV_LP0_FIRMWARE)
acc_usbpd_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_USBPD)
acc_rwsig_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_RWSIG)
cr50_firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_CR50_FIRMWARE)
test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
hwqual_tarball = 'chromeos-hwqual-%s-%s.tar.bz2' % (board, versionrev)
# The following build artifacts, if present, are always copied regardless of
# requested signing types.
files_to_copy_only = (
# (<src>, <dst>, <suffix>),
('image.zip', _ImageNameBase(), 'zip'),
(constants.TEST_IMAGE_TAR, test_basename, 'tar.xz'),
('debug.tgz', 'debug-%s' % boardpath, 'tgz'),
(hwqual_tarball, None, None),
('stateful.tgz', None, None),
('dlc', None, None),
(constants.QUICK_PROVISION_PAYLOAD_KERNEL, None, None),
(constants.QUICK_PROVISION_PAYLOAD_ROOTFS, None, None),
)
# The following build artifacts, if present, are always copied.
# If |sign_types| is None, all of them are marked for signing, otherwise
# only the image types specified in |sign_types| are marked for signing.
files_to_copy_and_maybe_sign = (
# (<src>, <dst>, <suffix>, <signing type>),
(constants.RECOVERY_IMAGE_TAR, recovery_basename, 'tar.xz',
constants.IMAGE_TYPE_RECOVERY),
('factory_image.zip', factory_basename, 'zip',
constants.IMAGE_TYPE_FACTORY),
('firmware_from_source.tar.bz2', firmware_basename, 'tar.bz2',
constants.IMAGE_TYPE_FIRMWARE),
('firmware_from_source.tar.bz2', nv_lp0_firmware_basename, 'tar.bz2',
constants.IMAGE_TYPE_NV_LP0_FIRMWARE),
('firmware_from_source.tar.bz2', acc_usbpd_basename, 'tar.bz2',
constants.IMAGE_TYPE_ACCESSORY_USBPD),
('firmware_from_source.tar.bz2', acc_rwsig_basename, 'tar.bz2',
constants.IMAGE_TYPE_ACCESSORY_RWSIG),
('firmware_from_source.tar.bz2', cr50_firmware_basename, 'tar.bz2',
constants.IMAGE_TYPE_CR50_FIRMWARE),
)
# The following build artifacts are copied and marked for signing, if
# they are present *and* if the image type is specified via |sign_types|.
files_to_maybe_copy_and_sign = (
# (<src>, <dst>, <suffix>, <signing type>),
(constants.BASE_IMAGE_TAR, base_basename, 'tar.xz',
constants.IMAGE_TYPE_BASE),
)
def _CopyFileToGS(src, dst=None, suffix=None):
"""Returns |dst| file name if the copying was successful."""
if dst is None:
dst = src
elif suffix is not None:
dst = '%s.%s' % (dst, suffix)
success = False
try:
ctx.Copy(os.path.join(src_path, src), os.path.join(dst_path, dst),
recursive=True)
success = True
except gs.GSNoSuchKey:
logging.warning('Skipping %s as it does not exist', src)
except gs.GSContextException:
unknown_error[0] = True
logging.error('Skipping %s due to unknown GS error', src, exc_info=True)
return dst if success else None
for src, dst, suffix in files_to_copy_only:
_CopyFileToGS(src, dst, suffix)
# Clear the list of files to sign before adding new artifacts.
files_to_sign = []
def _AddToFilesToSign(image_type, dst, suffix):
assert dst.endswith('.' + suffix), (
'dst: %s, suffix: %s' % (dst, suffix))
dst_base = dst[:-(len(suffix) + 1)]
files_to_sign.append([image_type, dst_base, suffix])
for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
dst = _CopyFileToGS(src, dst, suffix)
if dst and (not sign_types or image_type in sign_types):
_AddToFilesToSign(image_type, dst, suffix)
for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
if sign_types and image_type in sign_types:
dst = _CopyFileToGS(src, dst, suffix)
if dst:
_AddToFilesToSign(image_type, dst, suffix)
logging.debug('Files to sign: %s', files_to_sign)
# Now go through the subset for signing.
for image_type, dst_name, suffix in files_to_sign:
try:
input_insns = InputInsns(board, image_type=image_type,
buildroot=buildroot)
except MissingBoardInstructions as e:
logging.info('Nothing to sign: %s', e)
continue
dst_archive = '%s.%s' % (dst_name, suffix)
sect_general['archive'] = dst_archive
sect_general['type'] = image_type
# In the default/automatic mode, only flag files for signing if the
# archives were actually uploaded in a previous stage. This additional
# check can be removed in future once |sign_types| becomes a required
# argument.
# TODO: Make |sign_types| a required argument.
gs_artifact_path = os.path.join(dst_path, dst_archive)
exists = False
try:
exists = ctx.Exists(gs_artifact_path)
except gs.GSContextException:
unknown_error[0] = True
logging.error('Unknown error while checking %s', gs_artifact_path,
exc_info=True)
if not exists:
logging.info('%s does not exist. Nothing to sign.',
gs_artifact_path)
continue
first_image = True
for alt_insn_set in input_insns.GetAltInsnSets():
# Figure out which keysets have been requested for this type.
# We sort the forced set so tests/runtime behavior is stable.
keysets = sorted(force_keysets)
if not keysets:
keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
if not keysets:
logging.warning('Skipping %s image signing due to no keysets',
image_type)
for keyset in keysets:
sect_insns['keyset'] = keyset
# Generate the insn file for this artifact that the signer will use,
# and flag it for signing.
with cros_build_lib.UnbufferedNamedTemporaryFile(
prefix='pushimage.insns.') as insns_path:
input_insns.OutputInsns(insns_path.name, sect_insns, sect_general,
insns_merge=alt_insn_set)
gs_insns_path = '%s/%s' % (dst_path, dst_name)
if not first_image:
gs_insns_path += '-%s' % keyset
first_image = False
gs_insns_path += '.instructions'
try:
ctx.Copy(insns_path.name, gs_insns_path)
except gs.GSContextException:
unknown_error[0] = True
logging.error('Unknown error while uploading insns %s',
gs_insns_path, exc_info=True)
continue
try:
MarkImageToBeSigned(ctx, tbs_base, gs_insns_path, priority)
except gs.GSContextException:
unknown_error[0] = True
logging.error('Unknown error while marking for signing %s',
gs_insns_path, exc_info=True)
continue
logging.info('Signing %s image with keyset %s at %s', image_type,
keyset, gs_insns_path)
instruction_urls.setdefault(channel, []).append(gs_insns_path)
if unknown_error[0]:
raise PushError('hit some unknown error(s)', instruction_urls)
return instruction_urls