def main(argv):
parser = commandline.ArgumentParser(description=__doc__)
parser.add_argument('--board',
default=None,
help='board to generate symbols for')
parser.add_argument('--breakpad_root',
type='path',
default=None,
help='root directory for breakpad symbols')
parser.add_argument('--generate-count',
type=int,
default=None,
help='only generate # number of symbols')
parser.add_argument('--strip_cfi',
action='store_true',
default=False,
help='do not generate CFI data (pass -c to dump_syms)')
opts = parser.parse_args(argv)
if opts.board is None:
cros_build_lib.Die('--board is required')
ret = GenerateBreakpadSymbols(opts.board,
breakpad_dir=opts.breakpad_root,
strip_cfi=opts.strip_cfi,
generate_count=opts.generate_count)
if ret:
cros_build_lib.Error('encountered %i problem(s)', ret)
# Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
# return 0 in case we are a multiple of the mask.
ret = 1
return ret
def SyncSources(self):
repo = repository.RepoRepository(self.manifest_dir,
self.repo_root,
referenced_repo=self.reference,
manifest=MANIFEST_FILE,
branch='master')
# Trigger the network sync
repo.Sync(jobs=multiprocessing.cpu_count() + 1, network_only=True)
projects = [self.CHROMIUM_ROOT]
if self.internal:
projects.append(self.CHROME_ROOT)
for project in projects:
path = os.path.join(self.repo_root, project)
if os.path.exists(path):
try:
git.CleanAndCheckoutUpstream(path, refresh_upstream=False)
continue
except cros_build_lib.RunCommandError:
cros_build_lib.Error("Failed cleaning %r; wiping.", path)
cros_build_lib.SudoRunCommand(['rm', '-rf', path],
print_cmd=False)
cros_build_lib.RunCommand(['repo', 'sync', '-ld', project],
cwd=self.repo_root)
def _HandleStageException(self, exception):
"""Called when PerformStage throws an exception. Can be overriden.
Should return result, description. Description should be None if result
is not an exception.
"""
# Tell the user about the exception, and record it
description = self._StringifyException(exception)
cros_build_lib.PrintBuildbotStepFailure()
cros_build_lib.Error(description)
return exception, description
def GrabRemotePackageIndex(binhost_url):
"""Grab the latest binary package database from the specified URL.
Args:
binhost_url: Base URL of remote packages (PORTAGE_BINHOST).
Returns:
A PackageIndex object, if the Packages file can be retrieved. If the
packages file cannot be retrieved, then None is returned.
"""
url = '%s/Packages' % binhost_url.rstrip('/')
pkgindex = PackageIndex()
if binhost_url.startswith('http'):
try:
f = _RetryUrlOpen(url)
except urllib2.HTTPError as e:
if e.code in HTTP_FORBIDDEN_CODES:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Error('Cannot GET %s: %s' % (url, str(e)))
return None
# Not found errors are normal if old prebuilts were cleaned out.
if e.code in HTTP_NOT_FOUND_CODES:
return None
raise
elif binhost_url.startswith('gs://'):
cmd = [gs.GSUTIL_BIN, 'cat', url]
try:
output = cros_build_lib.RunCommand(cmd,
redirect_stdout=True,
print_cmd=False).output
except cros_build_lib.RunCommandError as e:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Error('Cannot GET %s: %s' % (url, str(e)))
return None
f = cStringIO.StringIO(output)
else:
return None
pkgindex.Read(f)
pkgindex.header.setdefault('URI', binhost_url)
f.close()
return pkgindex
def DownloadCrx(ext, extension, outputdir):
"""Download .crx file from WebStore and update entry."""
cros_build_lib.Info('Extension "%s"(%s)...', extension['name'], ext)
update_url = '%s?x=id%%3D%s%%26uc' % (extension['external_update_url'],
ext)
response = urllib.urlopen(update_url)
if response.getcode() != 200:
cros_build_lib.Error('Cannot get update response, URL: %s, error: %d',
update_url, response.getcode())
return False
dom = xml.dom.minidom.parse(response)
status = dom.getElementsByTagName('app')[0].getAttribute('status')
if status != 'ok':
cros_build_lib.Error('Cannot fetch extension, status: %s', status)
return False
node = dom.getElementsByTagName('updatecheck')[0]
url = node.getAttribute('codebase')
version = node.getAttribute('version')
filename = '%s-%s.crx' % (ext, version)
response = urllib.urlopen(url)
if response.getcode() != 200:
cros_build_lib.Error('Cannot download extension, URL: %s, error: %d',
url, response.getcode())
return False
osutils.WriteFile(os.path.join(outputdir, 'extensions', filename),
response.read())
# Has to delete because only one of 'external_crx' or
# 'external_update_url' should present for the extension.
del extension['external_update_url']
extension['external_crx'] = os.path.join(EXTENSIONS_CACHE_PREFIX, filename)
extension['external_version'] = version
cros_build_lib.Info('Downloaded, current version %s', version)
return True
def SubmitChange(self, change, dryrun=False):
"""Submits patch using Gerrit Review."""
cmd = self.GetGerritReviewCommand([
'--submit',
'%s,%s' % (change.gerrit_number, change.patch_number)
])
if dryrun:
logging.info('Would have run: %s', ' '.join(map(repr, cmd)))
return
try:
cros_build_lib.RunCommand(cmd)
except cros_build_lib.RunCommandError:
cros_build_lib.Error('Command failed', exc_info=True)
def TestNewManifest(self):
"""Runs a 'repo sync' off of new manifest to verify things aren't broken."""
# Do as cheap a sync as possible; network only is good enough,
# allow shallow cloning if we don't have a reference, and sync
# strictly the target branch.
repo = repository.RepoRepository(
self.manifest_dir, self.repo_root, branch=TEST_BRANCH,
referenced_repo=self.reference, manifest=MANIFEST_FILE)
try:
repo.Sync(jobs=multiprocessing.cpu_count() + 1, network_only=True)
except Exception:
cros_build_lib.Error('Failed to sync with new manifest!')
raise
def _Submit(self, testjob, dryrun):
"""Internal submission function. See Submit() for arg description."""
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
self.manifest.AssertProjectIsPushable(patch.project)
data = self.manifest.projects[patch.project]
print 'Uploading patch %s' % patch
patch.Upload(data['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
tag = constants.EXTERNAL_PATCH_TAG
if data['remote'] == constants.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s'
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
self._VerifyForBuildbot()
repository.CloneGitRepo(self.tryjob_repo, self.ssh_url)
version_path = os.path.join(self.tryjob_repo,
self.TRYJOB_FORMAT_FILE)
with open(version_path, 'r') as f:
try:
val = int(f.read().strip())
except ValueError:
raise ChromiteUpgradeNeeded()
if val > self.TRYJOB_FORMAT_VERSION:
raise ChromiteUpgradeNeeded(val)
push_branch = manifest_version.PUSH_BRANCH
remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None
git.CreatePushBranch(push_branch, self.tryjob_repo, sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user,
current_time)
user_dir = os.path.join(self.tryjob_repo, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
cros_build_lib.RunCommand(['git', 'add', fullpath], cwd=self.tryjob_repo)
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL' : self.user_email,
'GIT_AUTHOR_EMAIL' : self.user_email,
}
cros_build_lib.RunCommand(['git', 'commit', '-m', self.description],
cwd=self.tryjob_repo, extra_env=extra_env)
try:
git.PushWithRetry(
push_branch, self.tryjob_repo, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
cros_build_lib.Error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def UploadSymbol(sym_file,
upload_url,
file_limit=DEFAULT_FILE_LIMIT,
sleep=0,
num_errors=None):
"""Upload |sym_file| to |upload_url|
Args:
sym_file: The full path to the breakpad symbol to upload
upload_url: The crash server to upload things to
file_limit: The max file size of a symbol file before we try to strip it
sleep: Number of seconds to sleep before running
num_errors: An object to update with the error count (needs a .value member)
Returns:
The number of errors that were encountered.
"""
if num_errors is None:
num_errors = ctypes.c_int()
elif num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY:
# Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
return 0
upload_file = sym_file
if sleep:
# Keeps us from DoS-ing the symbol server.
time.sleep(sleep)
cros_build_lib.Debug('uploading %s' % sym_file)
# Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
with tempfile.NamedTemporaryFile(prefix='upload_symbols',
bufsize=0) as temp_sym_file:
if file_limit:
# If the symbols size is too big, strip out the call frame info. The CFI
# is unnecessary for 32bit x86 targets where the frame pointer is used (as
# all of ours have) and it accounts for over half the size of the symbols
# uploaded.
file_size = os.path.getsize(sym_file)
if file_size > file_limit:
cros_build_lib.Warning(
'stripping CFI from %s due to size %s > %s', sym_file,
file_size, file_limit)
temp_sym_file.writelines([
x for x in open(sym_file, 'rb').readlines()
if not x.startswith('STACK CFI')
])
upload_file = temp_sym_file.name
# Hopefully the crash server will let it through. But it probably won't.
# Not sure what the best answer is in this case.
file_size = os.path.getsize(upload_file)
if file_size > CRASH_SERVER_FILE_LIMIT:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Error(
'upload file %s is awfully large, risking rejection '
'by symbol server (%s > %s)', sym_file, file_size,
CRASH_SERVER_FILE_LIMIT)
num_errors.value += 1
# Upload the symbol file.
try:
cros_build_lib.RetryCommand(SymUpload,
MAX_RETRIES,
upload_file,
upload_url,
sleep=INITIAL_RETRY_DELAY)
cros_build_lib.Info('successfully uploaded %10i bytes: %s',
file_size, os.path.basename(sym_file))
except cros_build_lib.RunCommandError as e:
cros_build_lib.Warning(
'could not upload: %s:\n{stdout} %s\n{stderr} %s',
os.path.basename(sym_file), e.result.output, e.result.error)
num_errors.value += 1
return num_errors.value
def main(argv):
parser = commandline.ArgumentParser(description=__doc__)
parser.add_argument('sym_files', type='path', nargs='*', default=None)
parser.add_argument('--board',
default=None,
help='board to build packages for')
parser.add_argument('--breakpad_root',
type='path',
default=None,
help='root directory for breakpad symbols')
parser.add_argument('--official_build',
action='store_true',
default=False,
help='point to official symbol server')
parser.add_argument('--regenerate',
action='store_true',
default=False,
help='regenerate all symbols')
parser.add_argument('--upload-count',
type=int,
default=None,
help='only upload # number of symbols')
parser.add_argument('--strip_cfi',
type=int,
default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024),
help='strip CFI data for files above this size')
parser.add_argument('--testing',
action='store_true',
default=False,
help='run in testing mode')
parser.add_argument('--yes',
action='store_true',
default=False,
help='answer yes to all prompts')
opts = parser.parse_args(argv)
if opts.sym_files:
if opts.regenerate:
cros_build_lib.Die(
'--regenerate may not be used with specific files')
else:
if opts.board is None:
cros_build_lib.Die('--board is required')
if opts.breakpad_root and opts.regenerate:
cros_build_lib.Die('--regenerate may not be used with --breakpad_root')
if opts.testing:
# TODO(build): Kill off --testing mode once unittests are up-to-snuff.
cros_build_lib.Info('running in testing mode')
# pylint: disable=W0601,W0603
global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY
INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0
SymUpload = TestingSymUpload
if not opts.yes:
query = textwrap.wrap(
textwrap.dedent("""
Uploading symbols for an entire Chromium OS build is really only
necessary for release builds and in a few cases for developers
to debug problems. It will take considerable time to run. For
developer debugging purposes, consider instead passing specific
files to upload.
"""), 80)
cros_build_lib.Warning('\n%s', '\n'.join(query))
if not cros_build_lib.BooleanPrompt(
prompt='Are you sure you want to upload all build symbols',
default=False):
cros_build_lib.Die('better safe than sorry')
ret = 0
if opts.regenerate:
ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols(
opts.board, breakpad_dir=opts.breakpad_root)
ret += UploadSymbols(opts.board,
official=opts.official_build,
breakpad_dir=opts.breakpad_root,
file_limit=opts.strip_cfi,
sleep=DEFAULT_SLEEP_DELAY,
upload_count=opts.upload_count,
sym_files=opts.sym_files)
if ret:
cros_build_lib.Error('encountered %i problem(s)', ret)
# Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
# return 0 in case we are a multiple of the mask.
ret = 1
return ret
def GenerateBreakpadSymbol(elf_file,
debug_file=None,
breakpad_dir=None,
board=None,
strip_cfi=False,
num_errors=None):
"""Generate the symbols for |elf_file| using |debug_file|
Args:
elf_file: The file to dump symbols for
debug_file: Split debug file to use for symbol information
breakpad_dir: The dir to store the output symbol file in
board: If |breakpad_dir| is not specified, use |board| to find it
strip_cfi: Do not generate CFI data
num_errors: An object to update with the error count (needs a .value member)
Returns:
The number of errors that were encountered.
"""
if breakpad_dir is None:
breakpad_dir = FindBreakpadDir(board)
if num_errors is None:
num_errors = ctypes.c_int()
cmd_base = ['dump_syms']
if strip_cfi:
cmd_base += ['-c']
# Some files will not be readable by non-root (e.g. set*id /bin/su).
needs_sudo = not os.access(elf_file, os.R_OK)
def _DumpIt(cmd_args):
if needs_sudo:
run_command = cros_build_lib.SudoRunCommand
else:
run_command = cros_build_lib.RunCommand
return run_command(cmd_base + cmd_args,
redirect_stderr=True,
log_stdout_to_file=temp.name,
error_code_ok=True,
debug_level=logging.DEBUG)
def _CrashCheck(ret, msg):
if ret < 0:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Warning('dump_syms crashed with %s; %s',
osutils.StrSignal(-ret), msg)
osutils.SafeMakedirs(breakpad_dir)
with tempfile.NamedTemporaryFile(dir=breakpad_dir, bufsize=0) as temp:
if debug_file:
# Try to dump the symbols using the debug file like normal.
cmd_args = [elf_file, os.path.dirname(debug_file)]
result = _DumpIt(cmd_args)
if result.returncode:
# Sometimes dump_syms can crash because there's too much info.
# Try dumping and stripping the extended stuff out. At least
# this way we'll get the extended symbols. http://crbug.com/266064
_CrashCheck(result.returncode, 'retrying w/out CFI')
cmd_args = ['-c', '-r'] + cmd_args
result = _DumpIt(cmd_args)
_CrashCheck(result.returncode, 'retrying w/out debug')
basic_dump = result.returncode
else:
basic_dump = True
if basic_dump:
# If that didn't work (no debug, or dump_syms still failed), try
# dumping just the file itself directly.
result = _DumpIt([elf_file])
if result.returncode:
# A lot of files (like kernel files) contain no debug information,
# do not consider such occurrences as errors.
cros_build_lib.PrintBuildbotStepWarnings()
_CrashCheck(result.returncode, 'giving up entirely')
if 'file contains no debugging information' in result.error:
cros_build_lib.Warning('no symbols found for %s', elf_file)
else:
num_errors.value += 1
cros_build_lib.Error('dumping symbols for %s failed:\n%s',
elf_file, result.error)
return num_errors.value
# Move the dumped symbol file to the right place:
# /build/$BOARD/usr/lib/debug/breakpad/<module-name>/<id>/<module-name>.sym
header = ReadSymsHeader(temp)
cros_build_lib.Info('Dumped %s as %s : %s', elf_file, header.name,
header.id)
sym_file = os.path.join(breakpad_dir, header.name, header.id,
header.name + '.sym')
osutils.SafeMakedirs(os.path.dirname(sym_file))
os.rename(temp.name, sym_file)
os.chmod(sym_file, 0644)
temp.delete = False
return num_errors.value
