def main(argv): parser = commandline.ArgumentParser(description=__doc__) parser.add_argument('--board', default=None, help='board to generate symbols for') parser.add_argument('--breakpad_root', type='path', default=None, help='root directory for breakpad symbols') parser.add_argument('--generate-count', type=int, default=None, help='only generate # number of symbols') parser.add_argument('--strip_cfi', action='store_true', default=False, help='do not generate CFI data (pass -c to dump_syms)') opts = parser.parse_args(argv) if opts.board is None: cros_build_lib.Die('--board is required') ret = GenerateBreakpadSymbols(opts.board, breakpad_dir=opts.breakpad_root, strip_cfi=opts.strip_cfi, generate_count=opts.generate_count) if ret: cros_build_lib.Error('encountered %i problem(s)', ret) # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently # return 0 in case we are a multiple of the mask. ret = 1 return ret
def SyncSources(self): repo = repository.RepoRepository(self.manifest_dir, self.repo_root, referenced_repo=self.reference, manifest=MANIFEST_FILE, branch='master') # Trigger the network sync repo.Sync(jobs=multiprocessing.cpu_count() + 1, network_only=True) projects = [self.CHROMIUM_ROOT] if self.internal: projects.append(self.CHROME_ROOT) for project in projects: path = os.path.join(self.repo_root, project) if os.path.exists(path): try: git.CleanAndCheckoutUpstream(path, refresh_upstream=False) continue except cros_build_lib.RunCommandError: cros_build_lib.Error("Failed cleaning %r; wiping.", path) cros_build_lib.SudoRunCommand(['rm', '-rf', path], print_cmd=False) cros_build_lib.RunCommand(['repo', 'sync', '-ld', project], cwd=self.repo_root)
def _HandleStageException(self, exception): """Called when PerformStage throws an exception. Can be overriden. Should return result, description. Description should be None if result is not an exception. """ # Tell the user about the exception, and record it description = self._StringifyException(exception) cros_build_lib.PrintBuildbotStepFailure() cros_build_lib.Error(description) return exception, description
def GrabRemotePackageIndex(binhost_url): """Grab the latest binary package database from the specified URL. Args: binhost_url: Base URL of remote packages (PORTAGE_BINHOST). Returns: A PackageIndex object, if the Packages file can be retrieved. If the packages file cannot be retrieved, then None is returned. """ url = '%s/Packages' % binhost_url.rstrip('/') pkgindex = PackageIndex() if binhost_url.startswith('http'): try: f = _RetryUrlOpen(url) except urllib2.HTTPError as e: if e.code in HTTP_FORBIDDEN_CODES: cros_build_lib.PrintBuildbotStepWarnings() cros_build_lib.Error('Cannot GET %s: %s' % (url, str(e))) return None # Not found errors are normal if old prebuilts were cleaned out. if e.code in HTTP_NOT_FOUND_CODES: return None raise elif binhost_url.startswith('gs://'): cmd = [gs.GSUTIL_BIN, 'cat', url] try: output = cros_build_lib.RunCommand(cmd, redirect_stdout=True, print_cmd=False).output except cros_build_lib.RunCommandError as e: cros_build_lib.PrintBuildbotStepWarnings() cros_build_lib.Error('Cannot GET %s: %s' % (url, str(e))) return None f = cStringIO.StringIO(output) else: return None pkgindex.Read(f) pkgindex.header.setdefault('URI', binhost_url) f.close() return pkgindex
def DownloadCrx(ext, extension, outputdir): """Download .crx file from WebStore and update entry.""" cros_build_lib.Info('Extension "%s"(%s)...', extension['name'], ext) update_url = '%s?x=id%%3D%s%%26uc' % (extension['external_update_url'], ext) response = urllib.urlopen(update_url) if response.getcode() != 200: cros_build_lib.Error('Cannot get update response, URL: %s, error: %d', update_url, response.getcode()) return False dom = xml.dom.minidom.parse(response) status = dom.getElementsByTagName('app')[0].getAttribute('status') if status != 'ok': cros_build_lib.Error('Cannot fetch extension, status: %s', status) return False node = dom.getElementsByTagName('updatecheck')[0] url = node.getAttribute('codebase') version = node.getAttribute('version') filename = '%s-%s.crx' % (ext, version) response = urllib.urlopen(url) if response.getcode() != 200: cros_build_lib.Error('Cannot download extension, URL: %s, error: %d', url, response.getcode()) return False osutils.WriteFile(os.path.join(outputdir, 'extensions', filename), response.read()) # Has to delete because only one of 'external_crx' or # 'external_update_url' should present for the extension. del extension['external_update_url'] extension['external_crx'] = os.path.join(EXTENSIONS_CACHE_PREFIX, filename) extension['external_version'] = version cros_build_lib.Info('Downloaded, current version %s', version) return True
def SubmitChange(self, change, dryrun=False): """Submits patch using Gerrit Review.""" cmd = self.GetGerritReviewCommand([ '--submit', '%s,%s' % (change.gerrit_number, change.patch_number) ]) if dryrun: logging.info('Would have run: %s', ' '.join(map(repr, cmd))) return try: cros_build_lib.RunCommand(cmd) except cros_build_lib.RunCommandError: cros_build_lib.Error('Command failed', exc_info=True)
def TestNewManifest(self): """Runs a 'repo sync' off of new manifest to verify things aren't broken.""" # Do as cheap a sync as possible; network only is good enough, # allow shallow cloning if we don't have a reference, and sync # strictly the target branch. repo = repository.RepoRepository( self.manifest_dir, self.repo_root, branch=TEST_BRANCH, referenced_repo=self.reference, manifest=MANIFEST_FILE) try: repo.Sync(jobs=multiprocessing.cpu_count() + 1, network_only=True) except Exception: cros_build_lib.Error('Failed to sync with new manifest!') raise
def _Submit(self, testjob, dryrun): """Internal submission function. See Submit() for arg description.""" # TODO(rcui): convert to shallow clone when that's available. current_time = str(int(time.time())) ref_base = os.path.join('refs/tryjobs', self.user, current_time) for patch in self.local_patches: # Isolate the name; if it's a tag or a remote, let through. # Else if it's a branch, get the full branch name minus refs/heads. local_branch = git.StripRefsHeads(patch.ref, False) ref_final = os.path.join(ref_base, local_branch, patch.sha1) self.manifest.AssertProjectIsPushable(patch.project) data = self.manifest.projects[patch.project] print 'Uploading patch %s' % patch patch.Upload(data['push_url'], ref_final, dryrun=dryrun) # TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937. tag = constants.EXTERNAL_PATCH_TAG if data['remote'] == constants.INTERNAL_REMOTE: tag = constants.INTERNAL_PATCH_TAG self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s' % (patch.project, local_branch, ref_final, patch.tracking_branch, tag)) self._VerifyForBuildbot() repository.CloneGitRepo(self.tryjob_repo, self.ssh_url) version_path = os.path.join(self.tryjob_repo, self.TRYJOB_FORMAT_FILE) with open(version_path, 'r') as f: try: val = int(f.read().strip()) except ValueError: raise ChromiteUpgradeNeeded() if val > self.TRYJOB_FORMAT_VERSION: raise ChromiteUpgradeNeeded(val) push_branch = manifest_version.PUSH_BRANCH remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None git.CreatePushBranch(push_branch, self.tryjob_repo, sync=False, remote_push_branch=remote_branch) file_name = '%s.%s' % (self.user, current_time) user_dir = os.path.join(self.tryjob_repo, self.user) if not os.path.isdir(user_dir): os.mkdir(user_dir) fullpath = os.path.join(user_dir, file_name) with open(fullpath, 'w+') as job_desc_file: json.dump(self.values, job_desc_file) cros_build_lib.RunCommand(['git', 'add', fullpath], cwd=self.tryjob_repo) extra_env = { # The committer field makes sure the creds match what the remote # gerrit instance expects while the author field allows lookup # on the console to work. http://crosbug.com/27939 'GIT_COMMITTER_EMAIL' : self.user_email, 'GIT_AUTHOR_EMAIL' : self.user_email, } cros_build_lib.RunCommand(['git', 'commit', '-m', self.description], cwd=self.tryjob_repo, extra_env=extra_env) try: git.PushWithRetry( push_branch, self.tryjob_repo, retries=3, dryrun=dryrun) except cros_build_lib.RunCommandError: cros_build_lib.Error( 'Failed to submit tryjob. This could be due to too many ' 'submission requests by users. Please try again.') raise
def UploadSymbol(sym_file, upload_url, file_limit=DEFAULT_FILE_LIMIT, sleep=0, num_errors=None): """Upload |sym_file| to |upload_url| Args: sym_file: The full path to the breakpad symbol to upload upload_url: The crash server to upload things to file_limit: The max file size of a symbol file before we try to strip it sleep: Number of seconds to sleep before running num_errors: An object to update with the error count (needs a .value member) Returns: The number of errors that were encountered. """ if num_errors is None: num_errors = ctypes.c_int() elif num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY: # Abandon ship! It's on fire! NOoooooooooooOOOoooooo. return 0 upload_file = sym_file if sleep: # Keeps us from DoS-ing the symbol server. time.sleep(sleep) cros_build_lib.Debug('uploading %s' % sym_file) # Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use. with tempfile.NamedTemporaryFile(prefix='upload_symbols', bufsize=0) as temp_sym_file: if file_limit: # If the symbols size is too big, strip out the call frame info. The CFI # is unnecessary for 32bit x86 targets where the frame pointer is used (as # all of ours have) and it accounts for over half the size of the symbols # uploaded. file_size = os.path.getsize(sym_file) if file_size > file_limit: cros_build_lib.Warning( 'stripping CFI from %s due to size %s > %s', sym_file, file_size, file_limit) temp_sym_file.writelines([ x for x in open(sym_file, 'rb').readlines() if not x.startswith('STACK CFI') ]) upload_file = temp_sym_file.name # Hopefully the crash server will let it through. But it probably won't. # Not sure what the best answer is in this case. file_size = os.path.getsize(upload_file) if file_size > CRASH_SERVER_FILE_LIMIT: cros_build_lib.PrintBuildbotStepWarnings() cros_build_lib.Error( 'upload file %s is awfully large, risking rejection ' 'by symbol server (%s > %s)', sym_file, file_size, CRASH_SERVER_FILE_LIMIT) num_errors.value += 1 # Upload the symbol file. try: cros_build_lib.RetryCommand(SymUpload, MAX_RETRIES, upload_file, upload_url, sleep=INITIAL_RETRY_DELAY) cros_build_lib.Info('successfully uploaded %10i bytes: %s', file_size, os.path.basename(sym_file)) except cros_build_lib.RunCommandError as e: cros_build_lib.Warning( 'could not upload: %s:\n{stdout} %s\n{stderr} %s', os.path.basename(sym_file), e.result.output, e.result.error) num_errors.value += 1 return num_errors.value
def main(argv): parser = commandline.ArgumentParser(description=__doc__) parser.add_argument('sym_files', type='path', nargs='*', default=None) parser.add_argument('--board', default=None, help='board to build packages for') parser.add_argument('--breakpad_root', type='path', default=None, help='root directory for breakpad symbols') parser.add_argument('--official_build', action='store_true', default=False, help='point to official symbol server') parser.add_argument('--regenerate', action='store_true', default=False, help='regenerate all symbols') parser.add_argument('--upload-count', type=int, default=None, help='only upload # number of symbols') parser.add_argument('--strip_cfi', type=int, default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024), help='strip CFI data for files above this size') parser.add_argument('--testing', action='store_true', default=False, help='run in testing mode') parser.add_argument('--yes', action='store_true', default=False, help='answer yes to all prompts') opts = parser.parse_args(argv) if opts.sym_files: if opts.regenerate: cros_build_lib.Die( '--regenerate may not be used with specific files') else: if opts.board is None: cros_build_lib.Die('--board is required') if opts.breakpad_root and opts.regenerate: cros_build_lib.Die('--regenerate may not be used with --breakpad_root') if opts.testing: # TODO(build): Kill off --testing mode once unittests are up-to-snuff. cros_build_lib.Info('running in testing mode') # pylint: disable=W0601,W0603 global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0 SymUpload = TestingSymUpload if not opts.yes: query = textwrap.wrap( textwrap.dedent(""" Uploading symbols for an entire Chromium OS build is really only necessary for release builds and in a few cases for developers to debug problems. It will take considerable time to run. For developer debugging purposes, consider instead passing specific files to upload. """), 80) cros_build_lib.Warning('\n%s', '\n'.join(query)) if not cros_build_lib.BooleanPrompt( prompt='Are you sure you want to upload all build symbols', default=False): cros_build_lib.Die('better safe than sorry') ret = 0 if opts.regenerate: ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols( opts.board, breakpad_dir=opts.breakpad_root) ret += UploadSymbols(opts.board, official=opts.official_build, breakpad_dir=opts.breakpad_root, file_limit=opts.strip_cfi, sleep=DEFAULT_SLEEP_DELAY, upload_count=opts.upload_count, sym_files=opts.sym_files) if ret: cros_build_lib.Error('encountered %i problem(s)', ret) # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently # return 0 in case we are a multiple of the mask. ret = 1 return ret
def GenerateBreakpadSymbol(elf_file, debug_file=None, breakpad_dir=None, board=None, strip_cfi=False, num_errors=None): """Generate the symbols for |elf_file| using |debug_file| Args: elf_file: The file to dump symbols for debug_file: Split debug file to use for symbol information breakpad_dir: The dir to store the output symbol file in board: If |breakpad_dir| is not specified, use |board| to find it strip_cfi: Do not generate CFI data num_errors: An object to update with the error count (needs a .value member) Returns: The number of errors that were encountered. """ if breakpad_dir is None: breakpad_dir = FindBreakpadDir(board) if num_errors is None: num_errors = ctypes.c_int() cmd_base = ['dump_syms'] if strip_cfi: cmd_base += ['-c'] # Some files will not be readable by non-root (e.g. set*id /bin/su). needs_sudo = not os.access(elf_file, os.R_OK) def _DumpIt(cmd_args): if needs_sudo: run_command = cros_build_lib.SudoRunCommand else: run_command = cros_build_lib.RunCommand return run_command(cmd_base + cmd_args, redirect_stderr=True, log_stdout_to_file=temp.name, error_code_ok=True, debug_level=logging.DEBUG) def _CrashCheck(ret, msg): if ret < 0: cros_build_lib.PrintBuildbotStepWarnings() cros_build_lib.Warning('dump_syms crashed with %s; %s', osutils.StrSignal(-ret), msg) osutils.SafeMakedirs(breakpad_dir) with tempfile.NamedTemporaryFile(dir=breakpad_dir, bufsize=0) as temp: if debug_file: # Try to dump the symbols using the debug file like normal. cmd_args = [elf_file, os.path.dirname(debug_file)] result = _DumpIt(cmd_args) if result.returncode: # Sometimes dump_syms can crash because there's too much info. # Try dumping and stripping the extended stuff out. At least # this way we'll get the extended symbols. http://crbug.com/266064 _CrashCheck(result.returncode, 'retrying w/out CFI') cmd_args = ['-c', '-r'] + cmd_args result = _DumpIt(cmd_args) _CrashCheck(result.returncode, 'retrying w/out debug') basic_dump = result.returncode else: basic_dump = True if basic_dump: # If that didn't work (no debug, or dump_syms still failed), try # dumping just the file itself directly. result = _DumpIt([elf_file]) if result.returncode: # A lot of files (like kernel files) contain no debug information, # do not consider such occurrences as errors. cros_build_lib.PrintBuildbotStepWarnings() _CrashCheck(result.returncode, 'giving up entirely') if 'file contains no debugging information' in result.error: cros_build_lib.Warning('no symbols found for %s', elf_file) else: num_errors.value += 1 cros_build_lib.Error('dumping symbols for %s failed:\n%s', elf_file, result.error) return num_errors.value # Move the dumped symbol file to the right place: # /build/$BOARD/usr/lib/debug/breakpad/<module-name>/<id>/<module-name>.sym header = ReadSymsHeader(temp) cros_build_lib.Info('Dumped %s as %s : %s', elf_file, header.name, header.id) sym_file = os.path.join(breakpad_dir, header.name, header.id, header.name + '.sym') osutils.SafeMakedirs(os.path.dirname(sym_file)) os.rename(temp.name, sym_file) os.chmod(sym_file, 0644) temp.delete = False return num_errors.value