def RemoveChromeTemporaryFiles(): """A large hammer to nuke what could be leaked files from unittests or files left from a unittest that crashed, was killed, etc.""" # NOTE: print out what is cleaned up so the bots don't timeout if # there is a lot to cleanup and also se we see the leaks in the # build logs. # At some point a leading dot got added, support with and without it. kLogRegex = '^\.?(com\.google\.Chrome|org\.chromium)\.' if chromium_utils.IsWindows(): kLogRegex = r'^(base_dir|scoped_dir|nps|chrome_test|SafeBrowseringTest)' LogAndRemoveFiles(tempfile.gettempdir(), kLogRegex) # Dump and temporary files. LogAndRemoveFiles(tempfile.gettempdir(), r'^.+\.(dmp|tmp)$') LogAndRemoveFiles(tempfile.gettempdir(), r'^_CL_.*$') RemoveChromeDesktopFiles() RemoveJumpListFiles() elif chromium_utils.IsLinux(): kLogRegexHeapcheck = '\.(sym|heap)$' LogAndRemoveFiles(tempfile.gettempdir(), kLogRegex) LogAndRemoveFiles(tempfile.gettempdir(), kLogRegexHeapcheck) LogAndRemoveFiles('/dev/shm', kLogRegex) elif chromium_utils.IsMac(): nstempdir_path = '/usr/local/libexec/nstempdir' if os.path.exists(nstempdir_path): ns_temp_dir = chromium_utils.GetCommandOutput([nstempdir_path]).strip() if ns_temp_dir: LogAndRemoveFiles(ns_temp_dir, kLogRegex) for i in ('Chromium', 'Google Chrome'): # Remove dumps. crash_path = '%s/Library/Application Support/%s/Crash Reports' % ( os.environ['HOME'], i) LogAndRemoveFiles(crash_path, r'^.+\.dmp$') else: raise NotImplementedError( 'Platform "%s" is not currently supported.' % sys.platform)
def SubversionCat(wc_dir): """Output the content of specified files or URLs in SVN. """ try: return chromium_utils.GetCommandOutput([SubversionExe(), 'cat', wc_dir]) except chromium_utils.ExternalError: return None
def SubversionLastChangedRevision(wc_dir): """Finds the svn revision where this file/dir was last edited by running 'svn info', and returns it as an integer. """ svn_regexp = re.compile(r'.*Last Changed Rev: (\d+).*', re.DOTALL) try: svn_info = chromium_utils.GetCommandOutput( [SubversionExe(), 'info', wc_dir]) return_value = re.sub(svn_regexp, r'\1', svn_info) if (return_value.isalnum()): return int(return_value) else: return 0 except chromium_utils.ExternalError: return 0
def SubversionRevision(wc_dir): """Finds the last svn revision of a working copy by running 'svn info', and returns it as an integer. """ svn_regexp = re.compile(r'.*Revision: (\d+).*', re.DOTALL) try: svn_info = chromium_utils.GetCommandOutput( [SubversionExe(), 'info', wc_dir]) return_value = re.sub(svn_regexp, r'\1', svn_info) if (return_value.isalnum()): return int(return_value) else: return 0 except chromium_utils.ExternalError: return 0
def main(): option_parser = optparse.OptionParser() option_parser.add_option('--pepper-channel', default='stable', help='Pepper channel (stable|beta|canary)') options, _ = option_parser.parse_args() work_dir = os.path.abspath('.') print 'Locating NaCl SDK update script at %s' % NACL_SDK_UPDATE_URL file_name = NACL_SDK_UPDATE_URL.split('/')[-1] response = requests.get(NACL_SDK_UPDATE_URL, verify=True, stream=True) file_hash = None if os.path.exists(file_name): file_hash = GetFileHash(file_name) print 'Downloading: %s' % file_name Retrieve(response, file_name) # Only extract if file changed. Extraction overwrites the sdk tools and the # state about which pepper revisions are up to date. if file_hash != GetFileHash(file_name): print 'Unzipping %s into %s' % (file_name, work_dir) chromium_utils.ExtractZip(file_name, work_dir, verbose=True) else: print 'Existing %s is up to date.' % file_name print 'Listing available pepper bundles:' output = chromium_utils.GetCommandOutput([NACL_TOOL, 'list']) print output pepper_rev = GetRevisionName(output, options.pepper_channel) print 'Updating pepper bundle %s' % pepper_rev cmd = [NACL_TOOL, 'update', pepper_rev, '--force'] result = chromium_utils.RunCommand(cmd) if os.path.exists(CURRENT_PEPPER_BUNDLE): print 'Removing current pepper bundle %s' % CURRENT_PEPPER_BUNDLE shutil.rmtree(CURRENT_PEPPER_BUNDLE) pepper_rev_dir = os.path.join('nacl_sdk', pepper_rev) print 'Copying pepper bundle %s to current' % pepper_rev shutil.copytree(pepper_rev_dir, CURRENT_PEPPER_BUNDLE, symlinks=True) return result
def _GetXMLChangeLogByModule(module_name, module_src_dir, last_revision, current_revision): """Get the change log information for specified module and start/end revision. """ if (last_revision and current_revision > last_revision): command = [ slave_utils.SubversionExe(), 'log', module_src_dir, '--xml', '-r', '%d:%d' % (last_revision + 1, current_revision) ] changelog = chromium_utils.GetCommandOutput(command) changelog_description = '%s changeLogs from ]%d to %d]' % ( module_name, last_revision, current_revision) else: changelog = '' changelog_description = 'No new ChangeLogs on %s' % (module_name) return (changelog, changelog_description)
def _GetXMLChangeLogByModule(module_name, module_src_dir, last_revision, current_revision): """Get the change log information for specified module and start/end revision. """ changelog = '' changelog_description = 'No new ChangeLogs on %s' % (module_name) try: last_revision = int(last_revision) current_revision = int(current_revision) if (last_revision and current_revision > last_revision): command = [ slave_utils.SubversionExe(), 'log', module_src_dir, '--xml', '-r', '%d:%d' % (last_revision + 1, current_revision) ] changelog = chromium_utils.GetCommandOutput(command) changelog_description = '%s changeLogs from ]%d to %d]' % ( module_name, last_revision, current_revision) except ValueError, e: print >> os.stderr, e
def GetStackTrace(debugger_dir, symbol_path, dump_file): """Gets and prints the stack trace from a crash dump file. Args: debugger_dir: the directory where the debugger is installed. symbol_path: symbol path for debugger. dump_file: the path to dump file. Returns: A string representing the stack trace. """ # Run debugger to analyze crash dump. cmd = '%s\\cdb.exe -y "%s" -c ".ecxr;k30;q" -z "%s"' % ( debugger_dir, symbol_path, dump_file) try: output = chromium_utils.GetCommandOutput(cmd) except chromium_utils.ExternalError: return 'Cannot get stack trace.' # Retrieve stack trace from debugger output. stack_start = output.find('ChildEBP') stack_end = output.find('quit:') return output[stack_start:stack_end]
def main_xcode(options, args): """Interprets options, clobbers object files, and calls xcodebuild. """ # If the project isn't in args, add all.xcodeproj to simplify configuration. command = ['xcodebuild', '-configuration', options.target] # TODO(mmoss) Support the old 'args' usage until we're confident the master is # switched to passing '--solution' everywhere. if not '-project' in args: # TODO(mmoss) Temporary hack to ignore the Windows --solution flag that is # passed to all builders. This can be taken out once the master scripts are # updated to only pass platform-appropriate --solution values. if (not options.solution or os.path.splitext(options.solution)[1] != '.xcodeproj'): options.solution = 'all.xcodeproj' command.extend(['-project', options.solution]) if options.xcode_target: command.extend(['-target', options.xcode_target]) # Note: this clobbers all targets, not just Debug or Release. if options.clobber: build_output_dir = os.path.join(os.path.dirname(options.build_dir), 'xcodebuild') print('Removing %s' % build_output_dir) chromium_utils.RemoveDirectory(build_output_dir) env = EchoDict(os.environ) common_xcode_settings(command, options, env, options.compiler) # Add on any remaining args command.extend(args) # Set up the filter before changing directories so the raw build log can # be recorded. # Support a local file blocking filters (for debugging). Also check the # Xcode version to make sure it is 3.2, as that is what the filter is coded # to. xcodebuild_filter = None no_filter_path = os.path.join(os.getcwd(), 'no_xcodebuild_filter') xcode_info = chromium_utils.GetCommandOutput(['xcodebuild', '-version']) if os.path.exists(no_filter_path): print 'NOTE: "%s" exists, output is unfiltered' % no_filter_path elif not xcode_info.startswith('Xcode 3.2.'): print 'NOTE: Not using Xcode 3.2, output is unfiltered' else: full_log_path = os.path.join(os.getcwd(), 'full_xcodebuild_log.txt') full_log = open(full_log_path, 'w') now = datetime.datetime.now() full_log.write('Build started ' + now.isoformat() + '\n\n\n') print 'NOTE: xcodebuild output filtered, full log at: "%s"' % full_log_path xcodebuild_filter = XcodebuildFilter(full_log) os.chdir(options.build_dir) # If using the Goma compiler, first call goma_ctl with ensure_start # (or restart in clobber mode) to ensure the proxy is available. goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')] if options.compiler in ('goma', 'goma-clang', 'gomaclang'): goma_key = os.path.join(options.goma_dir, 'goma.key') env['GOMA_COMPILER_PROXY_DAEMON_MODE'] = 'true' if os.path.exists(goma_key): env['GOMA_API_KEY_FILE'] = goma_key if options.clobber: chromium_utils.RunCommand(goma_ctl_cmd + ['restart'], env=env) else: chromium_utils.RunCommand(goma_ctl_cmd + ['ensure_start'], env=env) # Run the build. env.print_overrides() result = chromium_utils.RunCommand(command, env=env, filter_obj=xcodebuild_filter) if options.compiler in ('goma', 'goma-clang', 'gomaclang'): # Always stop the proxy for now to allow in-place update. chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env) return result
def main_xcode(options, args): """Interprets options, clobbers object files, and calls xcodebuild. """ env = EchoDict(os.environ) goma_ready = goma_setup(options, env) if not goma_ready: assert options.compiler not in ('goma', 'goma-clang') assert options.goma_dir is None # Print some basic information about xcodebuild to ease debugging. chromium_utils.RunCommand(['xcodebuild', '-sdk', '-version'], env=env) # If the project isn't in args, add all.xcodeproj to simplify configuration. command = ['xcodebuild', '-configuration', options.target] # TODO(mmoss) Support the old 'args' usage until we're confident the master is # switched to passing '--solution' everywhere. if not '-project' in args: # TODO(mmoss) Temporary hack to ignore the Windows --solution flag that is # passed to all builders. This can be taken out once the master scripts are # updated to only pass platform-appropriate --solution values. if (not options.solution or os.path.splitext(options.solution)[1] != '.xcodeproj'): options.solution = '../build/all.xcodeproj' command.extend(['-project', options.solution]) if options.xcode_target: command.extend(['-target', options.xcode_target]) # Note: this clobbers all targets, not just Debug or Release. if options.clobber: clobber_dir = os.path.dirname(options.target_output_dir) print 'Removing %s' % clobber_dir # Deleting output_dir would also delete all the .ninja files. iOS builds # generates ninja configuration inside the xcodebuild directory to be able # to run sub builds. crbug.com/138950 is tracking this issue. # Moreover clobbering should run before runhooks (which creates # .ninja files). For now, only delete all non-.ninja files. # TODO(thakis): Make "clobber" a step that runs before "runhooks". Once the # master has been restarted, remove all clobber handling from compile.py. build_directory.RmtreeExceptNinjaOrGomaFiles(clobber_dir) common_xcode_settings(command, options, env, options.compiler) maybe_set_official_build_envvars(options, env) # Add on any remaining args command.extend(args) # Set up the filter before changing directories so the raw build log can # be recorded. # Support a local file blocking filters (for debugging). Also check the # Xcode version to make sure it is 3.2, as that is what the filter is coded # to. xcodebuild_filter = None no_filter_path = os.path.join(os.getcwd(), 'no_xcodebuild_filter') xcode_info = chromium_utils.GetCommandOutput(['xcodebuild', '-version']) if os.path.exists(no_filter_path): print 'NOTE: "%s" exists, output is unfiltered' % no_filter_path elif not xcode_info.startswith('Xcode 3.2.'): # Note: The filter sometimes hides real errors on 4.x+, see crbug.com/260989 print 'NOTE: Not using Xcode 3.2, output is unfiltered' else: full_log_path = os.path.join(os.getcwd(), 'full_xcodebuild_log.txt') full_log = open(full_log_path, 'w') now = datetime.datetime.now() full_log.write('Build started ' + now.isoformat() + '\n\n\n') print 'NOTE: xcodebuild output filtered, full log at: "%s"' % full_log_path xcodebuild_filter = XcodebuildFilter(full_log) try: os.makedirs(options.build_dir) except OSError, e: if e.errno != errno.EEXIST: raise