def RemoveChromeTemporaryFiles():
"""A large hammer to nuke what could be leaked files from unittests or
files left from a unittest that crashed, was killed, etc."""
# NOTE: print out what is cleaned up so the bots don't timeout if
# there is a lot to cleanup and also se we see the leaks in the
# build logs.
# At some point a leading dot got added, support with and without it.
kLogRegex = '^\.?(com\.google\.Chrome|org\.chromium)\.'
if chromium_utils.IsWindows():
kLogRegex = r'^(base_dir|scoped_dir|nps|chrome_test|SafeBrowseringTest)'
LogAndRemoveFiles(tempfile.gettempdir(), kLogRegex)
# Dump and temporary files.
LogAndRemoveFiles(tempfile.gettempdir(), r'^.+\.(dmp|tmp)$')
LogAndRemoveFiles(tempfile.gettempdir(), r'^_CL_.*$')
RemoveChromeDesktopFiles()
RemoveJumpListFiles()
elif chromium_utils.IsLinux():
kLogRegexHeapcheck = '\.(sym|heap)$'
LogAndRemoveFiles(tempfile.gettempdir(), kLogRegex)
LogAndRemoveFiles(tempfile.gettempdir(), kLogRegexHeapcheck)
LogAndRemoveFiles('/dev/shm', kLogRegex)
elif chromium_utils.IsMac():
nstempdir_path = '/usr/local/libexec/nstempdir'
if os.path.exists(nstempdir_path):
ns_temp_dir = chromium_utils.GetCommandOutput([nstempdir_path]).strip()
if ns_temp_dir:
LogAndRemoveFiles(ns_temp_dir, kLogRegex)
for i in ('Chromium', 'Google Chrome'):
# Remove dumps.
crash_path = '%s/Library/Application Support/%s/Crash Reports' % (
os.environ['HOME'], i)
LogAndRemoveFiles(crash_path, r'^.+\.dmp$')
else:
raise NotImplementedError(
'Platform "%s" is not currently supported.' % sys.platform)
def SubversionCat(wc_dir):
"""Output the content of specified files or URLs in SVN.
"""
try:
return chromium_utils.GetCommandOutput([SubversionExe(), 'cat',
wc_dir])
except chromium_utils.ExternalError:
return None
def SubversionLastChangedRevision(wc_dir):
"""Finds the svn revision where this file/dir was last edited by running
'svn info', and returns it as an integer.
"""
svn_regexp = re.compile(r'.*Last Changed Rev: (\d+).*', re.DOTALL)
try:
svn_info = chromium_utils.GetCommandOutput(
[SubversionExe(), 'info', wc_dir])
return_value = re.sub(svn_regexp, r'\1', svn_info)
if (return_value.isalnum()):
return int(return_value)
else:
return 0
except chromium_utils.ExternalError:
return 0
def SubversionRevision(wc_dir):
"""Finds the last svn revision of a working copy by running 'svn info',
and returns it as an integer.
"""
svn_regexp = re.compile(r'.*Revision: (\d+).*', re.DOTALL)
try:
svn_info = chromium_utils.GetCommandOutput(
[SubversionExe(), 'info', wc_dir])
return_value = re.sub(svn_regexp, r'\1', svn_info)
if (return_value.isalnum()):
return int(return_value)
else:
return 0
except chromium_utils.ExternalError:
return 0
def main():
option_parser = optparse.OptionParser()
option_parser.add_option('--pepper-channel',
default='stable',
help='Pepper channel (stable|beta|canary)')
options, _ = option_parser.parse_args()
work_dir = os.path.abspath('.')
print 'Locating NaCl SDK update script at %s' % NACL_SDK_UPDATE_URL
file_name = NACL_SDK_UPDATE_URL.split('/')[-1]
response = requests.get(NACL_SDK_UPDATE_URL, verify=True, stream=True)
file_hash = None
if os.path.exists(file_name):
file_hash = GetFileHash(file_name)
print 'Downloading: %s' % file_name
Retrieve(response, file_name)
# Only extract if file changed. Extraction overwrites the sdk tools and the
# state about which pepper revisions are up to date.
if file_hash != GetFileHash(file_name):
print 'Unzipping %s into %s' % (file_name, work_dir)
chromium_utils.ExtractZip(file_name, work_dir, verbose=True)
else:
print 'Existing %s is up to date.' % file_name
print 'Listing available pepper bundles:'
output = chromium_utils.GetCommandOutput([NACL_TOOL, 'list'])
print output
pepper_rev = GetRevisionName(output, options.pepper_channel)
print 'Updating pepper bundle %s' % pepper_rev
cmd = [NACL_TOOL, 'update', pepper_rev, '--force']
result = chromium_utils.RunCommand(cmd)
if os.path.exists(CURRENT_PEPPER_BUNDLE):
print 'Removing current pepper bundle %s' % CURRENT_PEPPER_BUNDLE
shutil.rmtree(CURRENT_PEPPER_BUNDLE)
pepper_rev_dir = os.path.join('nacl_sdk', pepper_rev)
print 'Copying pepper bundle %s to current' % pepper_rev
shutil.copytree(pepper_rev_dir, CURRENT_PEPPER_BUNDLE, symlinks=True)
return result
def _GetXMLChangeLogByModule(module_name, module_src_dir, last_revision,
current_revision):
"""Get the change log information for specified module and start/end
revision.
"""
if (last_revision and current_revision > last_revision):
command = [
slave_utils.SubversionExe(), 'log', module_src_dir, '--xml', '-r',
'%d:%d' % (last_revision + 1, current_revision)
]
changelog = chromium_utils.GetCommandOutput(command)
changelog_description = '%s changeLogs from ]%d to %d]' % (
module_name, last_revision, current_revision)
else:
changelog = ''
changelog_description = 'No new ChangeLogs on %s' % (module_name)
return (changelog, changelog_description)
def _GetXMLChangeLogByModule(module_name, module_src_dir, last_revision,
current_revision):
"""Get the change log information for specified module and start/end
revision.
"""
changelog = ''
changelog_description = 'No new ChangeLogs on %s' % (module_name)
try:
last_revision = int(last_revision)
current_revision = int(current_revision)
if (last_revision and current_revision > last_revision):
command = [
slave_utils.SubversionExe(), 'log', module_src_dir, '--xml',
'-r',
'%d:%d' % (last_revision + 1, current_revision)
]
changelog = chromium_utils.GetCommandOutput(command)
changelog_description = '%s changeLogs from ]%d to %d]' % (
module_name, last_revision, current_revision)
except ValueError, e:
print >> os.stderr, e
def GetStackTrace(debugger_dir, symbol_path, dump_file):
"""Gets and prints the stack trace from a crash dump file.
Args:
debugger_dir: the directory where the debugger is installed.
symbol_path: symbol path for debugger.
dump_file: the path to dump file.
Returns:
A string representing the stack trace.
"""
# Run debugger to analyze crash dump.
cmd = '%s\\cdb.exe -y "%s" -c ".ecxr;k30;q" -z "%s"' % (
debugger_dir, symbol_path, dump_file)
try:
output = chromium_utils.GetCommandOutput(cmd)
except chromium_utils.ExternalError:
return 'Cannot get stack trace.'
# Retrieve stack trace from debugger output.
stack_start = output.find('ChildEBP')
stack_end = output.find('quit:')
return output[stack_start:stack_end]
def main_xcode(options, args):
"""Interprets options, clobbers object files, and calls xcodebuild.
"""
# If the project isn't in args, add all.xcodeproj to simplify configuration.
command = ['xcodebuild', '-configuration', options.target]
# TODO(mmoss) Support the old 'args' usage until we're confident the master is
# switched to passing '--solution' everywhere.
if not '-project' in args:
# TODO(mmoss) Temporary hack to ignore the Windows --solution flag that is
# passed to all builders. This can be taken out once the master scripts are
# updated to only pass platform-appropriate --solution values.
if (not options.solution
or os.path.splitext(options.solution)[1] != '.xcodeproj'):
options.solution = 'all.xcodeproj'
command.extend(['-project', options.solution])
if options.xcode_target:
command.extend(['-target', options.xcode_target])
# Note: this clobbers all targets, not just Debug or Release.
if options.clobber:
build_output_dir = os.path.join(os.path.dirname(options.build_dir),
'xcodebuild')
print('Removing %s' % build_output_dir)
chromium_utils.RemoveDirectory(build_output_dir)
env = EchoDict(os.environ)
common_xcode_settings(command, options, env, options.compiler)
# Add on any remaining args
command.extend(args)
# Set up the filter before changing directories so the raw build log can
# be recorded.
# Support a local file blocking filters (for debugging). Also check the
# Xcode version to make sure it is 3.2, as that is what the filter is coded
# to.
xcodebuild_filter = None
no_filter_path = os.path.join(os.getcwd(), 'no_xcodebuild_filter')
xcode_info = chromium_utils.GetCommandOutput(['xcodebuild', '-version'])
if os.path.exists(no_filter_path):
print 'NOTE: "%s" exists, output is unfiltered' % no_filter_path
elif not xcode_info.startswith('Xcode 3.2.'):
print 'NOTE: Not using Xcode 3.2, output is unfiltered'
else:
full_log_path = os.path.join(os.getcwd(), 'full_xcodebuild_log.txt')
full_log = open(full_log_path, 'w')
now = datetime.datetime.now()
full_log.write('Build started ' + now.isoformat() + '\n\n\n')
print 'NOTE: xcodebuild output filtered, full log at: "%s"' % full_log_path
xcodebuild_filter = XcodebuildFilter(full_log)
os.chdir(options.build_dir)
# If using the Goma compiler, first call goma_ctl with ensure_start
# (or restart in clobber mode) to ensure the proxy is available.
goma_ctl_cmd = [os.path.join(options.goma_dir, 'goma_ctl.sh')]
if options.compiler in ('goma', 'goma-clang', 'gomaclang'):
goma_key = os.path.join(options.goma_dir, 'goma.key')
env['GOMA_COMPILER_PROXY_DAEMON_MODE'] = 'true'
if os.path.exists(goma_key):
env['GOMA_API_KEY_FILE'] = goma_key
if options.clobber:
chromium_utils.RunCommand(goma_ctl_cmd + ['restart'], env=env)
else:
chromium_utils.RunCommand(goma_ctl_cmd + ['ensure_start'], env=env)
# Run the build.
env.print_overrides()
result = chromium_utils.RunCommand(command,
env=env,
filter_obj=xcodebuild_filter)
if options.compiler in ('goma', 'goma-clang', 'gomaclang'):
# Always stop the proxy for now to allow in-place update.
chromium_utils.RunCommand(goma_ctl_cmd + ['stop'], env=env)
return result
def main_xcode(options, args):
"""Interprets options, clobbers object files, and calls xcodebuild.
"""
env = EchoDict(os.environ)
goma_ready = goma_setup(options, env)
if not goma_ready:
assert options.compiler not in ('goma', 'goma-clang')
assert options.goma_dir is None
# Print some basic information about xcodebuild to ease debugging.
chromium_utils.RunCommand(['xcodebuild', '-sdk', '-version'], env=env)
# If the project isn't in args, add all.xcodeproj to simplify configuration.
command = ['xcodebuild', '-configuration', options.target]
# TODO(mmoss) Support the old 'args' usage until we're confident the master is
# switched to passing '--solution' everywhere.
if not '-project' in args:
# TODO(mmoss) Temporary hack to ignore the Windows --solution flag that is
# passed to all builders. This can be taken out once the master scripts are
# updated to only pass platform-appropriate --solution values.
if (not options.solution or
os.path.splitext(options.solution)[1] != '.xcodeproj'):
options.solution = '../build/all.xcodeproj'
command.extend(['-project', options.solution])
if options.xcode_target:
command.extend(['-target', options.xcode_target])
# Note: this clobbers all targets, not just Debug or Release.
if options.clobber:
clobber_dir = os.path.dirname(options.target_output_dir)
print 'Removing %s' % clobber_dir
# Deleting output_dir would also delete all the .ninja files. iOS builds
# generates ninja configuration inside the xcodebuild directory to be able
# to run sub builds. crbug.com/138950 is tracking this issue.
# Moreover clobbering should run before runhooks (which creates
# .ninja files). For now, only delete all non-.ninja files.
# TODO(thakis): Make "clobber" a step that runs before "runhooks". Once the
# master has been restarted, remove all clobber handling from compile.py.
build_directory.RmtreeExceptNinjaOrGomaFiles(clobber_dir)
common_xcode_settings(command, options, env, options.compiler)
maybe_set_official_build_envvars(options, env)
# Add on any remaining args
command.extend(args)
# Set up the filter before changing directories so the raw build log can
# be recorded.
# Support a local file blocking filters (for debugging). Also check the
# Xcode version to make sure it is 3.2, as that is what the filter is coded
# to.
xcodebuild_filter = None
no_filter_path = os.path.join(os.getcwd(), 'no_xcodebuild_filter')
xcode_info = chromium_utils.GetCommandOutput(['xcodebuild', '-version'])
if os.path.exists(no_filter_path):
print 'NOTE: "%s" exists, output is unfiltered' % no_filter_path
elif not xcode_info.startswith('Xcode 3.2.'):
# Note: The filter sometimes hides real errors on 4.x+, see crbug.com/260989
print 'NOTE: Not using Xcode 3.2, output is unfiltered'
else:
full_log_path = os.path.join(os.getcwd(), 'full_xcodebuild_log.txt')
full_log = open(full_log_path, 'w')
now = datetime.datetime.now()
full_log.write('Build started ' + now.isoformat() + '\n\n\n')
print 'NOTE: xcodebuild output filtered, full log at: "%s"' % full_log_path
xcodebuild_filter = XcodebuildFilter(full_log)
try:
os.makedirs(options.build_dir)
except OSError, e:
if e.errno != errno.EEXIST:
raise
