def _ArchiveGoodBuild(platform, commit_position):
"""Archive chromedriver binary if the build is green."""
assert platform != 'android'
util.MarkBuildStepStart('archive build')
server_name = 'chromedriver'
if util.IsWindows():
server_name += '.exe'
zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]),
server_name))
build_name = 'chromedriver_%s_%s.%s.zip' % (
platform, _GetVersion(), commit_position)
build_url = '%s/%s' % (GS_CONTINUOUS_URL, build_name)
if slave_utils.GSUtilCopy(zip_path, build_url):
util.MarkBuildStepError()
if util.IsWindows():
zip_path = util.Zip(os.path.join(
chrome_paths.GetBuildDir([server_name + '.pdb']), server_name + '.pdb'))
pdb_name = 'chromedriver_%s_pdb_%s.%s.zip' % (
platform, _GetVersion(), commit_position)
pdb_url = '%s/%s' % (GS_CONTINUOUS_URL, pdb_name)
if slave_utils.GSUtilCopy(zip_path, pdb_url):
util.MarkBuildStepError()
(latest_fd, latest_file) = tempfile.mkstemp()
os.write(latest_fd, build_name)
os.close(latest_fd)
latest_url = '%s/latest_%s' % (GS_CONTINUOUS_URL, platform)
if slave_utils.GSUtilCopy(latest_file, latest_url, mimetype='text/plain'):
util.MarkBuildStepError()
os.remove(latest_file)
def _MaybeUpdateLatestRelease(version):
"""Update the file LATEST_RELEASE with the latest release version number."""
latest_release_fname = 'LATEST_RELEASE'
latest_release_url = '%s/%s' % (GS_CHROMEDRIVER_BUCKET, latest_release_fname)
# Check if LATEST_RELEASE is up to date.
latest_released_version = _GetWebPageContent(
'%s/%s' % (GS_CHROMEDRIVER_RELEASE_URL, latest_release_fname))
if version == latest_released_version:
return
# Check if chromedriver was released on all supported platforms.
supported_platforms = ['linux64', 'mac32', 'win32']
for platform in supported_platforms:
if not _WasReleased(version, platform):
return
util.MarkBuildStepStart('updating LATEST_RELEASE to %s' % version)
temp_latest_release_fname = tempfile.mkstemp()[1]
with open(temp_latest_release_fname, 'w') as f:
f.write(version)
if slave_utils.GSUtilCopy(temp_latest_release_fname, latest_release_url,
mimetype='text/plain'):
util.MarkBuildStepError()
def UploadToGomaLogGS(file_path, gs_filename, text_to_append=None):
"""Upload a file to Google Cloud Storage (gs://chrome-goma-log).
Note that the uploaded file would automatically be gzip compressed.
Args:
file_path: a path of a file to be uploaded.
gs_filename: a name of a file in Google Storage.
text_to_append: an addtional text to be added to a file in GS.
Returns:
a stored path name without the bucket name in GS.
"""
hostname = GetShortHostname()
today = datetime.datetime.utcnow().date()
log_path = '%s/%s/%s.gz' % (
today.strftime('%Y/%m/%d'), hostname, gs_filename)
gs_path = 'gs://%s/%s' % (GOMA_LOG_GS_BUCKET, log_path)
temp = tempfile.NamedTemporaryFile(delete=False)
try:
with temp as f_out:
with gzip.GzipFile(fileobj=f_out) as gzipf_out:
with open(file_path) as f_in:
shutil.copyfileobj(f_in, gzipf_out)
if text_to_append:
gzipf_out.write(text_to_append)
slave_utils.GSUtilCopy(temp.name, gs_path)
print "Copied log file to %s" % gs_path
finally:
os.remove(temp.name)
return log_path
def _Release(build, platform):
"""Releases the given candidate build."""
release_name = 'chromedriver_%s.zip' % platform
util.MarkBuildStepStart('releasing %s' % release_name)
slave_utils.GSUtilCopy(
build, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, _GetVersion(), release_name))
_MaybeUploadReleaseNotes()
def _ArchivePrebuilts(commit_position):
"""Uploads the prebuilts to google storage."""
util.MarkBuildStepStart('archive prebuilts')
zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir(['chromedriver']),
'chromedriver'))
if slave_utils.GSUtilCopy(
zip_path,
'%s/%s' % (GS_PREBUILTS_URL, 'r%s.zip' % commit_position)):
util.MarkBuildStepError()
def _Release(build, version, platform):
"""Releases the given candidate build."""
release_name = 'chromedriver_%s.zip' % platform
util.MarkBuildStepStart('releasing %s' % release_name)
temp_dir = util.MakeTempDir()
slave_utils.GSUtilCopy(build, temp_dir)
zip_path = os.path.join(temp_dir, os.path.basename(build))
if util.IsLinux():
util.Unzip(zip_path, temp_dir)
server_path = os.path.join(temp_dir, 'chromedriver')
util.RunCommand(['strip', server_path])
zip_path = util.Zip(server_path)
slave_utils.GSUtilCopy(
zip_path, '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, release_name))
_MaybeUpdateLatestRelease(version)
def _ArchiveServerLogs():
"""Uploads chromedriver server logs to google storage."""
util.MarkBuildStepStart('archive chromedriver server logs')
for server_log in glob.glob(
os.path.join(tempfile.gettempdir(), 'chromedriver_*')):
base_name = os.path.basename(server_log)
util.AddLink(base_name, '%s/%s' % (SERVER_LOGS_LINK, base_name))
slave_utils.GSUtilCopy(server_log,
'%s/%s' % (GS_SERVER_LOGS_URL, base_name),
mimetype='text/plain')
def download(self):
status = slave_utils.GSUtilCopy(self.url, '.')
if 0 != status:
return False
try:
shutil.move(os.path.basename(self.url), self.archive_name)
except OSError:
os.remove(self.archive_name)
shutil.move(os.path.basename(self.url), self.archive_name)
return True
def CopyToGoogleStorage(src, dst):
"""Copies a file to the given Google Storage destination url.
Args:
src: path to file to be copied
dst: Google Storage destination url (i.e., gs://...)
Returns:
whether the copy was successful
"""
if not os.path.exists(src):
print 'No such file', src
return False
return slave_utils.GSUtilCopy(src, dst, None, 'public-read')
def _ArchiveGoodBuild(platform, revision):
assert platform != 'android'
util.MarkBuildStepStart('archive build')
server_name = 'chromedriver'
if util.IsWindows():
server_name += '.exe'
zip_path = util.Zip(os.path.join(chrome_paths.GetBuildDir([server_name]),
server_name))
build_url = '%s/chromedriver_%s_%s.%s.zip' % (
GS_CONTINUOUS_URL, platform, _GetVersion(), revision)
if slave_utils.GSUtilCopy(zip_path, build_url):
util.MarkBuildStepError()
def download(self):
override_gsutil = None
if self.gsutil_py_path:
override_gsutil = [sys.executable, self.gsutil_py_path]
status = slave_utils.GSUtilCopy(
self.url, '.', override_gsutil=override_gsutil)
if 0 != status:
return False
try:
shutil.move(os.path.basename(self.url), self.archive_name)
except OSError:
os.remove(self.archive_name)
shutil.move(os.path.basename(self.url), self.archive_name)
return True
def _MaybeUploadReleaseNotes(version):
"""Upload release notes if conditions are right."""
# Check if the current version has already been released.
notes_name = 'notes.txt'
notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, notes_name)
prev_version = '.'.join(
[version.split('.')[0],
str(int(version.split('.')[1]) - 1)])
prev_notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, prev_version,
notes_name)
result, _ = slave_utils.GSUtilListBucket(notes_url, [])
if result == 0:
return
fixed_issues = []
query = ('https://code.google.com/p/chromedriver/issues/csv?'
'can=1&q=label%%3AChromeDriver-%s&colspec=ID%%20Summary' %
version)
issues = StringIO.StringIO(_GetWebPageContent(query).split('\n', 1)[1])
for issue in csv.reader(issues):
if not issue:
continue
issue_id = issue[0]
desc = issue[1]
labels = issue[2].split(', ')
labels.remove('ChromeDriver-%s' % version)
if 'Hotlist-GoodFirstBug' in labels:
labels.remove('Hotlist-GoodFirstBug')
fixed_issues += [
'Resolved issue %s: %s [%s]' % (issue_id, desc, labels)
]
old_notes = ''
temp_notes_fname = tempfile.mkstemp()[1]
if not slave_utils.GSUtilDownloadFile(prev_notes_url, temp_notes_fname):
with open(temp_notes_fname, 'rb') as f:
old_notes = f.read()
new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
version, datetime.date.today().isoformat(), 'Supports Chrome v%s-%s' %
_GetSupportedChromeVersions(), '\n'.join(fixed_issues), old_notes)
with open(temp_notes_fname, 'w') as f:
f.write(new_notes)
if slave_utils.GSUtilCopy(temp_notes_fname,
notes_url,
mimetype='text/plain'):
util.MarkBuildStepError()
def UploadGomaCompilerProxyInfo():
"""Upload goma compiler_proxy.INFO to Google Storage."""
latest_info = GetLatestGomaCompilerProxyInfo()
today = datetime.datetime.utcnow().date()
hostname = GetShortHostname()
# Since a filename of compiler_proxy.INFO is fairly unique,
# we might be able to upload it as-is.
goma_log_gs_path = (
'gs://chrome-goma-log/%s/%s/%s.gz' %
(today.strftime('%Y/%m/%d'), hostname, os.path.basename(latest_info)))
try:
fd, output_filename = tempfile.mkstemp()
with open(latest_info) as f_in:
with os.fdopen(fd, 'w') as f_out:
with gzip.GzipFile(fileobj=f_out, compresslevel=9) as gzipf:
gzipf.writelines(f_in)
slave_utils.GSUtilCopy(output_filename, goma_log_gs_path)
print "Copied log file to %s" % goma_log_gs_path
finally:
os.remove(output_filename)
ninja_log_filename = 'ninja_log.%s.%s.%s.%d' % (
hostname, username, mtime.strftime('%Y%m%d-%H%M%S'), pid)
today = datetime.datetime.utcnow().date()
ninja_log_gs_path = (
'gs://chrome-goma-log/%s/%s/%s.gz' %
(today.strftime('%Y/%m/%d'), hostname, ninja_log_filename))
try:
fd, output_filename = tempfile.mkstemp()
with open(ninja_log_path) as f_in:
with os.fdopen(fd, 'w') as f_out:
with gzip.GzipFile(fileobj=f_out, compresslevel=9) as gzipf:
gzipf.writelines(f_in)
gzipf.write('# end of ninja log\n')
gzipf.write(json.dumps(info))
slave_utils.GSUtilCopy(output_filename, ninja_log_gs_path)
print "Copied log file to %s" % ninja_log_gs_path
finally:
os.remove(output_filename)
def common_xcode_settings(command, options, env, compiler=None):
"""
Sets desirable Mac environment variables and command-line options
that are common to the Xcode builds.
"""
compiler = options.compiler
assert compiler in (None, 'clang', 'goma', 'goma-clang')
if compiler == 'goma':
print 'using goma'
