def getAllChanges(self, request, status, debug_info):
master = request.site.buildbot_service.master
max_rev_limit = skia_vars.GetGlobalVariable('console_max_rev_limit')
default_rev_limit = skia_vars.GetGlobalVariable(
'console_default_rev_limit')
limit = min(
max_rev_limit,
max(1, int(request.args.get('limit', [default_rev_limit])[0])))
wfd = defer.waitForDeferred(master.db.changes.getRecentChanges(limit))
yield wfd
chdicts = wfd.getResult()
# convert those to Change instances
wfd = defer.waitForDeferred(
defer.gatherResults([
changes_module.Change.fromChdict(master, chdict)
for chdict in chdicts
]))
yield wfd
all_changes = wfd.getResult()
all_changes.sort(key=self.comparator.getSortingKey())
# Remove the dups
prev_change = None
new_changes = []
for change in all_changes:
rev = change.revision
if not prev_change or rev != prev_change.revision:
new_changes.append(change)
prev_change = change
all_changes = new_changes
debug_info["source_len"] = len(all_changes)
yield all_changes
def content(self, request, cxt):
"This method builds the main console view display."
reload_time = None
# Check if there was an arg. Don't let people reload faster than
# every 15 seconds. 0 means no reload.
if "reload" in request.args:
try:
reload_time = int(request.args["reload"][0])
if reload_time != 0:
reload_time = max(reload_time, 15)
except ValueError:
pass
request.setHeader('Cache-Control', 'no-cache')
# Sets the default reload time to 60 seconds.
if not reload_time:
reload_time = skia_vars.GetGlobalVariable(
'default_webstatus_refresh')
# Append the tag to refresh the page.
if reload_time is not None and reload_time != 0:
cxt['refresh'] = reload_time
# List of categories for which we load information but hide initially.
hidden_categories_sets = request.args.get("hideCategories", [])
hide_categories = []
for category_set in hidden_categories_sets:
hide_categories.extend(category_set.split(','))
cxt['hide_categories'] = hide_categories
# List of subcategories for which we load information but hide initially.
hidden_subcategories_sets = request.args.get("hideSubcategories", [])
hide_subcategories = []
for subcategory_set in hidden_subcategories_sets:
hide_subcategories.extend(subcategory_set.split(','))
cxt['hide_subcategories'] = hide_subcategories
# Console event-loading limits.
cxt['default_console_limit'] = \
skia_vars.GetGlobalVariable('console_default_rev_limit')
cxt['max_console_limit'] = \
skia_vars.GetGlobalVariable('console_max_rev_limit')
templates = request.site.buildbot_service.templates
template = templates.get_template("console.html")
data = template.render(cxt)
return data
def _UploadDMResults(self):
# self._dm_dir holds a bunch of <hash>.png files and one dm.json.
# Upload the dm.json summary file.
summary_dir = tempfile.mkdtemp()
shutil.move(os.path.join(self._dm_dir, 'dm.json'),
os.path.join(summary_dir, 'dm.json'))
# /dm-json-v1/year/month/day/hour/git-hash/builder/build-number
now = datetime.datetime.utcnow()
summary_dest_dir = '/'.join([
'dm-json-v1',
str(now.year).zfill(4),
str(now.month).zfill(2),
str(now.day).zfill(2),
str(now.hour).zfill(2), self._got_revision, self._builder_name,
self._build_number
])
# Trybot results are further siloed by CL.
if self._is_try:
summary_dest_dir = '/'.join(
['trybot', summary_dest_dir, self._args['issue_number']])
gs = gs_utils.GSUtils()
acl = gs.PLAYBACK_CANNED_ACL # Private,
fine_acls = gs.PLAYBACK_FINEGRAINED_ACL_LIST # but Google-visible.
gs.upload_dir_contents(
source_dir=summary_dir,
dest_bucket=skia_vars.GetGlobalVariable('dm_summaries_bucket'),
dest_dir=summary_dest_dir,
upload_if=gs.UploadIf.ALWAYS,
predefined_acl=acl,
fine_grained_acl_list=fine_acls)
# Now we can upload everything that's left, all the .pngs.
gs.upload_dir_contents(
source_dir=self._dm_dir,
dest_bucket=skia_vars.GetGlobalVariable('dm_images_bucket'),
dest_dir='dm-images-v1',
upload_if=gs.UploadIf.IF_NEW,
predefined_acl=acl,
fine_grained_acl_list=fine_acls)
# Just for hygiene, put dm.json back.
shutil.move(os.path.join(summary_dir, 'dm.json'),
os.path.join(self._dm_dir, 'dm.json'))
os.rmdir(summary_dir)
def _Run(self):
# Copy actual-results.json to skimage/actuals
print '\n\n====Uploading skimage actual-results to Google Storage====\n\n'
src_dir = os.path.abspath(
os.path.join(self._skimage_out_dir, self._builder_name))
bucket_url = gs_utils.GSUtils.with_gs_prefix(
skia_vars.GetGlobalVariable('googlestorage_bucket'))
dest_dir = posixpath.join(bucket_url, 'skimage', 'actuals',
self._builder_name)
http_header_lines = ['Cache-Control:public,max-age=3600']
old_gs_utils.upload_dir_contents(local_src_dir=src_dir,
remote_dest_dir=dest_dir,
gs_acl='public-read',
http_header_lines=http_header_lines)
# Copy actual images to Google Storage at skimage/output. This will merge
# with the existing files.
print '\n\n========Uploading skimage results to Google Storage=======\n\n'
src_dir = os.path.abspath(os.path.join(self._skimage_out_dir,
'images'))
dest_dir = posixpath.join(bucket_url, 'skimage', 'output', 'images')
if os.path.isdir(src_dir) and os.listdir(src_dir):
old_gs_utils.upload_dir_contents(
local_src_dir=src_dir,
remote_dest_dir=dest_dir,
gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL)
def displayPage(self, request, status, builder_list, all_builds, revisions,
categories, repository, branch, debug_info):
"""Display the console page."""
# Build the main template directory with all the informations we have.
subs = dict()
subs["branch"] = branch or 'trunk'
subs["repository"] = repository
if categories:
subs["categories"] = ' '.join(categories)
subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S",
time.localtime(util.now()))
subs["debugInfo"] = debug_info
subs["ANYBRANCH"] = ANYBRANCH
if builder_list:
builders = builder_list
else:
builders = {}
subs['builders'] = builders
subs['revisions'] = []
# For each revision we show one line
for revision in revisions:
r = {}
# Fill the dictionary with this new information
r['id'] = revision.revision
r['link'] = revision.revlink
if (skia_vars.GetGlobalVariable('commit_bot_username')
in revision.who and 'Author: ' in revision.comments):
who = revision.comments.split('Author: ')[1].split('\n')[0]
who += ' (commit-bot)'
else:
who = revision.who
r['who'] = utils.FixGitSvnEmail(who)
r['date'] = revision.date
r['comments'] = revision.comments
r['repository'] = revision.repository
r['project'] = revision.project
# Display the status for all builders.
(builds, details) = self.displayStatusLine(builder_list,
all_builds, revision,
debug_info)
r['builds'] = builds
r['details'] = details
# Calculate the td span for the comment and the details.
r["span"] = sum ([len(builder_list[category]) \
for category in builder_list]) + 2
subs['revisions'].append(r)
#
# Display the footer of the page.
#
debug_info["load_time"] = time.time() - debug_info["load_time"]
return subs
class CompileSkia(Skia):
project_name = 'CompileSkia'
project_url = skia_vars.GetGlobalVariable('project_url')
master_host = skia_vars.GetGlobalVariable('compile_master_host')
master_fqdn = SKIA_COMPILE_MASTER_INTERNAL_FQDN
is_production_host = socket.getfqdn(
) == SKIA_COMPILE_MASTER_INTERNAL_FQDN
_skip_render_results_upload = False
_skip_bench_results_upload = False
master_port = skia_vars.GetGlobalVariable('compile_internal_port')
slave_port = skia_vars.GetGlobalVariable('compile_slave_port')
master_port_alt = skia_vars.GetGlobalVariable('compile_external_port')
tree_closing_notification_recipients = ['*****@*****.**']
from_address = skia_vars.GetGlobalVariable('gce_smtp_user')
is_publicly_visible = True
code_review_site = \
skia_vars.GetGlobalVariable('code_review_status_listener')
def create_schedulers_and_builders(self, cfg):
"""Create the Schedulers and Builders.
Args:
cfg: dict; configuration dict for the build master.
"""
# These imports needs to happen inside this function because modules
# imported by master_builders_cfg import this module.
import master_builders_cfg
import master_compile_builders_cfg
master_builders_cfg.create_schedulers_and_builders(
sys.modules[__name__], self, cfg,
master_compile_builders_cfg.setup_all_builders)
def CompareGMs(self):
"""Compare the actually-generated GM images to the checked-in baselines."""
self.AddSlaveScript(
script='compare_gms.py',
description='CompareGMs',
get_props_from_stdout={
'latest_gm_failures_url':
'%s([^\n]*)\n' %
skia_vars.GetGlobalVariable('latest_gm_failures_preamble')
},
is_rebaseline_step=True)
def rebaseline_server_url(directive, builder_name):
"""Returns the URL for results from this builder on rebaseline_server.
Args:
directive: part of the URL, indicating whether to generate a 'prefetch'
URL, a 'live-loader' URL, or a link to live-view.html
builder_name: name of this builder
"""
bucket = skia_vars.GetGlobalVariable('skp_summaries_bucket')
return URL_FORMATTER.format(
directive=directive,
actuals_summary_dir=urllib.quote(
'gs://%s/%s' % (bucket, builder_name), safe=''))
def _GSUploadAllImages(self, src_dir):
"""Upload all image files from src_dir to Google Storage.
We know that GM wrote out these image files with a filename pattern we
can use to generate the checksum-based Google Storage paths."""
all_files = sorted(os.listdir(src_dir))
def filematcher(filename):
return filename.endswith('.png')
files_to_upload = filter(filematcher, all_files)
print 'Uploading %d GM-actual files to Google Storage...' % (
len(files_to_upload))
if not files_to_upload:
return
filename_pattern = re.compile('^([^_]+)_(.+)_([^_]+)\.png$')
gm_actuals_subdir = 'gm'
temp_root = tempfile.mkdtemp()
try:
# Copy all of the desired files to a staging dir, with new filenames.
for filename in files_to_upload:
match = filename_pattern.match(filename)
if not match:
print 'Warning: found no images matching pattern "%s"' % filename
continue
(hashtype, test, hashvalue) = match.groups()
src_filepath = os.path.join(src_dir, filename)
temp_dir = os.path.join(temp_root, gm_actuals_subdir, hashtype,
test)
if not os.path.isdir(temp_dir):
os.makedirs(temp_dir)
shutil.copy(src_filepath,
os.path.join(temp_dir, hashvalue + '.png'))
# Upload the entire staging dir to Google Storage.
# At present, this will merge the entire contents of [temp_root]/gm
# into the existing contents of gs://chromium-skia-gm/gm .
#
# TODO(epoger): Add a "noclobber" mode to gs_utils.upload_dir_contents()
# and use it here so we don't re-upload image files we already have
# in Google Storage.
bucket_url = gs_utils.GSUtils.with_gs_prefix(
skia_vars.GetGlobalVariable('googlestorage_bucket'))
old_gs_utils.upload_dir_contents(
local_src_dir=os.path.abspath(
os.path.join(temp_root, gm_actuals_subdir)),
remote_dest_dir=posixpath.join(bucket_url, gm_actuals_subdir),
gs_acl='public-read',
http_header_lines=['Cache-Control:public,max-age=3600'])
finally:
shutil.rmtree(temp_root)
def got_changes(all_changes):
debug_info["source_all"] = len(all_changes)
rev_filter = {}
if branch != ANYBRANCH:
rev_filter['branch'] = branch
if dev_name:
rev_filter['who'] = dev_name
rev_filter['repository'] = skia_vars.GetGlobalVariable(
'skia_git_url')
revisions = list(
self.filterRevisions(all_changes,
max_revs=num_revs,
rev_filter=rev_filter))
debug_info["revision_final"] = len(revisions)
# Fetch all the builds for all builders until we get the next build
# after last_revision.
builder_list = None
all_builds = None
if revisions:
last_revision = revisions[len(revisions) - 1].revision
debug_info["last_revision"] = last_revision
(builder_list, all_builds) = self.getAllBuildsForRevision(
status, request, last_revision, num_builds, categories,
builders, debug_info)
debug_info["added_blocks"] = 0
debug_info["from_cache"] = 0
if request.args.get("display_cache", None):
data = ""
data += "\nGlobal Cache\n"
data += self.cache.display()
return data
cxt.update(
self.displayPage(request, status, builder_list, all_builds,
revisions, categories, repository, branch,
debug_info))
# Clean up the cache.
if debug_info["added_blocks"]:
self.cache.trim()
return {'builders': cxt['builders'], 'revisions': cxt['revisions']}
def _Run(self):
# Tar up the results.
result_tarball = '%s_%s.tgz' % (self._builder_name, self._got_revision)
shell_utils.run([
'tar', '-cvzf',
os.path.join(os.pardir, result_tarball),
self._flavor_utils.result_dir
])
# Upload to Google Storage
bucket_url = gs_utils.GSUtils.with_gs_prefix(
skia_vars.GetGlobalVariable('googlestorage_bucket'))
upload_to_bucket.upload_to_bucket(os.path.join(os.pardir,
result_tarball),
bucket_url,
subdir=GS_DRT_SUBDIR)
print 'To download the tarball, run this command:'
gs_url = posixpath.join(bucket_url, GS_DRT_SUBDIR, result_tarball)
print 'gsutil cp %s <local_dir>' % gs_url
def _Run(self):
json_summary_path = misc.GetAbsPath(
os.path.join(self._gm_actual_dir, run_gm.JSON_SUMMARY_FILENAME))
# Temporary list of builders who are allowed to fail this step without the
# bot turning red.
may_fail_with_warning = []
# This import must happen after BuildStep.__init__ because it requires that
# CWD is in PYTHONPATH, and BuildStep.__init__ may change the CWD.
from gm import display_json_results
success = display_json_results.Display(json_summary_path)
print('%s<a href="%s?resultsToLoad=/results/failures&builder=%s">'
'link</a>' %
(skia_vars.GetGlobalVariable('latest_gm_failures_preamble'),
LIVE_REBASELINE_SERVER_BASEURL, self._builder_name))
if not success:
if self._builder_name in may_fail_with_warning:
raise BuildStepWarning('Expectations mismatch in %s!' %
json_summary_path)
else:
raise Exception('Expectations mismatch in %s!' %
json_summary_path)
def HtmlResourceRender(self, request):
""" Override of buildbot.status.web.base.HtmlResource.render:
http://src.chromium.org/viewvc/chrome/trunk/tools/build/third_party/buildbot_8_4p1/buildbot/status/web/base.py?view=markup
We modify it to pass additional variables on to the web status pages, and
remove the "if False" section.
"""
# tell the WebStatus about the HTTPChannel that got opened, so they
# can close it if we get reconfigured and the WebStatus goes away.
# They keep a weakref to this, since chances are good that it will be
# closed by the browser or by us before we get reconfigured. See
# ticket #102 for details.
if hasattr(request, "channel"):
# web.distrib.Request has no .channel
request.site.buildbot_service.registerChannel(request.channel)
ctx = self.getContext(request)
############################## Added by borenet ##############################
status = self.getStatus(request)
all_builders = status.getBuilderNames()
all_full_category_names = set()
all_categories = set()
all_subcategories = set()
subcategories_by_category = {}
for builder_name in all_builders:
category_full = status.getBuilder(builder_name).category or 'default'
all_full_category_names.add(category_full)
category_split = category_full.split('|')
category = category_split[0]
subcategory = category_split[1] if len(
category_split) > 1 else 'default'
all_categories.add(category)
all_subcategories.add(subcategory)
if not subcategories_by_category.get(category):
subcategories_by_category[category] = []
if not subcategory in subcategories_by_category[category]:
subcategories_by_category[category].append(subcategory)
ctx['tree_status_baseurl'] = \
skia_vars.GetGlobalVariable('tree_status_baseurl')
ctx['all_full_category_names'] = sorted(list(all_full_category_names))
ctx['all_categories'] = sorted(list(all_categories))
ctx['all_subcategories'] = sorted(list(all_subcategories))
ctx['subcategories_by_category'] = subcategories_by_category
ctx['default_refresh'] = \
skia_vars.GetGlobalVariable('default_webstatus_refresh')
ctx['skia_repo'] = config_private.SKIA_GIT_URL
active_master = config_private.Master.get_active_master()
ctx['internal_port'] = active_master.master_port
ctx['external_port'] = active_master.master_port_alt
ctx['title_url'] = config_private.Master.Skia.project_url
ctx['slave_hosts_cfg'] = slave_hosts_cfg.SLAVE_HOSTS
ctx['slaves_cfg'] = slaves_cfg.SLAVES
ctx['active_master_name'] = active_master.project_name
ctx['master_revision'] = utils.get_current_revision()
ctx['master_running_revision'] = active_master.running_revision
ctx['master_launch_datetime'] = active_master.launch_datetime
ctx['is_internal_view'] = request.host.port == ctx['internal_port']
ctx['masters'] = []
for master in config_private.Master.valid_masters:
ctx['masters'].append({
'name': master.project_name,
'host': master.master_host,
'internal_port': master.master_port,
'external_port': master.master_port_alt,
})
##############################################################################
d = defer.maybeDeferred(lambda: self.content(request, ctx))
def handle(data):
if isinstance(data, unicode):
data = data.encode("utf-8")
request.setHeader("content-type", self.contentType)
if request.method == "HEAD":
request.setHeader("content-length", len(data))
return ''
return data
d.addCallback(handle)
def ok(data):
request.write(data)
request.finish()
def fail(f):
request.processingFailed(f)
return None # processingFailed will log this for us
d.addCallbacks(ok, fail)
return server.NOT_DONE_YET
This differs from UpdateScripts in that it updates ALL of the buildbot script
checkouts for ALL buildslaves, as opposed to a single buildslave's checkout of
the buildbot scripts on a single host machine.
"""
import re
import skia_vars
import sys
from build_step import BuildStep, BuildStepWarning
from scripts import run_cmd
from utils import force_update_checkout
BUILDBOT_GIT_URL = skia_vars.GetGlobalVariable('buildbot_git_url')
class UpdateAllBuildslaves(BuildStep):
def _Run(self):
script_path = run_cmd.ResolvablePath('slave', 'skia_slave_scripts', 'utils',
'force_update_checkout.py')
sync_cmd = ['python', script_path]
results = run_cmd.run_on_all_slaves_on_all_hosts(sync_cmd)
failed = []
for host in results.iterkeys():
print host
# If results[host] is a MultiCommandResults instance, then we have results
# for buildslaves running on that machine, which implies that we were able
# to log in to the machine successfully.
if isinstance(results[host], run_cmd.MultiCommandResults):
# pylint: enable=C0301
import posixpath
import sys
from build_step import BuildStep
from utils import gs_utils
from utils import old_gs_utils
import generate_doxygen
import skia_vars
# It's silly that we include 'doxygen' in the destination path twice, but
# that's to maintain current behavior while fixing http://skbug.com/2658 .
DOXYGEN_GSUTIL_PATH = posixpath.join(
gs_utils.GSUtils.with_gs_prefix(
skia_vars.GetGlobalVariable('googlestorage_bucket')), 'doxygen',
'doxygen')
# Directives for HTTP caching of these files served out by Google Storage.
# See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
GS_CACHE_CONTROL_HEADER = 'Cache-Control:public,max-age=3600'
class UploadDoxygen(BuildStep):
def _Run(self):
old_gs_utils.upload_dir_contents(
local_src_dir=generate_doxygen.DOXYGEN_WORKING_DIR,
remote_dest_dir=DOXYGEN_GSUTIL_PATH,
gs_acl='public-read',
http_header_lines=[GS_CACHE_CONTROL_HEADER])
class Master(config_default.Master):
googlecode_revlinktmpl = 'http://code.google.com/p/%s/source/browse?r=%s'
bot_password = '******'
default_clobber = False
# SMTP configurations.
smtp_server = skia_vars.GetGlobalVariable('gce_smtp_server')
smtp_port = skia_vars.GetGlobalVariable('gce_smtp_port')
smtp_use_tls = skia_vars.GetGlobalVariable('gce_smtp_use_tls')
smtp_user = skia_vars.GetGlobalVariable('gce_smtp_user')
# domains to which we will send blame emails
permitted_domains = ['google.com', 'chromium.org']
class Skia(object):
project_name = 'Skia'
project_url = skia_vars.GetGlobalVariable('project_url')
master_host = skia_vars.GetGlobalVariable('public_master_host')
master_fqdn = SKIA_PUBLIC_MASTER_INTERNAL_FQDN
is_production_host = socket.getfqdn(
) == SKIA_PUBLIC_MASTER_INTERNAL_FQDN
_skip_render_results_upload = False
_skip_bench_results_upload = False
master_port = skia_vars.GetGlobalVariable('public_internal_port')
slave_port = skia_vars.GetGlobalVariable('public_slave_port')
master_port_alt = skia_vars.GetGlobalVariable('public_external_port')
tree_closing_notification_recipients = ['*****@*****.**']
from_address = skia_vars.GetGlobalVariable('gce_smtp_user')
is_publicly_visible = True
code_review_site = \
skia_vars.GetGlobalVariable('code_review_status_listener')
tree_status_url = skia_vars.GetGlobalVariable('tree_status_url')
@property
def do_upload_render_results(self):
return self.is_production_host and not self._skip_render_results_upload
@property
def do_upload_bench_results(self):
return self.is_production_host and not self._skip_bench_results_upload
def create_schedulers_and_builders(self, cfg):
"""Create the Schedulers and Builders.
Args:
cfg: dict; configuration dict for the build master.
"""
# This import needs to happen inside this function because modules
# imported by master_builders_cfg import this module.
import master_builders_cfg
master_builders_cfg.create_schedulers_and_builders(
sys.modules[__name__], self, cfg,
master_builders_cfg.setup_all_builders)
class PrivateSkia(Skia):
project_name = 'PrivateSkia'
project_url = skia_vars.GetGlobalVariable('project_url')
master_host = skia_vars.GetGlobalVariable('private_master_host')
master_fqdn = SKIA_PRIVATE_MASTER_INTERNAL_FQDN
is_production_host = socket.getfqdn(
) == SKIA_PRIVATE_MASTER_INTERNAL_FQDN
_skip_render_results_upload = False
# Don't upload bench results on the private master, since we don't yet have
# a private destination for them.
_skip_bench_results_upload = True
master_port = skia_vars.GetGlobalVariable('private_internal_port')
slave_port = skia_vars.GetGlobalVariable('private_slave_port')
master_port_alt = skia_vars.GetGlobalVariable('private_external_port')
tree_closing_notification_recipients = []
from_address = skia_vars.GetGlobalVariable('gce_smtp_user')
is_publicly_visible = False
code_review_site = \
skia_vars.GetGlobalVariable('code_review_status_listener')
def create_schedulers_and_builders(self, cfg):
"""Create the Schedulers and Builders.
Args:
cfg: dict; configuration dict for the build master.
"""
# These imports needs to happen inside this function because modules
# imported by master_builders_cfg import this module.
import master_builders_cfg
import master_private_builders_cfg
master_builders_cfg.create_schedulers_and_builders(
sys.modules[__name__], self, cfg,
master_private_builders_cfg.setup_all_builders)
class FYISkia(Skia):
project_name = 'FYISkia'
project_url = skia_vars.GetGlobalVariable('project_url')
master_host = skia_vars.GetGlobalVariable('fyi_master_host')
master_fqdn = SKIA_FYI_MASTER_INTERNAL_FQDN
is_production_host = socket.getfqdn() == SKIA_FYI_MASTER_INTERNAL_FQDN
_skip_render_results_upload = False
_skip_bench_results_upload = False
master_port = skia_vars.GetGlobalVariable('fyi_internal_port')
slave_port = skia_vars.GetGlobalVariable('fyi_slave_port')
master_port_alt = skia_vars.GetGlobalVariable('fyi_external_port')
tree_closing_notification_recipients = []
from_address = skia_vars.GetGlobalVariable('gce_smtp_user')
is_publicly_visible = True
code_review_site = \
skia_vars.GetGlobalVariable('code_review_status_listener')
def create_schedulers_and_builders(self, cfg):
"""Create the Schedulers and Builders.
Args:
cfg: dict; configuration dict for the build master.
"""
# These imports needs to happen inside this function because modules
# imported by master_builders_cfg import this module.
import master_builders_cfg
import master_fyi_builders_cfg
master_builders_cfg.create_schedulers_and_builders(
sys.modules[__name__], self, cfg,
master_fyi_builders_cfg.setup_all_builders)
class AndroidSkia(Skia):
project_name = 'AndroidSkia'
project_url = skia_vars.GetGlobalVariable('project_url')
master_host = skia_vars.GetGlobalVariable('android_master_host')
master_fqdn = SKIA_ANDROID_MASTER_INTERNAL_FQDN
is_production_host = socket.getfqdn(
) == SKIA_ANDROID_MASTER_INTERNAL_FQDN
_skip_render_results_upload = False
_skip_bench_results_upload = False
master_port = skia_vars.GetGlobalVariable('android_internal_port')
slave_port = skia_vars.GetGlobalVariable('android_slave_port')
master_port_alt = skia_vars.GetGlobalVariable('android_external_port')
tree_closing_notification_recipients = []
from_address = skia_vars.GetGlobalVariable('gce_smtp_user')
is_publicly_visible = True
code_review_site = \
skia_vars.GetGlobalVariable('code_review_status_listener')
def create_schedulers_and_builders(self, cfg):
"""Create the Schedulers and Builders.
Args:
cfg: dict; configuration dict for the build master.
"""
# These imports needs to happen inside this function because modules
# imported by master_builders_cfg import this module.
import master_builders_cfg
import master_android_builders_cfg
master_builders_cfg.create_schedulers_and_builders(
sys.modules[__name__], self, cfg,
master_android_builders_cfg.setup_all_builders)
class CompileSkia(Skia):
project_name = 'CompileSkia'
project_url = skia_vars.GetGlobalVariable('project_url')
master_host = skia_vars.GetGlobalVariable('compile_master_host')
master_fqdn = SKIA_COMPILE_MASTER_INTERNAL_FQDN
is_production_host = socket.getfqdn(
) == SKIA_COMPILE_MASTER_INTERNAL_FQDN
_skip_render_results_upload = False
_skip_bench_results_upload = False
master_port = skia_vars.GetGlobalVariable('compile_internal_port')
slave_port = skia_vars.GetGlobalVariable('compile_slave_port')
master_port_alt = skia_vars.GetGlobalVariable('compile_external_port')
tree_closing_notification_recipients = ['*****@*****.**']
from_address = skia_vars.GetGlobalVariable('gce_smtp_user')
is_publicly_visible = True
code_review_site = \
skia_vars.GetGlobalVariable('code_review_status_listener')
def create_schedulers_and_builders(self, cfg):
"""Create the Schedulers and Builders.
Args:
cfg: dict; configuration dict for the build master.
"""
# These imports needs to happen inside this function because modules
# imported by master_builders_cfg import this module.
import master_builders_cfg
import master_compile_builders_cfg
master_builders_cfg.create_schedulers_and_builders(
sys.modules[__name__], self, cfg,
master_compile_builders_cfg.setup_all_builders)
# List of the valid master classes.
valid_masters = [Skia, PrivateSkia, FYISkia, AndroidSkia, CompileSkia]
@staticmethod
def set_active_master(master_name):
"""Sets the master with the given name as active and returns its instance.
Args:
master_name: string; name of the desired build master.
"""
global _ACTIVE_MASTER
master = Master.get(master_name)
if master:
_ACTIVE_MASTER = master()
return _ACTIVE_MASTER
raise Exception('Invalid master: %s' % master_name)
@staticmethod
def get(master_name):
"""Return the master with the given name or None if no such master exists.
Args:
master_name: string; name of the desired build master.
"""
for master in Master.valid_masters:
if master_name == master.__name__:
return master
return None
@staticmethod
def get_active_master():
"""Returns the instance of the active build master."""
return _ACTIVE_MASTER
import datetime
import json
import os
import smtplib
import sys
import urllib2
# Set the PYTHONPATH for this script to include skia site_config.
sys.path.append(os.path.join(os.pardir, 'site_config'))
import skia_vars
SHERIFF_EMAIL_TYPE = 'sheriff'
STATUS_EMAIL_TYPE = 'status'
ALL_EMAIL_TYPES = (SHERIFF_EMAIL_TYPE, STATUS_EMAIL_TYPE)
NEXT_SHERIFF_JSON_URL = skia_vars.GetGlobalVariable('next_sheriff_url')
DEFAULT_EMAIL_SENDER = '*****@*****.**'
ADDITIONAL_EMAIL_RECIPIENTS = ['*****@*****.**']
def _GetSheriffDetails():
"""Returns the next sheriff and his/her schedule."""
connection = urllib2.urlopen(NEXT_SHERIFF_JSON_URL)
sheriff_details = json.loads(connection.read())
connection.close()
return sheriff_details
def EmailSheriff():
"""Sends an email to the next sheriff."""
sheriff_details = _GetSheriffDetails()
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# These buildbot configurations are "private" in the sense that they are
# specific to Skia buildbots (not shared by other Chromium buildbots).
# But this file is stored within a public SVN repository, so don't put any
# secrets in here.
import socket
import skia_vars
import sys
# import base class from third_party/chromium_buildbot/site_config/
import config_default
CODE_REVIEW_SITE = skia_vars.GetGlobalVariable('code_review_site')
# On startup, the build master validates the bot configuration against a known
# expectation. If this variable is set to true (eg. in a buildbot self-test),
# the master will fail to start up if validation fails.
#
# For more information: https://code.google.com/p/skia/issues/detail?id=1289
die_on_validation_failure = False
# Skia's Google Compute Engine instances.
# The public master which is visible to everyone.
SKIA_PUBLIC_MASTER_INTERNAL_FQDN = skia_vars.GetGlobalVariable(
'public_master_internal_fqdn')
# The private master which is visible only to Google corp.
SKIA_PRIVATE_MASTER_INTERNAL_FQDN = skia_vars.GetGlobalVariable(
'private_master_internal_fqdn')
"source": ".boto",
"destination": "buildbot/third_party/chromium_buildbot/site_config",
},
]
CHROMEBUILD_COPIES = [
{
"source": ".boto",
"destination": "build/site_config",
},
{
"source": ".bot_password",
"destination": "build/site_config",
},
]
GCE_PROJECT = skia_vars.GetGlobalVariable('gce_project')
GCE_USERNAME = skia_vars.GetGlobalVariable('gce_username')
GCE_ZONE = skia_vars.GetGlobalVariable('gce_compile_bots_zone')
GCE_COMPILE_A_ONLINE = GCE_ZONE == 'a'
GCE_COMPILE_B_ONLINE = GCE_ZONE == 'b'
GCE_COMPILE_C_ONLINE = True
LAUNCH_SCRIPT_UNIX = ['scripts', 'skiabot-slave-start-on-boot.sh']
LAUNCH_SCRIPT_WIN = ['scripts', 'skiabot-slave-start-on-boot.bat']
SKIALAB_ROUTER_IP = skia_vars.GetGlobalVariable('skialab_router_ip')
SKIALAB_USERNAME = skia_vars.GetGlobalVariable('skialab_username')
# Procedures for logging in to the host machines.
def Sync(skia_revision=SKIA_REV_MASTER,
chrome_revision=CHROME_REV_LKGR,
fetch_target=DEFAULT_FETCH_TARGET,
gyp_defines=None,
gyp_generators=None):
""" Create and sync a checkout of Skia inside a checkout of Chrome. Returns
a tuple containing the actually-obtained revision of Skia and the actually-
obtained revision of Chrome.
skia_revision: revision of Skia to sync. Should be a commit hash or one of
(SKIA_REV_DEPS, SKIA_REV_MASTER).
chrome_revision: revision of Chrome to sync. Should be a commit hash or one
of (CHROME_REV_LKGR, CHROME_REV_MASTER).
fetch_target: string; Calls the fetch tool in depot_tools with the specified
argument. Default is DEFAULT_FETCH_TARGET.
gyp_defines: optional string; GYP_DEFINES to be passed to Gyp.
gyp_generators: optional string; which GYP_GENERATORS to use.
"""
# Figure out what revision of Skia we should use.
if skia_revision == SKIA_REV_MASTER:
output = git_utils.GetRemoteMasterHash(
skia_vars.GetGlobalVariable('skia_git_url'))
if output:
skia_revision = shlex.split(output)[0]
if not skia_revision:
raise Exception('Could not determine current Skia revision!')
skia_revision = str(skia_revision)
# Use Chrome LKGR, since gclient_utils will force a sync to origin/master.
if chrome_revision == CHROME_REV_LKGR:
chrome_revision = urllib2.urlopen(CHROME_LKGR_URL).read()
elif chrome_revision == CHROME_REV_MASTER:
chrome_revision = shlex.split(
git_utils.GetRemoteMasterHash(CHROME_GIT_URL))[0]
# Run "fetch chromium". The initial run is allowed to fail after it does some
# work. At the least, we expect the .gclient file to be present when it
# finishes.
if not os.path.isfile(GCLIENT_FILE):
try:
shell_utils.run([FETCH, fetch_target, '--nosvn=True'])
except shell_utils.CommandFailedException:
pass
if not os.path.isfile(GCLIENT_FILE):
raise Exception('Could not fetch %s!' % fetch_target)
# Run "gclient sync"
revisions = [('src', chrome_revision)]
if skia_revision != SKIA_REV_DEPS:
revisions.append(('src/third_party/skia', skia_revision))
try:
# Hack: We have to set some GYP_DEFINES, or upstream scripts will complain.
os.environ['GYP_DEFINES'] = os.environ.get('GYP_DEFINES') or ''
gclient_utils.Sync(revisions=revisions,
jobs=1,
no_hooks=True,
force=True)
except shell_utils.CommandFailedException as e:
# We frequently see sync failures because a lock file wasn't deleted. In
# that case, delete the lock file and try again.
pattern = r".*fatal: Unable to create '(\S+)': File exists\..*"
match = re.search(pattern, e.output)
if not match:
raise e
file_to_delete = match.groups()[0]
try:
print 'Attempting to remove %s' % file_to_delete
os.remove(file_to_delete)
except OSError:
# If the file no longer exists, just try again.
pass
gclient_utils.Sync(revisions=revisions,
jobs=1,
no_hooks=True,
force=True)
# Find the actually-obtained Chrome revision.
os.chdir('src')
actual_chrome_rev = shell_utils.run([GIT, 'rev-parse', 'HEAD'],
log_in_real_time=False).rstrip()
# Find the actually-obtained Skia revision.
with misc.ChDir(os.path.join('third_party', 'skia')):
actual_skia_rev = shell_utils.run([GIT, 'rev-parse', 'HEAD'],
log_in_real_time=False).rstrip()
# Run gclient hooks
gclient_utils.RunHooks(gyp_defines=gyp_defines,
gyp_generators=gyp_generators)
# Fix the submodules so that they don't show up in "git status"
# This fails on Windows...
if os.name != 'nt':
submodule_cmd = ('\'git config -f '
'$toplevel/.git/config submodule.$name.ignore all\'')
shell_utils.run(' '.join([GIT, 'submodule', 'foreach', submodule_cmd]),
shell=True)
# Verify that we got the requested revisions of Chrome and Skia.
if skia_revision != actual_skia_rev and skia_revision != SKIA_REV_DEPS:
raise Exception('Requested Skia revision %s but got %s!' %
(repr(skia_revision), repr(actual_skia_rev)))
if chrome_revision and chrome_revision != actual_chrome_rev:
raise Exception('Requested Chrome revision %s but got %s!' %
(repr(chrome_revision), repr(actual_chrome_rev)))
return (actual_skia_rev, actual_chrome_rev)
def asDict(self, request):
return {
'cq_required_steps':
skia_vars.GetGlobalVariable('cq_required_steps')
}
def __init__(self, attempts=3, **kwargs):
super(UploadRenderedSKPs, self).__init__(
attempts=attempts, **kwargs)
self._gs_images_bucket = skia_vars.GetGlobalVariable('skp_images_bucket')
self._gs_summaries_bucket = skia_vars.GetGlobalVariable(
'skp_summaries_bucket')
def Update(self, c):
for builder in self._builders:
# Update the schedulers with the builder.
schedulers = builder['schedulers']
if schedulers:
for scheduler in schedulers:
self._schedulers[scheduler]['builders'].append(builder['name'])
# Construct the category.
categories = []
if builder.get('category', None):
categories.append(builder['category'])
if builder.get('gatekeeper', None):
categories.extend(builder['gatekeeper'].split('|'))
category = '|'.join(categories)
# Append the builder to the list.
new_builder = {'name': builder['name'],
'factory': self._factories[builder['factory']],
'category': category,
'auto_reboot': builder['auto_reboot']}
if builder['builddir']:
new_builder['builddir'] = builder['builddir']
c['builders'].append(new_builder)
c['builders'].sort(key=lambda builder: builder['name'])
# Process the main schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Scheduler':
def filter_fn(change, builders):
"""Filters out if change.comments contains certain keywords.
The change is filtered out if the commit message contains:
* SKIP_BUILDBOT_SUBSTRING or
* RUN_BUILDERS_REGEX when the scheduler does not contain any of the
specified builders
Args:
change: An instance of changes.Change.
builders: Sequence of strings. The builders that are run by this
scheduler.
Returns:
If the change should be filtered out (i.e. not run by the buildbot
code) then False is returned else True is returned.
"""
if SKIP_BUILDBOT_SUBSTRING in change.comments:
return False
match_obj = RUN_BUILDERS_RE_COMPILED.search(change.comments)
if builders and match_obj:
for builder_to_run in match_obj.group(1).split(','):
if builder_to_run.strip() in builders:
break
else:
return False
return True
skia_change_filter = SkiaChangeFilter(
builders=scheduler['builders'],
branch=skia_vars.GetGlobalVariable('master_branch_name'),
filter_fn=filter_fn)
instance = Scheduler(name=s_name,
treeStableTimer=scheduler['treeStableTimer'],
builderNames=scheduler['builders'],
change_filter=skia_change_filter)
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the periodic schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'PeriodicScheduler':
instance = timed.Nightly(
name=s_name,
branch=skia_vars.GetGlobalVariable('master_branch_name'),
builderNames=scheduler['builders'],
minute=scheduler['minute'],
hour=scheduler['hour'],
dayOfMonth=scheduler['dayOfMonth'],
month=scheduler['month'],
dayOfWeek=scheduler['dayOfWeek'])
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the Rietveld-based try schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'TryJobRietveld':
pools = BuildersPools(s_name)
pools[s_name].extend(scheduler['builders'])
instance = try_job_rietveld.TryJobRietveld(
name=s_name,
pools=pools,
last_good_urls={'skia': None},
code_review_sites={'skia': config_private.CODE_REVIEW_SITE},
project='skia')
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the svn-based try schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'TryJobSubversion':
pools = BuildersPools(s_name)
pools[s_name].extend(scheduler['builders'])
instance = try_job_svn.TryJobSubversion(
name=s_name,
svn_url=TRY_SVN_BASEURL,
last_good_urls={'skia': None},
code_review_sites={'skia': config_private.CODE_REVIEW_SITE},
pools=pools)
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Process the dependent schedulers.
for s_name in self._schedulers:
scheduler = self._schedulers[s_name]
if scheduler['type'] == 'Dependent':
instance = Dependent(
s_name,
self._schedulers[scheduler['parent']]['instance'],
scheduler['builders'])
c['schedulers'].append(instance)
self._schedulers[s_name]['instance'] = instance
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Merge Skia into Android."""
import os
import sys
from build_step import BuildStep, BuildStepFailure, BuildStepWarning
import skia_vars
from sync_android import ANDROID_CHECKOUT_PATH, REPO, GitAuthenticate
from py.utils.git_utils import GIT
from py.utils import git_utils
from py.utils import misc
from py.utils import shell_utils
SKIA_REPO_URL = skia_vars.GetGlobalVariable('skia_git_url')
SKIA_REV_URL = skia_vars.GetGlobalVariable('revlink_tmpl')
MASTER_SKIA_URL = ('https://googleplex-android-review.googlesource.com/'
'platform/external/skia')
MASTER_SKIA_REFS = 'HEAD:refs/heads/master-skia'
UPSTREAM_REMOTE_NAME = 'upstream'
ANDROID_USER_CONFIG = 'include/core/SkUserConfig.h'
UPSTREAM_USER_CONFIG = 'include/config/SkUserConfig.h'
EXTERNAL_SKIA = os.path.join(ANDROID_CHECKOUT_PATH, 'external', 'skia')
# Path to gyp_to_android.py.
PLATFORM_TOOLS_BIN = os.path.join(EXTERNAL_SKIA, 'platform_tools', 'android',
'bin')
