def _make_logger(event_logfile=None, ts_mon_logfile=None,
logging_ignore_basedir=None, event_logging=None):
"""Create a status_logger and delete any temp files when done."""
tempdir = tempfile.mkdtemp(prefix='status-logger')
logger = status_logger.StatusEventLogger(
basedir=tempdir, event_logging_dir=tempdir,
event_logfile=event_logfile, ts_mon_logfile=ts_mon_logfile,
logging_ignore_basedir=logging_ignore_basedir,
event_logging=event_logging)
logger._create_event_logger()
logger._create_ts_mon_logger()
yield logger
if os.path.exists(tempdir):
shutil.rmtree(tempdir)
def _make_logger(file_content=None):
"""Create a status_logger and delete any temp files when done."""
tempdir = tempfile.mkdtemp(prefix='status-logger')
filename = 'logstatus'
if file_content is None:
filename = 'does_not_exist'
else:
with open(os.path.join(tempdir, filename), 'w') as f:
if not file_content:
f.write('\n')
else:
f.write(json.dumps(file_content))
logger = status_logger.StatusEventLogger(configfile=filename,
basedir=tempdir,
event_logging_dir=tempdir)
logger._create_logger()
logger._create_event_logger()
logger._create_ts_mon_logger()
yield logger
if os.path.exists(tempdir):
shutil.rmtree(tempdir)
def AutoSetupMaster(c,
active_master,
mail_notifier=False,
mail_notifier_mode=None,
public_html=None,
templates=None,
order_console_by_time=False,
tagComparator=None,
customEndpoints=None,
enable_http_status_push=False,
console_repo_filter=None,
console_builder_filter=None,
web_template_globals=None):
"""Add common settings and status services to a master.
If you wonder what all these mean, PLEASE go check the official doc!
http://buildbot.net/buildbot/docs/0.7.12/ or
http://buildbot.net/buildbot/docs/latest/full.html
- Default number of logs to keep
- WebStatus and MailNotifier
- Debug ssh port. Just add a file named .manhole beside master.cfg and
simply include one line containing 'port = 10101', then you can
'ssh localhost -p' and you can access your buildbot from the inside."""
if active_master.in_production and not active_master.is_production_host:
log.err('ERROR: Trying to start the master on the wrong host.')
log.err('ERROR: This machine is %s, expected %s.' %
(active_master.current_host, active_master.master_host))
raise WrongHostException
c['slavePortnum'] = active_master.slave_port
c['projectName'] = active_master.project_name
c['projectURL'] = config.Master.project_url
c['properties'] = {'mastername': GetMastername()}
if 'buildbotURL' in c:
c['properties']['buildbotURL'] = c['buildbotURL']
# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
c.setdefault('status', [])
if mail_notifier:
# pylint: disable=E1101
c['status'].append(
mail.MailNotifier(fromaddr=active_master.from_address,
mode=mail_notifier_mode or 'problem',
relayhost=config.Master.smtp,
lookup=FilterDomain()))
# Add in the pubsub pusher, which pushes all status updates to a pubsub
# topic. This will not run unless is_production_host is set to True.
# This will fail on a production host if it cannot find the service
# account file.
pubsub_pusher = pubsub_json_status_push.StatusPush.CreateStatusPush(
activeMaster=active_master)
if pubsub_pusher:
c['status'].append(pubsub_pusher)
else:
log.msg('Pubsub not enabled.')
# For all production masters, notify our health-monitoring webapp.
if enable_http_status_push:
blacklist = (
'buildETAUpdate',
#'buildFinished',
'buildStarted',
'buildedRemoved',
'builderAdded',
'builderChangedState',
'buildsetSubmitted',
'changeAdded',
'logFinished',
'logStarted',
'requestCancelled',
'requestSubmitted',
'slaveConnected',
'slaveDisconnected',
'stepETAUpdate',
'stepFinished',
'stepStarted',
'stepText2Changed',
'stepTextChanged',
)
c['status'].append(
HttpStatusPush(
'https://chromium-build-logs.appspot.com/status_receiver',
blackList=blacklist))
# Enable Chrome Build Extract status push if configured. This requires the
# configuration file to be defined and valid for this master.
status_push = None
try:
status_push = cbe_json_status_push.StatusPush.load(
active_master,
pushInterval=30, # Push every 30 seconds.
)
except cbe_json_status_push.ConfigError as e:
log.err(
None, 'Failed to load configuration; not installing CBE status '
'push: %s' % (e.message, ))
if status_push:
# A status push configuration was identified.
c['status'].append(status_push)
kwargs = {}
if public_html:
kwargs['public_html'] = public_html
kwargs['order_console_by_time'] = order_console_by_time
# In Buildbot 0.8.4p1, pass provide_feeds as a list to signal what extra
# services Buildbot should be able to provide over HTTP.
if buildbot.version == '0.8.4p1':
kwargs['provide_feeds'] = ['json']
if active_master.master_port:
# Actions we want to allow must be explicitly listed here.
# Deliberately omitted are:
# - gracefulShutdown
# - cleanShutdown
authz = Authz(forceBuild=True,
forceAllBuilds=True,
pingBuilder=True,
stopBuild=True,
stopAllBuilds=True,
cancelPendingBuild=True)
c['status'].append(
CreateWebStatus(active_master.master_port,
tagComparator=tagComparator,
customEndpoints=customEndpoints,
authz=authz,
num_events_max=3000,
templates=templates,
console_repo_filter=console_repo_filter,
console_builder_filter=console_builder_filter,
web_template_globals=web_template_globals,
**kwargs))
if active_master.master_port_alt:
c['status'].append(
CreateWebStatus(active_master.master_port_alt,
tagComparator=tagComparator,
customEndpoints=customEndpoints,
num_events_max=3000,
templates=templates,
console_repo_filter=console_repo_filter,
console_builder_filter=console_builder_filter,
web_template_globals=web_template_globals,
**kwargs))
# Add a status logger and a ts_mon flushing receiver.
c['status'].append(status_logger.StatusEventLogger())
c['status'].append(monitoring_status_receiver.MonitoringStatusReceiver())
# Keep last build logs, the default is too low.
c['buildHorizon'] = 1000
c['logHorizon'] = 500
# Must be at least 2x the number of slaves.
c['eventHorizon'] = 200
# Tune cache sizes to speed up web UI.
c['caches'] = {
'BuildRequests': 1000,
'Changes': 1000,
'SourceStamps': 1000,
'chdicts': 1000,
'ssdicts': 1000,
}
# Must be at least 2x the number of on-going builds.
c['buildCacheSize'] = 200
# See http://buildbot.net/buildbot/docs/0.8.1/Debug-Options.html for more
# details.
if os.path.isfile('.manhole'):
try:
from buildbot import manhole
except ImportError:
log.msg(
'Using manhole has an implicit dependency on Crypto.Cipher. You '
'need to install it manually:\n'
' sudo apt-get install python-crypto\n'
'on ubuntu or run:\n'
' pip install --user pycrypto\n'
' pip install --user pyasn1\n')
raise
# If 'port' is defined, it uses the same valid keys as the current user.
values = {}
execfile('.manhole', values)
if 'debugPassword' in values:
c['debugPassword'] = values['debugPassword']
interface = 'tcp:%s:interface=127.0.0.1' % values.get('port', 0)
if 'port' in values and 'user' in values and 'password' in values:
c['manhole'] = manhole.PasswordManhole(interface, values['user'],
values['password'])
elif 'port' in values:
c['manhole'] = manhole.AuthorizedKeysManhole(
interface, os.path.expanduser("~/.ssh/authorized_keys"))
if active_master.buildbucket_bucket and active_master.service_account_path:
SetupBuildbucket(c, active_master)
SetMasterProcessName()
def SetupMaster(ActiveMaster):
# Buildmaster config dict.
c = {}
config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)
####### CHANGESOURCES
# Polls config.Master.trunk_url for changes
poller = GitilesPoller(
repo_url=ActiveMaster.repo_url,
branches=[POLLING_BRANCH],
pollInterval=10,
revlinktmpl='https://skia.googlesource.com/skia/+/%s')
infra_poller = GitilesPoller(
repo_url=global_constants.INFRA_REPO,
branches=[POLLING_BRANCH],
pollInterval=10,
revlinktmpl='https://skia.googlesource.com/buildbot/+/%s')
c['change_source'] = [poller, infra_poller]
####### SLAVES
# Load the slave list. We need some information from it in order to
# produce the builders.
slaves = slaves_list.SlavesList('slaves.cfg', ActiveMaster.project_name)
####### BUILDERS
# Load the builders list.
builders = chromium_utils.ParsePythonCfg('builders.cfg')['builders']
# Configure the Builders and Schedulers.
SetupBuildersAndSchedulers(c=c,
builders=builders,
slaves=slaves,
ActiveMaster=ActiveMaster)
####### BUILDSLAVES
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
config.Master.GetBotPassword())
master_utils.VerifySetup(c, slaves)
####### STATUS TARGETS
c['buildbotURL'] = ActiveMaster.buildbot_url
# Adds common status and tools to this master.
master_utils.AutoSetupMaster(
c,
ActiveMaster,
public_html='../../../build/masters/master.client.skia/public_html',
templates=[
'../../../build/masters/master.client.skia/templates',
'../../../build/masters/master.chromium/templates'
],
tagComparator=poller.comparator,
enable_http_status_push=ActiveMaster.is_production_host,
order_console_by_time=True,
console_repo_filter=ActiveMaster.repo_url,
console_builder_filter=lambda b: not builder_name_schema.IsTrybot(b))
with status_json.JsonStatusHelper() as json_helper:
json_helper.putChild('trybots', status_json.TryBuildersJsonResource)
if (ActiveMaster.is_production_host
and ActiveMaster.project_name != 'SkiaInternal'):
# Build result emails.
c['status'].append(status_logger.StatusEventLogger())
c['status'].append(
skia_notifier.SkiaMailNotifier(
fromaddr=ActiveMaster.from_address,
mode='change',
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
# Try job result emails.
c['status'].append(
skia_notifier.SkiaTryMailNotifier(
fromaddr=ActiveMaster.from_address,
subject="try %(result)s for %(reason)s @ r%(revision)s",
mode='all',
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
c['mergeRequests'] = CanMergeBuildRequests
return c