def run(masters, restart_time, reviewers, bug, force, no_commit,
desired_state):
"""Restart all the masters in the list of masters.
Schedules the restart for restart_time.
Args:
masters - a list(str) of masters to restart
restart_time - a datetime in UTC of when to restart them. If None, restart
at a predefined "end of day".
reviewers - a list(str) of reviewers for the CL (may be empty)
bug - an integer bug number to include in the review or None
force - a bool which causes commit not to prompt if true
no_commit - doesn't set the CQ bit on upload
desired_state - nominally 'running', picks which desired_state
to put the buildbot in
"""
masters = [get_restart_spec(m, restart_time) for m in sorted(set(masters))]
# Step 1: Acquire a clean master state checkout.
# This repo is too small to consider caching.
with get_master_state_checkout() as master_state_dir:
master_state_json = os.path.join(
master_state_dir, 'desired_master_state.json')
# Step 2: make modifications to the master state json.
LOGGER.info('Reading %s' % master_state_json)
with open(master_state_json, 'r') as f:
desired_master_state = json.load(f)
LOGGER.info('Loaded')
# Validate the current master state file.
try:
desired_state_parser.validate_desired_master_state(desired_master_state)
except desired_state_parser.InvalidDesiredMasterState:
LOGGER.exception("Failed to validate current master state JSON.")
return 1
master_states = desired_master_state.get('master_states', {})
entries = 0
for master in masters:
if master.desired_state_name not in master_states:
msg = '%s not found in master state' % master.desired_state_name
LOGGER.error(msg)
raise MasterNotFoundException(msg)
master_states.setdefault(master.desired_state_name, []).append({
'desired_state': desired_state,
'transition_time_utc': zulu.to_zulu_string(master.restart_time),
})
entries += 1
LOGGER.info('Writing back to JSON file, %d new entries' % (entries,))
desired_state_parser.write_master_state(
desired_master_state, master_state_json)
# Step 3: Send the patch to Rietveld and commit it via the CQ.
LOGGER.info('Committing back into repository')
commit(master_state_dir, masters, reviewers, bug, force, no_commit,
desired_state)
def testTZAwareDTZuluString(self):
# If you're confused why GMT+8 is -08:00, see
# http://askubuntu.com/questions/519550/
# why-is-the-8-timezone-called-gmt-8-in-the-filesystem
dt = datetime.datetime(2015, 06, 11, 10, 17, 26, 123,
tzinfo=timezone('Etc/GMT+8'))
timestring = '2015-06-11T18:17:26.000123Z'
self.assertEqual(zulu.to_zulu_string(dt), timestring)
def run(masters, masters_regex, restart_time, rolling, reviewers, bug, force,
no_commit, desired_state, reason):
"""Schedules a restart of each master in <masters> and <masters_regex>.
Args:
masters - a list(str) of masters to restart
masters_regex - a regex string of master names to restart
restart_time - a datetime in UTC of when to restart them. If None, restart
at a predefined "end of day".
rolling - delay (in mins) to add between restart time of each master
reviewers - a list(str) of reviewers for the CL (may be empty)
bug - an integer bug number to include in the review or None
force - a bool which causes commit not to prompt if true
no_commit - doesn't set the CQ bit on upload
desired_state - nominally 'running', picks which desired_state
to put the buildbot in
reason - a short message saying why the master is being restarted
"""
# Step 1: Acquire a clean master state checkout.
# This repo is too small to consider caching.
with get_master_state_checkout() as master_state_dir:
# Step 1.5: make sure email of committer is @google.com.
if not _configure_git_name_and_email(master_state_dir):
return 0
# Step 2: Modify the master state json file.
master_state_json = os.path.join(master_state_dir,
'desired_master_state.json')
LOGGER.info('Reading %s' % master_state_json)
with open(master_state_json, 'r') as f:
desired_master_state = json.load(f)
LOGGER.info('Loaded')
# Validate the current master state file.
try:
desired_state_parser.validate_desired_master_state(
desired_master_state)
except desired_state_parser.InvalidDesiredMasterState:
LOGGER.exception('Failed to validate current master state JSON.')
return 1
if masters_regex:
masters.extend(
get_master_names(desired_master_state,
name_regex=masters_regex))
masters = sorted(set(masters))
if rolling:
rolling_offsets = range(0, len(masters) * rolling, rolling)
rolling_restarts = [
restart_time + datetime.timedelta(minutes=offset)
for offset in rolling_offsets
]
masters = [
get_restart_spec(master, time)
for master, time in zip(masters, rolling_restarts)
]
else:
masters = [get_restart_spec(m, restart_time) for m in masters]
# <masters> is now a list of <RestartSpec>s
reason = reason.strip()
if not reason:
default_reason = ''
if bug:
default_reason = 'Restart for https://crbug.com/%s' % bug
prompt = 'Please provide a reason for this restart'
if default_reason:
prompt += ' [%s]: ' % default_reason
else:
prompt += ': '
reason = raw_input(prompt).strip()
if not reason:
if default_reason:
reason = default_reason
else:
print 'No reason provided, exiting'
return 0
# Update desired_master_state according to list of RestartSpecs in <masters>
master_states = desired_master_state.get('master_states', {})
entries = 0
for master in masters:
if master.desired_state_name not in master_states:
msg = '%s not found in master state' % master.desired_state_name
LOGGER.error(msg)
raise MasterNotFoundException(msg)
master_states.setdefault(master.desired_state_name, []).append({
'desired_state':
desired_state,
'transition_time_utc':
zulu.to_zulu_string(master.restart_time),
})
entries += 1
LOGGER.info('Writing back to JSON file, %d new entries' % (entries, ))
desired_state_parser.write_master_state(
desired_master_state,
master_state_json,
prune_only_masters=set(m.desired_state_name for m in masters))
# Step 3: Send the patch to Rietveld and commit it via the CQ.
LOGGER.info('Committing back into repository')
commit(master_state_dir, masters, reviewers, bug, force, no_commit,
desired_state, reason)
def commit(target, specs, reviewers, bug, force, no_commit, desired_state,
reason):
"""Commits the local CL via the CQ."""
if desired_state == 'running':
action = 'Restarting'
else:
action = desired_state.title() + 'ing'
desc = '%(action)s master%(plural)s %(names)s\n\n%(reason)s\n' % {
'action': action,
'plural': 's' if len(specs) > 1 else '',
'names': ', '.join([s.name for s in specs]),
'reason': reason,
}
if bug:
desc += '\nBUG=%s' % bug
tbr_whom = 'an owner'
if reviewers:
google, other = autocomplete_and_partition(reviewers)
if other:
print
print 'Error: not @google.com email(s) for reviewers found:'
print ' %s' % ('\n '.join(other))
print 'Hint: save your fingertips - use just ldap: -r <ldap>'
return 1
tbr_whom = ', '.join(google)
desc += '\nTBR=%s' % tbr_whom
subprocess.check_call(['git', 'commit', '--all', '--message', desc],
cwd=target)
print
print 'Actions for the following masters:'
max_name_len = max(len(s.name) for s in specs)
for s in specs:
delta = s.restart_time - datetime.datetime.utcnow()
restart_time_str = zulu.to_zulu_string(s.restart_time)
local_time = s.restart_time.replace(tzinfo=dateutil.tz.tzutc())
local_time = local_time.astimezone(dateutil.tz.tzlocal())
print '\t- %s %-*s in %d minutes (UTC: %s, Local: %s)' % (
action, max_name_len, s.name, delta.total_seconds() / 60,
restart_time_str, local_time)
for s in specs:
if not s.message:
continue
print
print '=== %s ===' % (s.name, )
print s.message
print
print 'This will upload a CL for master_manager.git, TBR %s, and ' % tbr_whom
if no_commit:
print 'wait for you to manually commit.'
else:
print 'commit the CL through the CQ.'
print
if not force:
if no_commit:
print 'Upload CL? (will not set CQ bit) [Y/n]:',
else:
print 'Commit? [Y/n]:',
input_string = raw_input()
if input_string != '' and not distutils.util.strtobool(input_string):
print 'Aborting.'
return
print 'To cancel, edit desired_master_state.json in %s.' % MM_REPO
print
LOGGER.info('Uploading to Rietveld and CQ.')
upload_cmd = [
'git',
'cl',
'upload',
'-m',
desc,
'-f',
]
if not reviewers:
upload_cmd.append('--tbr-owners')
if not no_commit:
upload_cmd.append('-c')
else:
LOGGER.info('CQ bit not set, please commit manually. (--no-commit)')
subprocess.check_call(upload_cmd, cwd=target)
def commit(
target, specs, reviewers, bug, force, no_commit, desired_state):
"""Commits the local CL via the CQ."""
if desired_state == 'running':
action = 'Restarting'
else:
action = desired_state.title() + 'ing'
desc = '%s master(s) %s\n' % (
action, ', '.join([s.name for s in specs]))
if bug:
desc += '\nBUG=%s' % bug
if reviewers:
desc += '\nTBR=%s' % ', '.join(reviewers)
subprocess.check_call(
['git', 'commit', '--all', '--message', desc], cwd=target)
print
print 'Actions for the following masters:'
for s in specs:
delta = s.restart_time - datetime.datetime.utcnow()
restart_time_str = zulu.to_zulu_string(s.restart_time)
print '\t- %s %s in %d minutes (%s)' % (
action, s.name, delta.total_seconds() / 60, restart_time_str)
for s in specs:
if not s.message:
continue
print
print '=== %s ===' % (s.name,)
print s.message
print
print "This will upload a CL for master_manager.git, TBR an owner, and "
if no_commit:
print "wait for you to manually commit."
else:
print "commit the CL through the CQ."
print
if not force:
if no_commit:
print 'Upload CL? (will not set CQ bit) [Y/n]:',
else:
print 'Commit? [Y/n]:',
input_string = raw_input()
if input_string != '' and not distutils.util.strtobool(input_string):
print 'Aborting.'
return
print 'To cancel, edit desired_master_state.json in %s.' % MM_REPO
print
LOGGER.info('Uploading to Rietveld and CQ.')
upload_cmd = [
'git', 'cl', 'upload',
'-m', desc,
'-t', desc, # Title becomes the message of CL. TBR and BUG must be there.
'-f',
]
if not reviewers:
upload_cmd.append('--tbr-owners')
if not no_commit:
upload_cmd.append('-c')
else:
LOGGER.info('CQ bit not set, please commit manually. (--no-commit)')
subprocess.check_call(upload_cmd, cwd=target)
def testNaiveDTZuluStringFractional(self): dt = datetime.datetime(2015, 06, 11, 23, 17, 26, 123) timestring = '2015-06-11T23:17:26.000123Z' self.assertEqual(zulu.to_zulu_string(dt), timestring)