def run(masters, restart_time, reviewers, bug, force, no_commit,
desired_state):
"""Restart all the masters in the list of masters.
Schedules the restart for restart_time.
Args:
masters - a list(str) of masters to restart
restart_time - a datetime in UTC of when to restart them. If None, restart
at a predefined "end of day".
reviewers - a list(str) of reviewers for the CL (may be empty)
bug - an integer bug number to include in the review or None
force - a bool which causes commit not to prompt if true
no_commit - doesn't set the CQ bit on upload
desired_state - nominally 'running', picks which desired_state
to put the buildbot in
"""
masters = [get_restart_spec(m, restart_time) for m in sorted(set(masters))]
# Step 1: Acquire a clean master state checkout.
# This repo is too small to consider caching.
with get_master_state_checkout() as master_state_dir:
master_state_json = os.path.join(
master_state_dir, 'desired_master_state.json')
# Step 2: make modifications to the master state json.
LOGGER.info('Reading %s' % master_state_json)
with open(master_state_json, 'r') as f:
desired_master_state = json.load(f)
LOGGER.info('Loaded')
# Validate the current master state file.
try:
desired_state_parser.validate_desired_master_state(desired_master_state)
except desired_state_parser.InvalidDesiredMasterState:
LOGGER.exception("Failed to validate current master state JSON.")
return 1
master_states = desired_master_state.get('master_states', {})
entries = 0
for master in masters:
if master.desired_state_name not in master_states:
msg = '%s not found in master state' % master.desired_state_name
LOGGER.error(msg)
raise MasterNotFoundException(msg)
master_states.setdefault(master.desired_state_name, []).append({
'desired_state': desired_state,
'transition_time_utc': zulu.to_zulu_string(master.restart_time),
})
entries += 1
LOGGER.info('Writing back to JSON file, %d new entries' % (entries,))
desired_state_parser.write_master_state(
desired_master_state, master_state_json)
# Step 3: Send the patch to Rietveld and commit it via the CQ.
LOGGER.info('Committing back into repository')
commit(master_state_dir, masters, reviewers, bug, force, no_commit,
desired_state)
def testInvalidState(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
with temporary_directory() as dirname:
filename = os.path.join(dirname, 'desired_state.json')
desired_state_parser.write_master_state(
{
'master_states': {
'master.chromium.fyi': [
{
'desired_state': 'running',
'transition_time_utc': 'toast'
},
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_7000
},
]
},
'master_params': {},
'version': desired_state_parser.VERSION,
}, filename)
def testNothing(self):
with temporary_directory() as dirname:
filename = os.path.join(dirname, 'desired_state.json')
desired_state_parser.write_master_state({}, filename)
with open(filename) as f:
parsed_data = json.load(f)
self.assertEqual(parsed_data, {
'master_states': {},
'master_params': {},
'version': desired_state_parser.PREV_VERSION,
})
def testNothing(self):
with temporary_directory() as dirname:
filename = os.path.join(dirname, 'desired_state.json')
desired_state_parser.write_master_state({}, filename)
with open(filename) as f:
parsed_data = json.load(f)
self.assertEqual(
parsed_data, {
'master_states': {},
'master_params': {},
'version': desired_state_parser.VERSION,
})
def testNothingInPast(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
with temporary_directory() as dirname:
filename = os.path.join(dirname, 'desired_state.json')
desired_state_parser.write_master_state({
'master_states': {
'master.chromium.fyi': [
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_7000},
]},
'master_params': {},
'version': desired_state_parser.PREV_VERSION,
}, filename)
def testPruneOldEntriesFilter(self):
with temporary_directory() as dirname:
filename = os.path.join(dirname, 'desired_state.json')
master_state = [
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_0500
},
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_1000
},
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_7000
},
]
# No pruning should happen here.
desired_state_parser.write_master_state(
{
'master_states': {
'master.leave.as.is': master_state[:],
'master.pruned': master_state[:],
},
'master_params': {},
'version': desired_state_parser.VERSION,
},
filename,
prune_only_masters=set(['master.pruned']))
with open(filename) as f:
parsed_data = json.load(f)
self.assertEqual(
parsed_data['master_states']['master.leave.as.is'],
master_state)
self.assertEqual(parsed_data['master_states']['master.pruned'],
master_state[1:])
def testPruneOldEntries(self):
with temporary_directory() as dirname:
filename = os.path.join(dirname, 'desired_state.json')
desired_state_parser.write_master_state({
'master_states': {
'master.chromium.fyi': [
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_0500},
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_1000},
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_7000},
]},
'master_params': {},
'version': desired_state_parser.PREV_VERSION,
}, filename)
with open(filename) as f:
parsed_data = json.load(f)
self.assertEqual(parsed_data, {
'master_states': {
'master.chromium.fyi': [
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_1000},
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_7000},
]},
'master_params': {},
'version': desired_state_parser.PREV_VERSION,
}, filename)
def run(masters, masters_regex, restart_time, rolling, reviewers, bug, force,
no_commit, desired_state, reason):
"""Schedules a restart of each master in <masters> and <masters_regex>.
Args:
masters - a list(str) of masters to restart
masters_regex - a regex string of master names to restart
restart_time - a datetime in UTC of when to restart them. If None, restart
at a predefined "end of day".
rolling - delay (in mins) to add between restart time of each master
reviewers - a list(str) of reviewers for the CL (may be empty)
bug - an integer bug number to include in the review or None
force - a bool which causes commit not to prompt if true
no_commit - doesn't set the CQ bit on upload
desired_state - nominally 'running', picks which desired_state
to put the buildbot in
reason - a short message saying why the master is being restarted
"""
# Step 1: Acquire a clean master state checkout.
# This repo is too small to consider caching.
with get_master_state_checkout() as master_state_dir:
# Step 1.5: make sure email of committer is @google.com.
if not _configure_git_name_and_email(master_state_dir):
return 0
# Step 2: Modify the master state json file.
master_state_json = os.path.join(master_state_dir,
'desired_master_state.json')
LOGGER.info('Reading %s' % master_state_json)
with open(master_state_json, 'r') as f:
desired_master_state = json.load(f)
LOGGER.info('Loaded')
# Validate the current master state file.
try:
desired_state_parser.validate_desired_master_state(
desired_master_state)
except desired_state_parser.InvalidDesiredMasterState:
LOGGER.exception('Failed to validate current master state JSON.')
return 1
if masters_regex:
masters.extend(
get_master_names(desired_master_state,
name_regex=masters_regex))
masters = sorted(set(masters))
if rolling:
rolling_offsets = range(0, len(masters) * rolling, rolling)
rolling_restarts = [
restart_time + datetime.timedelta(minutes=offset)
for offset in rolling_offsets
]
masters = [
get_restart_spec(master, time)
for master, time in zip(masters, rolling_restarts)
]
else:
masters = [get_restart_spec(m, restart_time) for m in masters]
# <masters> is now a list of <RestartSpec>s
reason = reason.strip()
if not reason:
default_reason = ''
if bug:
default_reason = 'Restart for https://crbug.com/%s' % bug
prompt = 'Please provide a reason for this restart'
if default_reason:
prompt += ' [%s]: ' % default_reason
else:
prompt += ': '
reason = raw_input(prompt).strip()
if not reason:
if default_reason:
reason = default_reason
else:
print 'No reason provided, exiting'
return 0
# Update desired_master_state according to list of RestartSpecs in <masters>
master_states = desired_master_state.get('master_states', {})
entries = 0
for master in masters:
if master.desired_state_name not in master_states:
msg = '%s not found in master state' % master.desired_state_name
LOGGER.error(msg)
raise MasterNotFoundException(msg)
master_states.setdefault(master.desired_state_name, []).append({
'desired_state':
desired_state,
'transition_time_utc':
zulu.to_zulu_string(master.restart_time),
})
entries += 1
LOGGER.info('Writing back to JSON file, %d new entries' % (entries, ))
desired_state_parser.write_master_state(
desired_master_state,
master_state_json,
prune_only_masters=set(m.desired_state_name for m in masters))
# Step 3: Send the patch to Rietveld and commit it via the CQ.
LOGGER.info('Committing back into repository')
commit(master_state_dir, masters, reviewers, bug, force, no_commit,
desired_state, reason)
def testPruneOldEntries(self):
with temporary_directory() as dirname:
filename = os.path.join(dirname, 'desired_state.json')
desired_state_parser.write_master_state(
{
'master_states': {
'master.chromium.fyi': [
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_0500
},
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_1000
},
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_7000
},
]
},
'master_params': {},
'version': desired_state_parser.VERSION,
}, filename)
with open(filename) as f:
parsed_data = json.load(f)
self.assertEqual(
parsed_data, {
'master_states': {
'master.chromium.fyi': [
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_1000
},
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_7000
},
]
},
'master_params': {},
'version': desired_state_parser.VERSION,
}, filename)