def testNoTransitionTime(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(self._stateConfig([
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline'},
]))
def testTransitionTimeInvalid(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state({
'master.chromium.fyi': [
{'desired_state': 'running', 'transition_time_utc': 'boats'},
{'desired_state': 'offline', 'transition_time_utc': 'llama'},
]})
def testUncertainPresent(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state({
'master.chromium.fyi': [
{'desired_state': 'running', 'transition_time_utc': 6000},
{'desired_state': 'offline', 'transition_time_utc': 8000},
]})
def testValidStateZulu(self):
desired_state_parser.validate_desired_master_state(self._stateConfig([
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
]))
def run(masters, restart_time, reviewers, bug, force, no_commit,
desired_state):
"""Restart all the masters in the list of masters.
Schedules the restart for restart_time.
Args:
masters - a list(str) of masters to restart
restart_time - a datetime in UTC of when to restart them. If None, restart
at a predefined "end of day".
reviewers - a list(str) of reviewers for the CL (may be empty)
bug - an integer bug number to include in the review or None
force - a bool which causes commit not to prompt if true
no_commit - doesn't set the CQ bit on upload
desired_state - nominally 'running', picks which desired_state
to put the buildbot in
"""
masters = [get_restart_spec(m, restart_time) for m in sorted(set(masters))]
# Step 1: Acquire a clean master state checkout.
# This repo is too small to consider caching.
with get_master_state_checkout() as master_state_dir:
master_state_json = os.path.join(
master_state_dir, 'desired_master_state.json')
# Step 2: make modifications to the master state json.
LOGGER.info('Reading %s' % master_state_json)
with open(master_state_json, 'r') as f:
desired_master_state = json.load(f)
LOGGER.info('Loaded')
# Validate the current master state file.
try:
desired_state_parser.validate_desired_master_state(desired_master_state)
except desired_state_parser.InvalidDesiredMasterState:
LOGGER.exception("Failed to validate current master state JSON.")
return 1
master_states = desired_master_state.get('master_states', {})
entries = 0
for master in masters:
if master.desired_state_name not in master_states:
msg = '%s not found in master state' % master.desired_state_name
LOGGER.error(msg)
raise MasterNotFoundException(msg)
master_states.setdefault(master.desired_state_name, []).append({
'desired_state': desired_state,
'transition_time_utc': zulu.to_zulu_string(master.restart_time),
})
entries += 1
LOGGER.info('Writing back to JSON file, %d new entries' % (entries,))
desired_state_parser.write_master_state(
desired_master_state, master_state_json)
# Step 3: Send the patch to Rietveld and commit it via the CQ.
LOGGER.info('Committing back into repository')
commit(master_state_dir, masters, reviewers, bug, force, no_commit,
desired_state)
def testInvalidState(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state({
'master.chromium.fyi': [
{'desired_state': 'pajamas', 'transition_time_utc': 4000},
{'desired_state': 'offline', 'transition_time_utc': 6000},
]})
def testValidStateZulu(self):
desired_state_parser.validate_desired_master_state({
'master.chromium.fyi': [
{'desired_state': 'running',
'transition_time_utc': 4000},
{'desired_state': 'offline',
'transition_time_utc': '1970-01-01T01:40:00Z'}, # Unix timestamp 6000
]})
def testNotSortedZulu(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state({
'master.chromium.fyi': [
{'desired_state': 'offline',
'transition_time_utc': '1970-01-01T01:40:00Z'}, # Unix timestamp 6000
{'desired_state': 'running',
'transition_time_utc': 4000},
]})
def testIllegallyManaged(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(self._stateConfig([
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
],
manually_managed='*****@*****.**',
))
def testUnknownKeyPresent(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(self._stateConfig(
[
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
],
unknown_key=1337,
))
def testNonNumericDrainTimeout(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(self._stateConfig(
[
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
],
drain_timeout_sec='abc',
))
def testNoTransitionTime(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(
self._stateConfig([
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline'
},
]))
def testValidStateZulu(self):
desired_state_parser.validate_desired_master_state(
self._stateConfig([
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
]))
def testTransitionTimeInvalid(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(
self._stateConfig([
{
'desired_state': 'running',
'transition_time_utc': 'boats'
},
{
'desired_state': 'offline',
'transition_time_utc': 'llama'
},
]))
def testInvalidBuilderFilter(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(self._stateConfig(
[
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
],
builder_filters=[
r'+invalid-regex+',
],
))
def testValidState(self):
desired_state_parser.validate_desired_master_state(self._stateConfig(
[
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
],
drain_timeout_sec=1300,
builder_filters=[
r'^valid$',
],
))
def testDifferentVersion(self):
# Confirm that the configuration loads.
c = self._stateConfig([
{'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000},
{'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000},
])
desired_state_parser.validate_desired_master_state(c)
# Modify the version to invalidate it.
c['version'] = 'test'
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(c)
def testIllegallyManaged(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(
self._stateConfig(
[
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
],
manually_managed='*****@*****.**',
))
def testNonNumericDrainTimeout(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(
self._stateConfig(
[
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
],
drain_timeout_sec='abc',
))
def testUnknownKeyPresent(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(
self._stateConfig(
[
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
],
unknown_key=1337,
))
def testValidState(self):
desired_state_parser.validate_desired_master_state(
self._stateConfig(
[
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
],
drain_timeout_sec=1300,
builder_filters=[
r'^valid$',
],
))
def testDifferentVersion(self):
# Confirm that the configuration loads.
c = self._stateConfig([
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
])
desired_state_parser.validate_desired_master_state(c)
# Modify the version to invalidate it.
c['version'] = 'test'
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(c)
def testInvalidBuilderFilter(self):
with self.assertRaises(desired_state_parser.InvalidDesiredMasterState):
desired_state_parser.validate_desired_master_state(
self._stateConfig(
[
{
'desired_state': 'running',
'transition_time_utc': UNIX_TIMESTAMP_4000
},
{
'desired_state': 'offline',
'transition_time_utc': UNIX_TIMESTAMP_6000
},
],
builder_filters=[
r'+invalid-regex+',
],
))
def run(masters, masters_regex, restart_time, rolling, reviewers, bug, force,
no_commit, desired_state, reason):
"""Schedules a restart of each master in <masters> and <masters_regex>.
Args:
masters - a list(str) of masters to restart
masters_regex - a regex string of master names to restart
restart_time - a datetime in UTC of when to restart them. If None, restart
at a predefined "end of day".
rolling - delay (in mins) to add between restart time of each master
reviewers - a list(str) of reviewers for the CL (may be empty)
bug - an integer bug number to include in the review or None
force - a bool which causes commit not to prompt if true
no_commit - doesn't set the CQ bit on upload
desired_state - nominally 'running', picks which desired_state
to put the buildbot in
reason - a short message saying why the master is being restarted
"""
# Step 1: Acquire a clean master state checkout.
# This repo is too small to consider caching.
with get_master_state_checkout() as master_state_dir:
# Step 1.5: make sure email of committer is @google.com.
if not _configure_git_name_and_email(master_state_dir):
return 0
# Step 2: Modify the master state json file.
master_state_json = os.path.join(master_state_dir,
'desired_master_state.json')
LOGGER.info('Reading %s' % master_state_json)
with open(master_state_json, 'r') as f:
desired_master_state = json.load(f)
LOGGER.info('Loaded')
# Validate the current master state file.
try:
desired_state_parser.validate_desired_master_state(
desired_master_state)
except desired_state_parser.InvalidDesiredMasterState:
LOGGER.exception('Failed to validate current master state JSON.')
return 1
if masters_regex:
masters.extend(
get_master_names(desired_master_state,
name_regex=masters_regex))
masters = sorted(set(masters))
if rolling:
rolling_offsets = range(0, len(masters) * rolling, rolling)
rolling_restarts = [
restart_time + datetime.timedelta(minutes=offset)
for offset in rolling_offsets
]
masters = [
get_restart_spec(master, time)
for master, time in zip(masters, rolling_restarts)
]
else:
masters = [get_restart_spec(m, restart_time) for m in masters]
# <masters> is now a list of <RestartSpec>s
reason = reason.strip()
if not reason:
default_reason = ''
if bug:
default_reason = 'Restart for https://crbug.com/%s' % bug
prompt = 'Please provide a reason for this restart'
if default_reason:
prompt += ' [%s]: ' % default_reason
else:
prompt += ': '
reason = raw_input(prompt).strip()
if not reason:
if default_reason:
reason = default_reason
else:
print 'No reason provided, exiting'
return 0
# Update desired_master_state according to list of RestartSpecs in <masters>
master_states = desired_master_state.get('master_states', {})
entries = 0
for master in masters:
if master.desired_state_name not in master_states:
msg = '%s not found in master state' % master.desired_state_name
LOGGER.error(msg)
raise MasterNotFoundException(msg)
master_states.setdefault(master.desired_state_name, []).append({
'desired_state':
desired_state,
'transition_time_utc':
zulu.to_zulu_string(master.restart_time),
})
entries += 1
LOGGER.info('Writing back to JSON file, %d new entries' % (entries, ))
desired_state_parser.write_master_state(
desired_master_state,
master_state_json,
prune_only_masters=set(m.desired_state_name for m in masters))
# Step 3: Send the patch to Rietveld and commit it via the CQ.
LOGGER.info('Committing back into repository')
commit(master_state_dir, masters, reviewers, bug, force, no_commit,
desired_state, reason)
def testValidState(self):
desired_state_parser.validate_desired_master_state({
'master.chromium.fyi': [
{'desired_state': 'running', 'transition_time_utc': 4000},
{'desired_state': 'offline', 'transition_time_utc': 6000},
]})
