def testNoTransitionTime(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state(self._stateConfig([ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline'}, ]))
def testTransitionTimeInvalid(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state({ 'master.chromium.fyi': [ {'desired_state': 'running', 'transition_time_utc': 'boats'}, {'desired_state': 'offline', 'transition_time_utc': 'llama'}, ]})
def testUncertainPresent(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state({ 'master.chromium.fyi': [ {'desired_state': 'running', 'transition_time_utc': 6000}, {'desired_state': 'offline', 'transition_time_utc': 8000}, ]})
def testValidStateZulu(self): desired_state_parser.validate_desired_master_state(self._stateConfig([ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000}, ]))
def run(masters, restart_time, reviewers, bug, force, no_commit, desired_state): """Restart all the masters in the list of masters. Schedules the restart for restart_time. Args: masters - a list(str) of masters to restart restart_time - a datetime in UTC of when to restart them. If None, restart at a predefined "end of day". reviewers - a list(str) of reviewers for the CL (may be empty) bug - an integer bug number to include in the review or None force - a bool which causes commit not to prompt if true no_commit - doesn't set the CQ bit on upload desired_state - nominally 'running', picks which desired_state to put the buildbot in """ masters = [get_restart_spec(m, restart_time) for m in sorted(set(masters))] # Step 1: Acquire a clean master state checkout. # This repo is too small to consider caching. with get_master_state_checkout() as master_state_dir: master_state_json = os.path.join( master_state_dir, 'desired_master_state.json') # Step 2: make modifications to the master state json. LOGGER.info('Reading %s' % master_state_json) with open(master_state_json, 'r') as f: desired_master_state = json.load(f) LOGGER.info('Loaded') # Validate the current master state file. try: desired_state_parser.validate_desired_master_state(desired_master_state) except desired_state_parser.InvalidDesiredMasterState: LOGGER.exception("Failed to validate current master state JSON.") return 1 master_states = desired_master_state.get('master_states', {}) entries = 0 for master in masters: if master.desired_state_name not in master_states: msg = '%s not found in master state' % master.desired_state_name LOGGER.error(msg) raise MasterNotFoundException(msg) master_states.setdefault(master.desired_state_name, []).append({ 'desired_state': desired_state, 'transition_time_utc': zulu.to_zulu_string(master.restart_time), }) entries += 1 LOGGER.info('Writing back to JSON file, %d new entries' % (entries,)) desired_state_parser.write_master_state( desired_master_state, master_state_json) # Step 3: Send the patch to Rietveld and commit it via the CQ. LOGGER.info('Committing back into repository') commit(master_state_dir, masters, reviewers, bug, force, no_commit, desired_state)
def testInvalidState(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state({ 'master.chromium.fyi': [ {'desired_state': 'pajamas', 'transition_time_utc': 4000}, {'desired_state': 'offline', 'transition_time_utc': 6000}, ]})
def testValidStateZulu(self): desired_state_parser.validate_desired_master_state({ 'master.chromium.fyi': [ {'desired_state': 'running', 'transition_time_utc': 4000}, {'desired_state': 'offline', 'transition_time_utc': '1970-01-01T01:40:00Z'}, # Unix timestamp 6000 ]})
def testNotSortedZulu(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state({ 'master.chromium.fyi': [ {'desired_state': 'offline', 'transition_time_utc': '1970-01-01T01:40:00Z'}, # Unix timestamp 6000 {'desired_state': 'running', 'transition_time_utc': 4000}, ]})
def testIllegallyManaged(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state(self._stateConfig([ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000}, ], manually_managed='*****@*****.**', ))
def testUnknownKeyPresent(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state(self._stateConfig( [ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000}, ], unknown_key=1337, ))
def testNonNumericDrainTimeout(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state(self._stateConfig( [ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000}, ], drain_timeout_sec='abc', ))
def testNoTransitionTime(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state( self._stateConfig([ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline' }, ]))
def testValidStateZulu(self): desired_state_parser.validate_desired_master_state( self._stateConfig([ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000 }, ]))
def testTransitionTimeInvalid(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state( self._stateConfig([ { 'desired_state': 'running', 'transition_time_utc': 'boats' }, { 'desired_state': 'offline', 'transition_time_utc': 'llama' }, ]))
def testInvalidBuilderFilter(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state(self._stateConfig( [ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000}, ], builder_filters=[ r'+invalid-regex+', ], ))
def testValidState(self): desired_state_parser.validate_desired_master_state(self._stateConfig( [ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000}, ], drain_timeout_sec=1300, builder_filters=[ r'^valid$', ], ))
def testDifferentVersion(self): # Confirm that the configuration loads. c = self._stateConfig([ {'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000}, {'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000}, ]) desired_state_parser.validate_desired_master_state(c) # Modify the version to invalidate it. c['version'] = 'test' with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state(c)
def testIllegallyManaged(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state( self._stateConfig( [ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000 }, ], manually_managed='*****@*****.**', ))
def testNonNumericDrainTimeout(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state( self._stateConfig( [ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000 }, ], drain_timeout_sec='abc', ))
def testUnknownKeyPresent(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state( self._stateConfig( [ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000 }, ], unknown_key=1337, ))
def testValidState(self): desired_state_parser.validate_desired_master_state( self._stateConfig( [ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000 }, ], drain_timeout_sec=1300, builder_filters=[ r'^valid$', ], ))
def testDifferentVersion(self): # Confirm that the configuration loads. c = self._stateConfig([ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000 }, ]) desired_state_parser.validate_desired_master_state(c) # Modify the version to invalidate it. c['version'] = 'test' with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state(c)
def testInvalidBuilderFilter(self): with self.assertRaises(desired_state_parser.InvalidDesiredMasterState): desired_state_parser.validate_desired_master_state( self._stateConfig( [ { 'desired_state': 'running', 'transition_time_utc': UNIX_TIMESTAMP_4000 }, { 'desired_state': 'offline', 'transition_time_utc': UNIX_TIMESTAMP_6000 }, ], builder_filters=[ r'+invalid-regex+', ], ))
def run(masters, masters_regex, restart_time, rolling, reviewers, bug, force, no_commit, desired_state, reason): """Schedules a restart of each master in <masters> and <masters_regex>. Args: masters - a list(str) of masters to restart masters_regex - a regex string of master names to restart restart_time - a datetime in UTC of when to restart them. If None, restart at a predefined "end of day". rolling - delay (in mins) to add between restart time of each master reviewers - a list(str) of reviewers for the CL (may be empty) bug - an integer bug number to include in the review or None force - a bool which causes commit not to prompt if true no_commit - doesn't set the CQ bit on upload desired_state - nominally 'running', picks which desired_state to put the buildbot in reason - a short message saying why the master is being restarted """ # Step 1: Acquire a clean master state checkout. # This repo is too small to consider caching. with get_master_state_checkout() as master_state_dir: # Step 1.5: make sure email of committer is @google.com. if not _configure_git_name_and_email(master_state_dir): return 0 # Step 2: Modify the master state json file. master_state_json = os.path.join(master_state_dir, 'desired_master_state.json') LOGGER.info('Reading %s' % master_state_json) with open(master_state_json, 'r') as f: desired_master_state = json.load(f) LOGGER.info('Loaded') # Validate the current master state file. try: desired_state_parser.validate_desired_master_state( desired_master_state) except desired_state_parser.InvalidDesiredMasterState: LOGGER.exception('Failed to validate current master state JSON.') return 1 if masters_regex: masters.extend( get_master_names(desired_master_state, name_regex=masters_regex)) masters = sorted(set(masters)) if rolling: rolling_offsets = range(0, len(masters) * rolling, rolling) rolling_restarts = [ restart_time + datetime.timedelta(minutes=offset) for offset in rolling_offsets ] masters = [ get_restart_spec(master, time) for master, time in zip(masters, rolling_restarts) ] else: masters = [get_restart_spec(m, restart_time) for m in masters] # <masters> is now a list of <RestartSpec>s reason = reason.strip() if not reason: default_reason = '' if bug: default_reason = 'Restart for https://crbug.com/%s' % bug prompt = 'Please provide a reason for this restart' if default_reason: prompt += ' [%s]: ' % default_reason else: prompt += ': ' reason = raw_input(prompt).strip() if not reason: if default_reason: reason = default_reason else: print 'No reason provided, exiting' return 0 # Update desired_master_state according to list of RestartSpecs in <masters> master_states = desired_master_state.get('master_states', {}) entries = 0 for master in masters: if master.desired_state_name not in master_states: msg = '%s not found in master state' % master.desired_state_name LOGGER.error(msg) raise MasterNotFoundException(msg) master_states.setdefault(master.desired_state_name, []).append({ 'desired_state': desired_state, 'transition_time_utc': zulu.to_zulu_string(master.restart_time), }) entries += 1 LOGGER.info('Writing back to JSON file, %d new entries' % (entries, )) desired_state_parser.write_master_state( desired_master_state, master_state_json, prune_only_masters=set(m.desired_state_name for m in masters)) # Step 3: Send the patch to Rietveld and commit it via the CQ. LOGGER.info('Committing back into repository') commit(master_state_dir, masters, reviewers, bug, force, no_commit, desired_state, reason)
def testValidState(self): desired_state_parser.validate_desired_master_state({ 'master.chromium.fyi': [ {'desired_state': 'running', 'transition_time_utc': 4000}, {'desired_state': 'offline', 'transition_time_utc': 6000}, ]})