def upgrade(self, node): nso = NodeSaltOps(node) wait_for_rejoin = False ''' Prep ''' _LOG.info('Verifying I can ping node=%s through Salt', node) assert nso.ping() ''' Highstate ''' _LOG.info('Blazing it up (lighting a highstate) on node=%s', node) ret = nso.cmd('state.highstate', quiet=True) # Check for changes in the elasticsearch service from highstate run svc_changes = ret['service_|-elasticsearch_|-elasticsearch_|-running']['changes'] if svc_changes: wait_for_rejoin = True _LOG.info('Salt elasticsearch service changes: %s', svc_changes) else: _LOG.info('Salt reported that no changes were performed on the elasticsearch service.') ''' HACK Work around broken pkg.latest in Salt ''' upgradable = nso.cmd('pkg.available_version', ['elasticsearch']) if upgradable: _LOG.info('Working around broken pkg.latest in Salt') # We force a stop here because elasticsearch upgrades can make # service stop no longer work, leaving a zombie ES process that # sysvinit cannot control assert nso.ensure_elasticsearch_is_dead() wait_for_rejoin = True ret = nso.cmd('pkg.install', ['elasticsearch']) if ret.get('elasticsearch'): wait_for_rejoin = True ''' Wait for node to rejoin (if applicable) ''' if wait_for_rejoin: _LOG.info('Waiting for node=%s to rejoin', node) if not nso.service_status('elasticsearch'): assert nso.service_start('elasticsearch') time.sleep(15) assert nso.wait_for_service_status('elasticsearch', True) self.wait_until_node_joins(node.name, uptime_less_than=node.uptime.total_seconds())
def restart(self, node): _LOG.info('Found node with heap above threshold=%d: %s', heap_used_percent_threshold, node) nso = NodeSaltOps(node) ''' Prep ''' _LOG.info('Verifying I can ping node=%s through Salt', node) assert nso.ping() ''' Shutdown ''' assert nso.ensure_elasticsearch_is_dead() ''' Start ''' assert nso.service_start('elasticsearch') time.sleep(15) assert nso.wait_for_service_status('elasticsearch', True) ''' Wait until node joins ''' self.wait_until_node_joins(node.name, uptime_less_than=node.uptime.total_seconds())