def upgrade(self, node):
nso = NodeSaltOps(node)
wait_for_rejoin = False
''' Prep '''
_LOG.info('Verifying I can ping node=%s through Salt', node)
assert nso.ping()
''' Highstate '''
_LOG.info('Blazing it up (lighting a highstate) on node=%s', node)
ret = nso.cmd('state.highstate', quiet=True)
# Check for changes in the elasticsearch service from highstate run
svc_changes = ret['service_|-elasticsearch_|-elasticsearch_|-running']['changes']
if svc_changes:
wait_for_rejoin = True
_LOG.info('Salt elasticsearch service changes: %s', svc_changes)
else:
_LOG.info('Salt reported that no changes were performed on the elasticsearch service.')
''' HACK Work around broken pkg.latest in Salt '''
upgradable = nso.cmd('pkg.available_version', ['elasticsearch'])
if upgradable:
_LOG.info('Working around broken pkg.latest in Salt')
# We force a stop here because elasticsearch upgrades can make
# service stop no longer work, leaving a zombie ES process that
# sysvinit cannot control
assert nso.ensure_elasticsearch_is_dead()
wait_for_rejoin = True
ret = nso.cmd('pkg.install', ['elasticsearch'])
if ret.get('elasticsearch'):
wait_for_rejoin = True
''' Wait for node to rejoin (if applicable) '''
if wait_for_rejoin:
_LOG.info('Waiting for node=%s to rejoin', node)
if not nso.service_status('elasticsearch'):
assert nso.service_start('elasticsearch')
time.sleep(15)
assert nso.wait_for_service_status('elasticsearch', True)
self.wait_until_node_joins(node.name, uptime_less_than=node.uptime.total_seconds())
def restart(self, node):
_LOG.info('Found node with heap above threshold=%d: %s', heap_used_percent_threshold, node)
nso = NodeSaltOps(node)
''' Prep '''
_LOG.info('Verifying I can ping node=%s through Salt', node)
assert nso.ping()
''' Shutdown '''
assert nso.ensure_elasticsearch_is_dead()
''' Start '''
assert nso.service_start('elasticsearch')
time.sleep(15)
assert nso.wait_for_service_status('elasticsearch', True)
''' Wait until node joins '''
self.wait_until_node_joins(node.name, uptime_less_than=node.uptime.total_seconds())
