def upgrade_master(cls, version, config=None):
'''
Upgrades Yarn Master services: RM and JHS
:param version: Version to be upgraded to
:param config: Config location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Job history server upgrade started ")
logger.info("**** Begin history server upgrade ****")
MAPRED.stopHistoryserver()
historyNode = MAPRED.getHistoryserver()
# TODO use hdp-select script to upgrade JHS
from beaver.component.rollingupgrade.ruCommon import hdpSelect
hdpSelect.changeVersion("hadoop-mapreduce-historyserver", version,
historyNode)
MAPRED.startHistoryserver(config=config)
time.sleep(5)
logger.info(
"**** End history server upgrade, begin timeline server upgrade ****"
)
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade]Job history server upgrade finished ")
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Timeline server upgrade started ")
ytsNode = YARN.getATSHost()
YARN.stopATSServer(ytsNode)
hdpSelect.changeVersion("hadoop-yarn-timelineserver", version, ytsNode)
YARN.startATSServer(config=config)
time.sleep(5)
logger.info("**** End timeline server upgrade, begin rm upgrade ****")
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Timeline server upgrade finished ")
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Resource manager upgrade started ")
if YARN.isHAEnabled():
nodes = []
nodes.append(YARN.getRMHostByState('standby'))
nodes.append(YARN.getRMHostByState('active'))
for node in nodes:
YARN.resetResourceManager('stop', config=config, host=node)
time.sleep(2)
hdpSelect.changeVersion("hadoop-yarn-resourcemanager", version,
node)
YARN.resetResourceManager('start', config=config, host=node)
time.sleep(5)
else:
node = MAPRED.getJobtracker()
MAPRED.stopJobtracker()
time.sleep(2)
# TODO use hdp-select script to upgrade JHs
hdpSelect.changeVersion("hadoop-yarn-resourcemanager", version,
node)
MAPRED.startJobtracker(config=config)
time.sleep(5)
logger.info("*** Ending RM upgrade ****")
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Resource manager Upgrade Finished")
def YARN_isHAEnabled(cls, logoutput=True):
try:
from beaver.component.hadoop import YARN
return YARN.isHAEnabled()
except Exception:
if logoutput:
logger.error(
"Exception occured during YARN_isHAEnabled() call")
logger.error(traceback.format_exc())
return False
def getComponnetsToTest(cls, compFile, depFile):
'''
Get the components that are being tested according to depFile
'''
# read in the config file
conf = RuSetup.readJson(compFile)
isStandalone = conf[RuSetup.CONF_STANDALONE]
RuSetup._skipQueue = set(conf[RuSetup.CONF_SKIP_QUEUE])
RuSetup._defaultQueue = conf[RuSetup.CONF_DEFAULT_QUEUE]
returnSet = None
if isStandalone:
# get the components to test
returnSet = set(conf[RuSetup.CONF_COMPONENTS_TEST])
else:
returnSet = set(RuSetup.getComponentsAffected(compFile, depFile))
# skip tests according to cluster settings
if not HDFS.isHAEnabled():
logger.info("Skip HDFS since HA is not enabled")
returnSet.discard("hdfs")
# as discussed in Ru standup for 11/13, enabling storm-slider for non HA cluster and storm standalone for HA cluster
if YARN.isHAEnabled():
returnSet.discard("storm-slider")
else:
returnSet.discard("storm")
if Hadoop.isEncrypted():
returnSet.discard("knox")
returnSet.discard("falcon")
if Hadoop.isTez():
logger.info("Add tez since Hadoop.isTez()")
returnSet.add("tez")
else:
logger.info(
"Make sure tez is not in the list since Hadoop.isTez() is false"
)
returnSet.discard("tez")
# Note: component.xa is always available, even if xa is not installed
# So this line should work even if the cluster does not have xa installed
from beaver.component.xa import Xa
if Xa.isArgusInstalled():
logger.info("Add argus since argus is there")
returnSet.add("argus")
else:
logger.info(
"Make sure argus is not in the list since it's not available")
returnSet.discard("argus")
return list(returnSet)
def resetYarn(skip_check=False):
# updates for Hadoop 2
if YARN.isHAEnabled():
logger.info("Resetting YARN...")
# only do this on nano as we these services are unreliable on nano
tasktrackers = MAPRED.getTasktrackers()
if skip_check or (Hadoop.isHadoop2() and Machine.isLinux()
and Machine.isNano()):
YARN.restartHARMNodes()
# add sleep to give RM enough time to register all the nodes
# and be ready
MAPRED.waitForNMToRegister(len(tasktrackers))
logger.info("Resetting YARN Completed.")