def get_argus_admin_backup(cls):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
localTestWorkDir1 = os.path.join(Config.getEnv('ARTIFACTS_DIR'))
## to have logic for repos backup
UpgradePerNode.reportProgress(
'#### getting repositories backup for ####')
if cls.get_repositories_backup(localTestWorkDir1):
UpgradePerNode.reportProgress(
'#### repositories backup successful ####')
## to have logic for policies backup
UpgradePerNode.reportProgress('#### getting policies backup ####')
if cls.get_policies_backup(localTestWorkDir1):
UpgradePerNode.reportProgress(
'#### policies backup successful ####')
def upgrade_slave(cls, version, node, config=None, logText="Upgrade"):
'''
Upgrade HDFS slave sevice: Datanode
:param version: latestVersion.
:param node: The node name where DN is running
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][HDFS][%s] Datanode %s for node %s to version %s started " %
(logText, logText, node, version))
ipcPort = HDFS.getDataNodeIPCPort()
cmd = "dfsadmin -shutdownDatanode %s:%s upgrade" % (node, ipcPort)
exitcode, stdout = HDFS.runas(Config.get('hadoop', 'HDFS_USER'),
cmd,
env=None,
logoutput=True,
config=None,
host=None,
skipAuth=False)
if exitcode != 0:
UpgradePerNode.reportProgress(
"[INFO][HDFS][%s] Datanode shutdownDatanode command failed for %s "
% (logText, node))
HDFS.waitForDNDown(node, ipcPort, "ipc.client.connect.max.retries=1")
from beaver.component.rollingupgrade.ruCommon import hdpSelect
hdpSelect.changeVersion("hadoop-hdfs-datanode", version, node)
HDFS.startDatanodes(config=config, nodes=[node])
cls.wait4DNLive(node)
UpgradePerNode.reportProgress(
"[INFO][HDFS][%s] Datanode %s for node %s to version %s finished "
% (logText, logText, node, version))
def background_job_teardown(cls):
'''
Cleanup for long running Yarn job
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][FLUME][BGJobTeardown] teardown for Long running job of Flume component started"
)
if agent1.isalive():
logger.info("Terminating the first Flume agent")
agent1.stop()
if agent2.isalive():
logger.info("Terminating the second Flume agent")
agent2.stop()
for outdir in (cls._agent1_chkpt_dir, cls._agent1_data_dir,
cls._agent2_chkpt_dir, cls._agent2_data_dir):
if os.path.exists(outdir):
shutil.rmtree(outdir)
if os.path.isfile(cls._data_file):
os.remove(cls._data_file)
if os.path.isfile(cls._data_stop):
os.remove(cls._data_stop)
def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Setup for background long running job
Upload Data to HDFS before Upgrade starts
Creates /user/hrt_qa/ru-pig dir on HDFS
Creates and Upload large data file to /user/hrt_qa/ru-pig/input/
:param runSmokeTestSetup: Runs smoke test setup if set to true
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"### Running Pig BackGround Job Setup ####")
HDFS.deleteDirectory(cls._base_hdfs_dir)
exit_code, stdout = HDFS.createDirectory(cls._base_hdfs_dir,
user=cls._job_user,
perm=777,
force=True)
ruAssert("Pig", exit_code == 0,
'[BGJobSetup] could not create dir on hdfs.')
HDFS.createDirectory(cls._hdfs_input_dir, force=True)
srcFile = os.path.join(cls._artifacts_dir, 'pig-ru-input.txt')
if os.path.exists(srcFile):
os.remove(srcFile)
tmpFile = os.path.join(cls._artifacts_dir, 'pig-ru-tmp-input.txt')
if os.path.exists(tmpFile):
os.remove(tmpFile)
util.copyFileToAnotherFile(cls._golden_src_file, srcFile)
util.copyFileToAnotherFile(srcFile, tmpFile)
itr = 12
if Machine.isFlubber():
itr = 16
for i in range(itr):
util.copyFileToAnotherFile(srcFile, tmpFile)
util.copyFileToAnotherFile(tmpFile, srcFile)
exit_code, stdout = HDFS.copyFromLocal(srcFile, cls._hdfs_input_path)
ruAssert("Pig", exit_code == 0, '[BGJobSetup] Data Load failed')
if runSmokeTestSetup:
cls.smoke_test_setup()
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run word count as HDFS smoke test
- Create file of 4096 bytes using randomwriter job
- Run wordcount job
'''
logger.info("Running HDFS Smoke test")
# make sure base hdfs dir is set.
if not cls._base_hdfs_dir:
cls._base_hdfs_dir = '/user/%s/test_rollingupgrade' % Config.get(
'hadoop', 'HADOOPQA_USER')
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][HDFS][Smoke] Smoke test for HDFS started ")
HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -ls /user/hrt_qa",
env=None,
logoutput=True,
config=None,
host=None,
skipAuth=False)
if not cls._SmokeInputDir:
cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput"
SmokeOutputDir = cls._base_hdfs_dir + '/smokeHdfsOutput' + str(
smoketestnumber)
HDFS.deleteDirectory(SmokeOutputDir,
Config.get('hadoop', 'HADOOPQA_USER'))
jobCmd = 'jar %s wordcount \"-Dmapreduce.reduce.input.limit=-1\" \"-D%s=%s\" %s %s' % (
Config.get('hadoop',
'HADOOP_EXAMPLES_JAR'), "mapred.job.queue.name",
cls._queue, cls._SmokeInputDir, SmokeOutputDir)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert("HDFS", exit_code == 0, "[Smoke] Hdfs smoketest failed")
HDFS.deleteDirectory(SmokeOutputDir)
ruAssert("HDFS", exit_code == 0,
"[Smoke] could not delete: " + SmokeOutputDir)
UpgradePerNode.reportProgress(
"[INFO][HDFS][Smoke] Smoke test for HDFS Finished ")
def run_smoke_test(cls):
'''
Run smoke test for yarn
:param smoketestnumber: Used for unique output log location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
host = Atlas.get_host()
if not host:
logger.error("No ACTIVE Atlas instance is available")
else:
logger.info("Active instance of Atlas is found: " + host)
response, response_status = Atlas.create_entities(input_json_string=cls.entity_create_request_payload)
if response_status != 201:
UpgradePerNode.reportProgress("[FAILED][Atlas][Smoke] Entity creation failed = %d" % response_status)
else:
guid = response["definition"]["id"]["id"]
logger.info("Entity creation was successful. GUID = %s" % guid)
cls.guid_list.append(guid)
UpgradePerNode.reportProgress("[INFO][Atlas][Smoke] Smoke test for Atlas component finished")
def doBackgroundJobSetup(cls, hdfs_test_dir):
from beaver.component.hive import Hive
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
logger.info("Preparing the test setup for Hive background job")
udfjar = os.path.join(Config.getEnv('WORKSPACE'), "tests", "hive",
"hive-udf", "hive-udfs-0.1.jar")
HDFS.createDirectory(hdfs_test_dir,
user=cls._hdfs_user,
perm='777',
force=True)
HDFS.copyFromLocal(udfjar, hdfs_test_dir)
query = "drop function sleep; create function sleep as 'org.apache.hive.udf.generic.GenericUDFSleep' using jar 'hdfs://%s/hive-udfs-0.1.jar';" % hdfs_test_dir
exit_code, stdout = Hive.runQuery(query)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][Hive][Setup] Long running failed due to exitcode = %d"
% exit_code)
else:
UpgradePerNode.reportProgress(
"[PASSED][Hive][Setup] Long running finished successfully")
def ensure_nn_is_active(cls, timeout=11 * 60):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
active_nn = None
curr_time = int(time.time())
end_time = int(time.time()) + timeout
while not active_nn and curr_time <= end_time:
active_nn = HDFS.getNamenodeByState('active')
# we we found active NN return it
if active_nn:
UpgradePerNode.reportProgress(
"[INFO][HDFS] Active Namenode is %s" % active_nn)
return
# wait for 30s
time.sleep(30)
curr_time = int(time.time())
if not active_nn:
UpgradePerNode.reportProgress("[FAILED][HDFS] No Active Namenode")
assert active_nn
return
def upgrade_slave(cls, version, node, config=None):
'''
Upgrades Yarn slave services : NM
:param version: Version to be upgraded to
:param node: Slave Node
:param config: Config location
:return:
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Nodemanager Upgrade for %s started " % node)
logger.info("**** Beginning upgrade for nodemanager on %s ****" % node)
MAPRED.stopTasktrackers(nodes=[node])
from beaver.component.rollingupgrade.ruCommon import hdpSelect
hdpSelect.changeVersion("hadoop-yarn-nodemanager", version, node)
# BUG-27328
# make sure we set the linux container executor permissions
if Hadoop.isSecure() and not Machine.isWindows():
container_exec_cfg = os.path.join(
Config.get('hadoop', 'HADOOP_CONF'), 'container-executor.cfg')
container_exec = os.path.join(
Config.get('hadoop',
'YARN_HOME').replace("client", "nodemanager"),
'bin', 'container-executor')
cmd="chown root:hadoop %s %s ; chmod 400 %s ; chmod 6050 %s" % \
(container_exec_cfg, container_exec, container_exec_cfg, container_exec)
Machine.runas(Machine.getAdminUser(),
cmd,
host=node,
cwd=None,
env=None,
logoutput=True,
passwd=Machine.getAdminPasswd())
MAPRED.startTasktrackers(config, nodes=[node])
logger.info("**** Ending upgrade for nodemanager on %s ****" % node)
UpgradePerNode.reportProgress(
"[INFO][YARN][Upgrade] Nodemanager Upgrade on %s Finished " % node)
def validate_correct_tar_used(cls, output):
'''
Validates that correct tez tar ball is used
:param output: Console output from application
:return: True if corret tar ball is used otherwise returns False
'''
from beaver.component.rollingupgrade.ruCommon import hdpRelease
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
currVersion = hdpRelease.getCurrentRelease("hadoop-client")
pattern = "Using tez.lib.uris value from configuration: /hdp/apps/%s/tez/tez.tar.gz" % currVersion
m = re.search(pattern, output)
if m:
logger.info("Tez application is using correct version")
UpgradePerNode.reportProgress(
"### [PASSED][Tez][Smoke] Tez smoke test is using correct version. Passed successfully ####"
)
return True
else:
logger.info("Tez application is not using correct version")
UpgradePerNode.reportProgress(
"### [FAILED][Tez][Smoke] Tez smoke test is not using correct version. Validation Failed ####"
)
return False
def background_job_setup(cls,
components,
runSmokeTestSetup=True,
config=None):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
'#### call for background-job setup for argus admin started ####')
if runSmokeTestSetup:
# componentList = ["hive","hbase","hdfs","knox","storm"]
componentList = []
if Xa.isHdfsInstalled():
componentList.append('hdfs')
if Xa.isHiveInstalled():
componentList.append('hive')
if Xa.isHBaseInstalled():
componentList.append('hbase')
if Xa.isKnoxInstalled():
componentList.append('knox')
if Xa.isStormInstalled():
componentList.append('storm')
cls.smoke_test_setup(componentList)
UpgradePerNode.reportProgress(
'#### call for background-job setup for argus admin done ####')
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run smoke test for yarn
:param smoketestnumber: Used for unique output log location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][Smoke] Starting smoke test for Yarn")
numNodes = len(MAPRED.getTasktrackers())
# run sleep job
(exitCode,
stdout) = HadoopJobHelper.runSleepJob(numOfMaps=numNodes,
numOfReduce=numNodes,
mapSleepTime=10,
reduceSleepTime=10,
extraJobArg=cls._jobArgs,
runInBackground=False,
config=config)
# Make sure succeeded
ruAssert("YARN", exitCode == 0, "[Smoke] YARN smoke failed")
UpgradePerNode.reportProgress(
"[INFO][YARN][Smoke] Smoke test for Yarn finished ")
cls.run_rest_apis_test()
def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Setup for background long running job
:param runSmokeTestSetup: Runs smoke test setup if set to true
'''
from beaver.component.sqoop import Sqoop
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][SQOOP][BGSetup] Background job set for sqoop is called")
dbflavor = Sqoop.getDatabaseFlavor()
if dbflavor == "oracle":
cls._lrtest_tbl = cls._lrtest_tbl.upper()
cls._smtest_tbl = cls._smtest_tbl.upper()
logger.info(
"*** Creating the test data for Sqoop long running test on dbflavorr= %s ***"
% dbflavor)
cls.generate_test_data(cls._hdfs_lrtest_dir, cls._test_lrrow_count)
create_tbl_query = "create table %s (userid int, age int, primary key (userid));" % cls._lrtest_tbl
cls.init_test_database(dbflavor)
cls.run_database_query(create_tbl_query, dbflavor)
if runSmokeTestSetup:
logger.info("**** Running Sqoop Smoke Test Setup ****")
cls.smoke_test_setup()
def run_second_background_job(cls, runSmokeTestSetup=False, config=None):
'''
Runs long running distributed shell Yarn job after RM upgrade
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][BGJob2] Starting second background job for Yarn ")
Local_Test_dir = os.path.join(Config.getEnv("WORKSPACE"), "tests",
"rolling_upgrade", "yarn")
Multi_Version_App_Dir = os.path.join(Local_Test_dir, "data")
Mapper = os.path.join(Multi_Version_App_Dir, "mvMapper.py")
Reducer = os.path.join(Multi_Version_App_Dir, "mvReducer.py")
HadoopJobHelper.runSleepJob(numOfMaps=1,
numOfReduce=1,
mapSleepTime="100000",
reduceSleepTime="100",
extraJobArg=cls._jobArgs,
runInBackground=True,
config=config,
directoutput=False)
#(jobID, STREAMING_APP_ID) = HadoopJobHelper.checkAndRunStreamJob(Mapper, Reducer, cls._hdfs_input, cls._hdfs_output, files = Multi_Version_App_Dir, config = config, forRU=True)
logger.info("Second Long Running job for YARN has started")
return 1
def write_hbase_site(cls, config=None):
'''
Obtain hbase-site.xml from Slider HBase and write to modified config path
'''
global new_conf_path
from beaver.component.sliderhbase import SliderHBase
from beaver.component.slider import Slider
tmpHBaseConfFile = os.path.join(Machine.getTempDir(), "hbase-site.xml")
if Hadoop.isSecure():
hbasesite = Slider.registry(
"sliderhbase",
flags="--getconf hbase-site --out " + tmpHBaseConfFile,
format="xml",
user=cls.HBASE_USER,
userFlag=cls.HBASE_USER
)
else:
hbasesite = Slider.registry(
"sliderhbase",
flags="--getconf hbase-site --out " + tmpHBaseConfFile,
format="xml",
user=cls.HBASE_USER
)
propertyMap = {'hbase.tmp.dir': '/tmp/hbase-tmp'}
generated_hbase_conf = os.path.join(new_conf_path, "hbase-site.xml")
# QE-3108
# if the hbase app is created successfully, a /tmp/hbase-site.xml will be
# generated from the app. Otherwise if the /tmp/hbase-site.xml is missing
# it means the slider app creation fails
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
if os.path.isfile(generated_hbase_conf):
util.writePropertiesToConfigXMLFile(tmpHBaseConfFile, generated_hbase_conf, propertyMap)
else:
UpgradePerNode.reportProgress(
"[FAILED][Slider][background] Slider hbase background setup failed due to hbase-site.xml not generated"
)
def upgrade_master(cls, version, config=None):
'''
Upgrades Master services:
:param version: Version to be upgraded to
:param config: Config location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
function_name = inspect.stack()[0][3]
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from beaver.component.oozie import Oozie
cls.background_job_when_master_upgrade()
oozieServers = Oozie.getOozieServers()
for server in oozieServers:
UpgradePerNode.reportProgress("### Oozie %s has started %s ####" %
(server, function_name))
Oozie.resetOozie('stop', host=server)
time.sleep(60)
hdpSelect.changeVersion("oozie-server", version, server)
Oozie.resetOozie('start', configDir=config, host=server)
time.sleep(60)
UpgradePerNode.reportProgress("### Oozie %s has finished %s ####" %
(server, function_name))
cls.verify_background_job_when_master_upgrade()
cls.background_job_teardown_when_master_upgrade()
def run_background_job(cls, runSmokeTestSetup=True, config=None):
'''
Runs background long running Yarn Job
:param runSmokeTestSetup: Runs smoke test setup if set to true
:param config: expected configuration location
:return: Total number of long running jobs started
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][BGJob] Starting background job for Yarn ")
Local_Test_dir = os.path.join(Config.getEnv("WORKSPACE"), "tests",
"rolling_upgrade", "yarn")
Multi_Version_App_Dir = os.path.join(Local_Test_dir, "data")
HDFS.deleteDirectory(cls._hdfs_output,
user=Config.get('hadoop', 'HADOOPQA_USER'))
Mapper = "data/mvMapper.py"
Reducer = "data/mvReducer.py"
# Launch job
(jobID, STREAMING_APP_ID) = HadoopJobHelper.checkAndRunStreamJob(
Mapper,
Reducer,
cls._hdfs_input,
cls._hdfs_output,
files=Multi_Version_App_Dir,
config=config,
forRU=True,
extraJobArg=cls._jobArgs,
env=None,
sleepTimeAfterJobSubmission=60)
cls._background_job_appId = STREAMING_APP_ID
cls._background_job_jobId = jobID
logger.info("Background job started, application ID: " +
STREAMING_APP_ID + " job ID: " + jobID)
logger.info("Start second long running job for Yarn")
num_sec_bkjob = cls.run_second_background_job(config=config)
return 2
def runCommand(cls, user, cmd, runInBackground=False):
"""
Function to run pig cmd as a certain user
:param user:
:param cmd:
:return:
"""
try:
from beaver.component.pig import Pig
except ImportError:
## Import fails when Pig is not installed on this machine. Nothing to do
return 0, ""
if runInBackground:
cls._process = Pig.runas(user, cmd, runInBackground=True)
else:
exit_code, stdout = Pig.runas(user, cmd, runInBackground=False)
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
if exit_code == 0:
UpgradePerNode.reportProgress(
"### [PASSED][Pig][Smoke] Pig Smoke test passed ####")
else:
UpgradePerNode.reportProgress(
"### [FAILED][Pig][Smoke] Pig Smoke test Failed ####")
def downgrade_master(cls, version, config=None):
'''
Downgrade Master services
:param version: Version to be downgraded to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress("### Knox Downgrade started ####")
#Stop knox gateway and apacheDS
Knox.stopKnox()
Knox.stopLdap()
node = Config.get('knox', 'KNOX_HOST').split(',')[0]
#Switch to the new version
hdpSelect.changeVersion("knox-server", version, node)
#Start apacheDs and knox gateway service
Knox.startLdap()
Knox.startKnox()
time.sleep(10)
UpgradePerNode.reportProgress("### Knox Downgrade Finished ####")
def get_current_release(cls):
'''
get current version in action
:return: current version in use Like 2.2.0.1-177
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
argus_root = "/usr/lib/xapolicymgr"
linkname = os.path.join(argus_root)
UpgradePerNode.reportProgress('#### linkname = %s ####' % linkname)
if os.path.islink(linkname):
(head, tail) = os.path.split(os.readlink(linkname))
UpgradePerNode.reportProgress('#### head = %s ####' % head)
UpgradePerNode.reportProgress('#### tail = %s ####' % tail)
argus_version = tail.split('-', 1)[1][6:len(tail.split('-', 1)[1])]
return argus_version
else:
UpgradePerNode.reportProgress(
"#### No valid symlink for argus ####")
return None
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run MRRSleep tez job
:param smoketestnumber: Used for unique output log location
'''
logger.info("***** Starting MRRSleep job in Tez ******")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress("### Starting smoke job for tez ####")
numNodes = len(MAPRED.getTasktrackers())
# Post a sleep job
(exit_code, stdout) = Tez.runTezExampleJar(
'mrrsleep "-Dmapreduce.framework.name=yarn-tez" "-Dtez.queue.name=%s" -m %d -r %d -ir 3 -irs 5 -mt 10 -rt 10'
% (cls._queue, numNodes, numNodes))
# Make sure succeeded
if exit_code == 0:
UpgradePerNode.reportProgress(
"### [PASSED][Tez][Smoke] smoke job for tez passed ####")
else:
UpgradePerNode.reportProgress(
"### [FAILED][Tez][Smoke] smoke job for tez failed ####")
time.sleep(10)
logger.info("***** Completed MRRSleep job in Tez ******")
UpgradePerNode.reportProgress("### smoke test finished for tez ####")
cls.validate_correct_tar_used(stdout)
def verifyLongRunningJob(cls, config=None):
'''
Verify long running background job after it finishes
:return:
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
retVal, checkContent = HDFS.cat(cls._hdfs_output + '/part-00000')
#assert retVal == 0
if retVal == 0:
UpgradePerNode.reportProgress(
"[PASSED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, retVal = 0. Successful check "
)
else:
UpgradePerNode.reportProgress(
"[FAILED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, retVal != 0. Failed check "
)
#assert 'true' in checkContent, "Stream job returns false: " + checkContent
if 'true' in checkContent:
UpgradePerNode.reportProgress(
"[PASSED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, true in checkContent. Successful check "
)
else:
UpgradePerNode.reportProgress(
"[FAILED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, true not in checkContent. Failed check "
)
#verify application's attempt doesn't increase and no failed tasks.
appID = cls._background_job_appId
jobID = cls._background_job_jobId
# temporarily skipping check
#assert YARN.getNumAttemptsForApp(appID) == 1
#YARN.verifyMRTasksCount(jobID, appID, 0, skipAssert=True)
from beaver.component.rollingupgrade.ruCommon import hdpRelease
Version_Num = hdpRelease.getCurrentRelease("hadoop-client")
cls.checkClasspathVersion(Version_Num, config)
def downgrade_master(cls, version, config=None):
'''
Downgrade Master services
:param version: Version to be downgraded to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
function_name = inspect.stack()[0][3]
UpgradePerNode.reportProgress("### Oozie %s has started ####" %
function_name)
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from beaver.component.oozie import Oozie
oozieServers = Oozie.getOozieServers()
for server in oozieServers:
UpgradePerNode.reportProgress("### Oozie %s has started %s ####" %
(server, function_name))
Oozie.resetOozie('stop', host=server)
time.sleep(60)
hdpSelect.changeVersion("oozie-server", version, server)
Oozie.resetOozie('start', configDir=config, host=server)
time.sleep(60)
UpgradePerNode.reportProgress("### Oozie %s has finished %s ####" %
(server, function_name))
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run smoke test for yarn
:param smoketestnumber: Used for unique output log location
'''
global new_conf_path
global buildNo
from beaver.component.hbase import HBase
# Run slider agent labels funtests
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress("### Slider smoke test started ####")
exit_code, stdout = HBase.runShellCmds(cls.SCAN_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path)
UpgradePerNode.reportProgress("### Slider smoke test scanned " + cls.TABLE_NAME)
hbase_ver = HBase.getVersion(configPath=new_conf_path)
if buildNo not in hbase_ver:
UpgradePerNode.reportProgress("### smoke test failed: " + buildNo + " not found in " + hbase_ver)
else:
UpgradePerNode.reportProgress("### smoke test passed: " + buildNo + " found in " + hbase_ver)
UpgradePerNode.reportProgress("scanned " + cls.TABLE_NAME)
TABLE_NM = cls.id_generator(10)
CREATE_TABLE_CMD = ["create '" + TABLE_NM + "', 'family1', 'family2', 'family3'"]
exit_code, stdout = HBase.runShellCmds(CREATE_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path)
if exit_code == 0:
UpgradePerNode.reportProgress("created " + TABLE_NM)
else:
UpgradePerNode.reportProgress("creation of " + TABLE_NM + "failed")
if not cls._LOCAL_WORK_DIR:
UpgradePerNode.reportProgress(
"[FAILED][Slider][Smoke] Slider smoke test failed due to Slider source code unavailability"
)
return
exit_code, stdout = Maven.run(
"verify -Dslider.conf.dir=%s "
"-Dhadoop.version=%s "
"-Dprivate.repo.url=%s "
"-Dit.test=AppsUpgradeIT" % (cls._LOCAL_CONF_DIR, Hadoop.getVersion(), Maven.getPublicRepoUrl()),
cwd=os.path.join(cls._LOCAL_WORK_DIR, 'slider-funtest')
)
testresults = {}
testResultFiles = []
TEST_RESULT = {}
# get a list of all the test result files
for name, dirs, files in os.walk(cls._LOCAL_WORK_DIR):
if os.path.basename(name) == 'target':
# Add in each failsafe-report we find -- this test only runs failsafe reports
testResultFiles.extend(util.findMatchingFiles(os.path.join(name, 'failsafe-reports'), 'TEST-*.xml'))
for resultFile in testResultFiles:
testresults.update(util.parseJUnitXMLResult(resultFile))
for key, value in testresults.items():
TEST_RESULT[key] = value
logger.info("=======================")
logger.info(TEST_RESULT)
logger.info("=======================")
TestCases = TEST_RESULT.keys()
for testcase in TestCases:
result = TEST_RESULT[testcase]['result']
if result == "pass":
UpgradePerNode.reportProgress("[PASSED][Slider][Smoke] Slider smoke test passed")
else:
UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider smoke test failed")
def smoke_test_setup(cls):
'''
Setup required to run Smoke test
'''
from beaver.component.slider import Slider
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
HADOOPQA_USER = Config.get('hadoop', 'HADOOPQA_USER')
UpgradePerNode.reportProgress("### Starting set up for Slider smoke test ####")
if Hadoop.isSecure():
keytabFile = Machine.getHeadlessUserKeytab(HADOOPQA_USER)
kinitloc = Machine.which("kinit", "root")
cmd = "%s -k -t %s %s" % (kinitloc, keytabFile, Machine.get_user_principal(HADOOPQA_USER))
exit_code, stdout = Machine.run(cmd)
if exit_code != 0:
UpgradePerNode.reportProgress("### smoke test setup for Slider failed due to kinit failed ####")
# Local directory in artifacts that we'll run tests from
# it is possible the download_source() will fail
try:
cls._LOCAL_WORK_DIR = Slider.download_source(useHDPBaseRepoFile=False, isRUcluster=True)
logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR)
except TypeError as err:
UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source download fail.")
return
if not cls._LOCAL_WORK_DIR:
UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source Code missing.")
return
# Local conf directory with modified conf for tests
if not os.path.exists(cls._LOCAL_CONF_DIR):
os.makedirs(cls._LOCAL_CONF_DIR)
slider_conf = os.path.join(Slider.getSliderHome(), 'conf')
logger.info("slider_conf = %s" % slider_conf)
logger.info("os path exist slider_conf = %s" % os.path.exists(slider_conf))
if os.path.exists(slider_conf):
Machine.copy(
os.path.join(slider_conf, 'log4j.properties'), os.path.join(cls._LOCAL_CONF_DIR, 'log4j.properties')
)
Machine.copy(
os.path.join(slider_conf, 'slider-env.sh'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-env.sh')
)
else:
UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Conf %s missing" % slider_conf)
return
if Hadoop.isSecure():
util.writePropertiesToConfigXMLFile(
os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'),
os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), {
"slider.funtest.enabled": "true",
"slider.test.agent.enabled": "true",
"HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'),
"slider.am.keytab.local.path": Machine.getHeadlessUserKeytab(HADOOPQA_USER),
"slider.keytab.principal.name": Machine.get_user_principal(HADOOPQA_USER)
}
)
else:
util.writePropertiesToConfigXMLFile(
os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'),
os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), {
"slider.funtest.enabled": "true",
"HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'),
"slider.test.agent.enabled": "true"
}
)
logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR)
# Check that the precondition is met and the source is available
if not os.path.exists(cls._LOCAL_WORK_DIR) or not os.path.exists(os.path.join(cls._LOCAL_WORK_DIR, 'pom.xml')):
logger.info('Slider source does not appear to exist at %s' % (cls._LOCAL_WORK_DIR))
UpgradePerNode.reportProgress(
"### Slider source does not appear to exist at %s ####" % (cls._LOCAL_WORK_DIR)
)
logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR)
if cls._LOCAL_WORK_DIR == None:
logger.info("ERROR: cls._LOCAL_WORK_DIR is None")
# Install first so isolated modules can be tested
exit_code, stdout = Maven.run(
"clean install -DskipTests "
"-Dhadoop.version=%s "
"-Dprivate.repo.url=%s " % (Hadoop.getVersion(), Maven.getPublicRepoUrl()),
cwd=cls._LOCAL_WORK_DIR
)
if exit_code != 0:
UpgradePerNode.reportProgress("### Error installing Slider source : %d: %s ####" % (exit_code, stdout))
else:
UpgradePerNode.reportProgress("### Slider source install passed ####")
def upgrade_argus_admin(cls, latestVersion, config, currVersion):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress('#### install argus admin. ####')
# 'knox.crt'
# 'cacertswithknox'
node = Xa.getPolicyAdminHost()
user = Machine.getAdminUser()
xa_admin_stop_cmd = 'sudo service ranger-admin stop'
xa_admin_start_cmd = 'sudo service ranger-admin start'
ranger_old_version = '/usr/hdp/' + currVersion + '/ranger-admin/'
ranger_new_version = '/usr/hdp/' + latestVersion + '/ranger-admin/'
localTestWorkDir1 = os.path.join(Config.getEnv('ARTIFACTS_DIR'))
knox_cert_files = '{knox.crt,cacertswithknox}'
source_files_to_copy = os.path.join(ranger_old_version,
knox_cert_files)
command_copy_knoxcerts = 'cp -f %s %s' % (source_files_to_copy,
localTestWorkDir1)
exit_code, stdout = Machine.runas(user,
command_copy_knoxcerts,
host=node,
logoutput=True)
logger.info(
'*************************** admin copy command_copy_knoxcerts exit_code = '
+ str(exit_code))
logger.info(
'*************************** admin copy command_copy_knoxcerts stdout = '
+ str(stdout))
exit_code, stdout = Machine.runas(user,
xa_admin_stop_cmd,
host=node,
logoutput=True)
logger.info('****************** xa admin stop exit_code = ' +
str(exit_code))
source_properties_file = os.path.join(ranger_old_version,
'install.properties')
destination_properties_file = os.path.join(ranger_new_version,
'install.properties')
command = 'cp -f %s %s' % (source_properties_file,
destination_properties_file)
exit_code, stdout = Machine.runas(user,
command,
host=node,
logoutput=True)
logger.info(
'*************************** admin copy command exit_code = ' +
str(exit_code))
logger.info(
'*************************** admin copy command stdout = ' +
str(stdout))
command = '(cd %s && export JAVA_HOME=%s && ./setup.sh)' % (
ranger_new_version, Machine.getJavaHome())
UpgradePerNode.reportProgress(
'#### ranger-admin: installing new version with command %s ###' %
command)
exit_code, stdout = Machine.runas(user,
command,
host=node,
logoutput=True)
if exit_code == 0 and 'Installation of XASecure PolicyManager Web Application is completed.' in stdout:
UpgradePerNode.reportProgress(
'#### ranger-admin: installation successful ###')
from beaver.component.rollingupgrade.ruCommon import hdpSelect
hdpSelect.changeVersion("ranger-admin", latestVersion, node)
logger.info(
'*************************** ranger-admin: hdp-select to new version done = '
)
source_files_to_copy = os.path.join(localTestWorkDir1,
knox_cert_files)
command_copy_knoxcerts = 'cp -f %s %s' % (source_files_to_copy,
ranger_new_version)
exit_code, stdout = Machine.runas(user,
command_copy_knoxcerts,
host=node,
logoutput=True)
logger.info(
'*************************** admin copy command_copy_knoxcerts back exit_code = '
+ str(exit_code))
logger.info(
'*************************** admin copy command_copy_knoxcerts back stdout = '
+ str(stdout))
UpgradePerNode.reportProgress(
'#### ranger-admin: starting new version with command %s ###' %
xa_admin_start_cmd)
exit_code, stdout = Machine.runas(user,
xa_admin_start_cmd,
host=node,
logoutput=True)
logger.info('****************** xa admin start exit_code = ' +
str(exit_code))
logger.info('****************** xa admin start stdout = ' +
str(stdout))
if exit_code == 0:
UpgradePerNode.reportProgress(
'#### ranger-admin: new version started successfully ####')
else:
UpgradePerNode.reportProgress(
'#### ranger-admin: failed to start new version! exit_code=%d ####'
% exit_code)
else:
logger.info(
'****************** setup.sh script failed for admin ******* ')
UpgradePerNode.reportProgress(
'#### ranger-admin: installation of new version failed! exit_code=%d ###'
% exit_code)
def get_repositories_backup(cls, localTestWorkDir1):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
'#### saving existing repositories for backup ####')
cwObject = XACWTestCase()
baseUrl = 'http://' + xa_testenv.getEnv(
'XA_ADMIN_HOST', 'localhost') + ':' + str(
xa_testenv.getEnvInt('XA_ADMIN_PORT', 6080))
urlReposUsingApi = baseUrl + '/service/public/api/repository/'
username_password = xa_testenv.getEnv(
'XA_ADMIN_USER', 'admin') + ':' + xa_testenv.getEnv(
'XA_ADMIN_PASS', 'admin')
headerResponse, response = cwObject.callPyCurlRequest(
url=urlReposUsingApi,
data=None,
method='get',
usernamepassword=username_password)
UpgradePerNode.reportProgress(
'#### get all repositories, headerResponse is : %s ####' %
headerResponse)
UpgradePerNode.reportProgress(
'#### get all repositories, response is : %s ####' % response)
if 'HTTP/1.1 200 OK' in headerResponse:
file_for_repositories = os.path.join(localTestWorkDir1,
'repositories.bak')
UpgradePerNode.reportProgress(
'#### saving repositories to path = %s ####' %
str(file_for_repositories))
openReposFile = open(file_for_repositories, 'wb')
openReposFile.write(response)
openReposFile.close()
else:
UpgradePerNode.reportProgress(
'#### unable to save repositories for backup ####')
return False
UpgradePerNode.reportProgress(
'#### repositories back-up file saved successfully ####')
return True
def verifySchemaFunctionality(cls):
'''
We verify that the system can operate with SCHEMA functionality.
'''
from beaver.component.phoenix import Phoenix
from beaver.component.hbase import HBase
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
HBASE_HOME = Config.get('hbase', 'HBASE_HOME')
HBASE_CONF_DIR = os.path.join(HBASE_HOME, "conf")
#We verify the schema functionality.
HBASE_CHANGES = {}
HBASE_CHANGES['hbase-site.xml'] = {}
HBASE_CHANGES['hbase-site.xml'][
'phoenix.schema.isNamespaceMappingEnabled'] = 'true'
HBASE_CHANGES['hbase-site.xml'][
'phoenix.schema.mapSystemTablesToNamespace'] = 'true'
TEST_TABLE_A = 'Table_A'
SCHEMA_1 = 'SCHEMA_1'
masterNodes = HBase.getAllMasterNodes()
regionNodes = HBase.getRegionServers()
hbase_allnodes = masterNodes + regionNodes
gateway_node = Machine.getfqdn()
if gateway_node not in hbase_allnodes:
hbase_allnodes.append(gateway_node)
HBase.stopHBaseCluster()
HBase.modifyConfig(changes=HBASE_CHANGES,
nodeSelection={'nodes': hbase_allnodes})
util.copy_back_to_original_config(HBase.getModifiedConfigPath(),
HBASE_CONF_DIR,
file_list=["hbase-site.xml"],
node_list=hbase_allnodes)
HBase.startHBaseCluster(HBase.getModifiedConfigPath())
#We grant permissions to all tables.
Phoenix.grantPermissionsToSystemTables(schemaFunctionalityEnabled=True)
#We check that we can still query the original table.
cls.verifyBasicTable()
#We check that we can create/query schemas.
exit_code, stdout = Phoenix.runSQLLineCmds(
'CREATE SCHEMA IF NOT EXISTS %s;' % SCHEMA_1)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Creation of schema %s failed due to exitcode = %s "
% (SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Schema creation %s succeeded." %
(SCHEMA_1))
#we create tables inside that schema
primaryKey = {'name': 'ID', 'type': 'BIGINT'}
columns = [{
'name': 'FirstName',
'type': 'VARCHAR(30)'
}, {
'name': 'SecondName',
'type': 'VARCHAR(30)'
}, {
'name': 'City',
'type': 'VARCHAR(30)'
}]
exit_code, stdout = Phoenix.createTable(SCHEMA_1 + '.' + TEST_TABLE_A,
primaryKey, columns)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table creation %s on schema %s failed due to exitcode = %s "
% (TEST_TABLE_A, SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table creation %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
#We insert some data into the table through upsert.
for i in range(0, 5):
exit_code, stdout = Phoenix.runSQLLineCmds(
'UPSERT INTO %s VALUES (%s, "name_%s","secondName_%s","city_%s");'
% (SCHEMA_1 + '.' + TEST_TABLE_A, str(i), str(i), str(i),
str(i)))
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table UPSERT %s on schema %s failed due to exitcode = %s "
% (TEST_TABLE_A, SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table UPSERT %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
#We verify that the data has been correctly inserted
exit_code, stdout = Phoenix.runSQLLineCmds(
'SELECT * FROM %s WHERE ID=3;' % (SCHEMA_1 + '.' + TEST_TABLE_A))
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table SELECT %s on schema %s failed due to exitcode = %s "
% (TEST_TABLE_A, SCHEMA_1, exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table SELECT %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
if stdout.find('name_3') == -1 or stdout.find(
'secondName_3') == -1 or stdout.find('city_3') == -1:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table SELECT %s on schema %s returned the wrong results: %s"
% (TEST_TABLE_A, SCHEMA_1, stdout))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table SELECT %s on schema %s succeeded."
% (TEST_TABLE_A, SCHEMA_1))
#We verify that we can drop the schemas with tables on it.
exit_code, stdout = Phoenix.runSQLLineCmds('DROP SCHEMA %s;' %
SCHEMA_1)
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Schema drop failed due to exitcode = %s "
% (exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Schema drop succeeded.")
#We verify that the schema has been dropped.
exit_code, stdout = Phoenix.runSQLLineCmds(
'SELECT TABLE_NAME FROM SYSTEM.CATALOG WHERE SCHEMA = %s' %
SCHEMA_1,
outputFormat='xmlattr')
if exit_code != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Schema drop failed due to exitcode = %s "
% (exit_code))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Schema drop succeeded.")
if stdout.find(TEST_TABLE_A) != 0:
UpgradePerNode.reportProgress(
"[FAILED][PHOENIX][Smoke] Table %s did not drop on drop schema command "
% (TEST_TABLE_A))
else:
UpgradePerNode.reportProgress(
"[PASSED][PHOENIX][Smoke] Table %s successfuly dropped." %
TEST_TABLE_A)
def prepare_and_start_long_running_jobs(self):
############################Prepare and start long running jobs
self.find_components_to_test()
import beaver.component.rollingupgrade.ruSetup as ruSetup
ruSetup.COMPONENTS_TO_FLIP = self.COMPONENTS_TO_TEST
ruSetup.COMPONENTS_AFFECTED = self.COMPONENTS_TO_TEST
ruSetup.COMPONENTS_TO_TEST = self.COMPONENTS_TO_TEST
ruSetup.COMPONENTS_TO_IMPORT = self.COMPONENTS_TO_TEST
from beaver.component.hadoop import HDFS
if "slider" in self.COMPONENTS_TO_TEST:
from beaver.component.rollingupgrade.ruSlider import ruSlider
from beaver.component.rollingupgrade.ruCommon import Rollingupgrade
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
HDFS.createUserDirWithGroup('/user/' +
Config.get('hadoop', 'HADOOPQA_USER'),
'hdfs',
Config.get('hadoop', 'HADOOPQA_USER'),
'hdfs',
diffGroup=True)
logger.info(Config.get('hadoop', 'HADOOPQA_USER'))
logger.info(
"1==========================================================================="
)
self.setup_ru_cluster()
logger.info(
"2==========================================================================="
)
logger.info(
"3==========================================================================="
)
DN = HDFS.getDatanodes()
# Find core components (HDFS, YARN, HBase) if exist.
#core_components = UpgradePerNode.find_existing_core_components(COMPONENTS_TO_TEST)
logger.info(
"4==========================================================================="
)
#Prepare and save state before upgrade
Rollingupgrade.ru_prepare_save_state_for_upgrade(
self.COMPONENTS_TO_TEST)
logger.info(
"5==========================================================================="
)
# Run setup for background Jobs for all components
Rollingupgrade.background_job_setup(self.COMPONENTS_TO_TEST,
config=None)
logger.info("Running smoke tests before upgrade")
Rollingupgrade.run_smokeTests(self.COMPONENTS_TO_TEST)
logger.info(
"6==========================================================================="
)
# Starts Long running background Jobs for all components
numBackgroundJobs = Rollingupgrade.run_longRunning_Application(
self.COMPONENTS_TO_TEST, config=None)
logger.info(
"7==========================================================================="
)
logger.info(
"Total number of long running background jobs before starting upgrade is %s"
% numBackgroundJobs)
UpgradePerNode.reportProgress(
"### Just started %s background jobs ###" % numBackgroundJobs)
UpgradeLogger.reportProgress(
"### Just started %s background jobs ###" % numBackgroundJobs,
True)
def validate_lr_job(self):
#################################################Finsih long running jobs
### Need to stop HDFS Falcon,Yarn long runningJobs ####
# create flagFile to kill HDFS background job
from beaver.component.hadoop import HDFS, YARN
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
from beaver.component.rollingupgrade.ruCommon import Rollingupgrade
TEST_USER = Config.get('hadoop', 'HADOOPQA_USER')
createCmd = "dfs -touchz " + UpgradePerNode._HDFS_FLAG_FILE
exit_code, output = HDFS.runas(TEST_USER, createCmd)
logger.info(
"8==========================================================================="
)
if self.DO_DOWNGRADE:
self.removeComponentFromTest("falcon")
if "falcon" in self.COMPONENTS_TO_TEST:
from beaver.component.rollingupgrade.ruFalcon import ruFalcon
ruFalcon.stopFalconLongRunningJob()
if "yarn" in self.COMPONENTS_TO_TEST:
from beaver.component.rollingupgrade.ruYarn import ruYARN
ruYARN.stopYarnLongRunningJob()
# if "hive" in self.COMPONENTS_TO_TEST:
# from beaver.component.rollingupgrade.ruHive import ruHive
# ruHive.stopHiveLongRunningJob()
if "slider" in self.COMPONENTS_TO_TEST:
from beaver.component.rollingupgrade.ruSlider import ruSlider
ruSlider.stopSliderLongRunningJob()
if "storm-slider" in self.COMPONENTS_TO_TEST:
from beaver.component.rollingupgrade.ruStorm import ruStorm
ruStorm.teardown_storm_slider_app()
logger.info(
"9==========================================================================="
)
## TODO - wait for long running jobs to finish
isZero = YARN.waitForZeroRunningApps()
logger.info(
"10==========================================================================="
)
if isZero:
UpgradePerNode.reportProgress("#### None apps are running. ####")
UpgradeLogger.reportProgress("#### None apps are running. ####",
True)
else:
UpgradePerNode.reportProgress(
"#### Check Failed. some apps are running. ####")
UpgradeLogger.reportProgress(
"#### Check Failed. some apps are running. ####", False)
#assert isZero, "all long running jobs are not finished"
### List down Failed/Killed applications ####
Failed_Killed_apps = YARN.getFailedKilledAppList()
UpgradePerNode.reportProgress(
"### Listing Killed/Failed applications while performing upgrade ####"
)
UpgradeLogger.reportProgress(
"### Listing Killed/Failed applications while performing upgrade ####",
False)
for app in Failed_Killed_apps:
queue = YARN.getQueueForApp(app)
logger.info(" %s running on %s queue Failed/Killed." %
(app, queue))
UpgradePerNode.reportProgress(
"#### %s running on %s queue Failed/Killed. ####" %
(app, queue))
UpgradeLogger.reportProgress(
"#### %s running on %s queue Failed/Killed. ####" %
(app, queue), False)
## TODO - Validate long running jobs
Rollingupgrade.verifyLongRunningJob(self.COMPONENTS_TO_TEST)
## KILL APPLICATIONS ####
YARN.killAllApplications(useYarnUser=True)
#logger.info("Running smoke tests after upgrade")
#Rollingupgrade.run_smokeTests(COMPONENTS_TO_TEST)
## TODO - call Teardown for long running jobs
Rollingupgrade.background_job_teardown(self.COMPONENTS_TO_TEST, None)
UpgradePerNode.reportProgress("### Completed upgrade ")
UpgradeLogger.reportProgress("### Completed upgrade ", True)
