def get_argus_admin_backup(cls): from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode localTestWorkDir1 = os.path.join(Config.getEnv('ARTIFACTS_DIR')) ## to have logic for repos backup UpgradePerNode.reportProgress( '#### getting repositories backup for ####') if cls.get_repositories_backup(localTestWorkDir1): UpgradePerNode.reportProgress( '#### repositories backup successful ####') ## to have logic for policies backup UpgradePerNode.reportProgress('#### getting policies backup ####') if cls.get_policies_backup(localTestWorkDir1): UpgradePerNode.reportProgress( '#### policies backup successful ####')
def upgrade_slave(cls, version, node, config=None, logText="Upgrade"): ''' Upgrade HDFS slave sevice: Datanode :param version: latestVersion. :param node: The node name where DN is running ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][HDFS][%s] Datanode %s for node %s to version %s started " % (logText, logText, node, version)) ipcPort = HDFS.getDataNodeIPCPort() cmd = "dfsadmin -shutdownDatanode %s:%s upgrade" % (node, ipcPort) exitcode, stdout = HDFS.runas(Config.get('hadoop', 'HDFS_USER'), cmd, env=None, logoutput=True, config=None, host=None, skipAuth=False) if exitcode != 0: UpgradePerNode.reportProgress( "[INFO][HDFS][%s] Datanode shutdownDatanode command failed for %s " % (logText, node)) HDFS.waitForDNDown(node, ipcPort, "ipc.client.connect.max.retries=1") from beaver.component.rollingupgrade.ruCommon import hdpSelect hdpSelect.changeVersion("hadoop-hdfs-datanode", version, node) HDFS.startDatanodes(config=config, nodes=[node]) cls.wait4DNLive(node) UpgradePerNode.reportProgress( "[INFO][HDFS][%s] Datanode %s for node %s to version %s finished " % (logText, logText, node, version))
def background_job_teardown(cls): ''' Cleanup for long running Yarn job ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][FLUME][BGJobTeardown] teardown for Long running job of Flume component started" ) if agent1.isalive(): logger.info("Terminating the first Flume agent") agent1.stop() if agent2.isalive(): logger.info("Terminating the second Flume agent") agent2.stop() for outdir in (cls._agent1_chkpt_dir, cls._agent1_data_dir, cls._agent2_chkpt_dir, cls._agent2_data_dir): if os.path.exists(outdir): shutil.rmtree(outdir) if os.path.isfile(cls._data_file): os.remove(cls._data_file) if os.path.isfile(cls._data_stop): os.remove(cls._data_stop)
def background_job_setup(cls, runSmokeTestSetup=True, config=None): ''' Setup for background long running job Upload Data to HDFS before Upgrade starts Creates /user/hrt_qa/ru-pig dir on HDFS Creates and Upload large data file to /user/hrt_qa/ru-pig/input/ :param runSmokeTestSetup: Runs smoke test setup if set to true ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "### Running Pig BackGround Job Setup ####") HDFS.deleteDirectory(cls._base_hdfs_dir) exit_code, stdout = HDFS.createDirectory(cls._base_hdfs_dir, user=cls._job_user, perm=777, force=True) ruAssert("Pig", exit_code == 0, '[BGJobSetup] could not create dir on hdfs.') HDFS.createDirectory(cls._hdfs_input_dir, force=True) srcFile = os.path.join(cls._artifacts_dir, 'pig-ru-input.txt') if os.path.exists(srcFile): os.remove(srcFile) tmpFile = os.path.join(cls._artifacts_dir, 'pig-ru-tmp-input.txt') if os.path.exists(tmpFile): os.remove(tmpFile) util.copyFileToAnotherFile(cls._golden_src_file, srcFile) util.copyFileToAnotherFile(srcFile, tmpFile) itr = 12 if Machine.isFlubber(): itr = 16 for i in range(itr): util.copyFileToAnotherFile(srcFile, tmpFile) util.copyFileToAnotherFile(tmpFile, srcFile) exit_code, stdout = HDFS.copyFromLocal(srcFile, cls._hdfs_input_path) ruAssert("Pig", exit_code == 0, '[BGJobSetup] Data Load failed') if runSmokeTestSetup: cls.smoke_test_setup()
def run_smoke_test(cls, smoketestnumber, config=None): ''' Run word count as HDFS smoke test - Create file of 4096 bytes using randomwriter job - Run wordcount job ''' logger.info("Running HDFS Smoke test") # make sure base hdfs dir is set. if not cls._base_hdfs_dir: cls._base_hdfs_dir = '/user/%s/test_rollingupgrade' % Config.get( 'hadoop', 'HADOOPQA_USER') from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][HDFS][Smoke] Smoke test for HDFS started ") HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -ls /user/hrt_qa", env=None, logoutput=True, config=None, host=None, skipAuth=False) if not cls._SmokeInputDir: cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput" SmokeOutputDir = cls._base_hdfs_dir + '/smokeHdfsOutput' + str( smoketestnumber) HDFS.deleteDirectory(SmokeOutputDir, Config.get('hadoop', 'HADOOPQA_USER')) jobCmd = 'jar %s wordcount \"-Dmapreduce.reduce.input.limit=-1\" \"-D%s=%s\" %s %s' % ( Config.get('hadoop', 'HADOOP_EXAMPLES_JAR'), "mapred.job.queue.name", cls._queue, cls._SmokeInputDir, SmokeOutputDir) exit_code, stdout = Hadoop.run(jobCmd) ruAssert("HDFS", exit_code == 0, "[Smoke] Hdfs smoketest failed") HDFS.deleteDirectory(SmokeOutputDir) ruAssert("HDFS", exit_code == 0, "[Smoke] could not delete: " + SmokeOutputDir) UpgradePerNode.reportProgress( "[INFO][HDFS][Smoke] Smoke test for HDFS Finished ")
def run_smoke_test(cls): ''' Run smoke test for yarn :param smoketestnumber: Used for unique output log location ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode host = Atlas.get_host() if not host: logger.error("No ACTIVE Atlas instance is available") else: logger.info("Active instance of Atlas is found: " + host) response, response_status = Atlas.create_entities(input_json_string=cls.entity_create_request_payload) if response_status != 201: UpgradePerNode.reportProgress("[FAILED][Atlas][Smoke] Entity creation failed = %d" % response_status) else: guid = response["definition"]["id"]["id"] logger.info("Entity creation was successful. GUID = %s" % guid) cls.guid_list.append(guid) UpgradePerNode.reportProgress("[INFO][Atlas][Smoke] Smoke test for Atlas component finished")
def doBackgroundJobSetup(cls, hdfs_test_dir): from beaver.component.hive import Hive from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode logger.info("Preparing the test setup for Hive background job") udfjar = os.path.join(Config.getEnv('WORKSPACE'), "tests", "hive", "hive-udf", "hive-udfs-0.1.jar") HDFS.createDirectory(hdfs_test_dir, user=cls._hdfs_user, perm='777', force=True) HDFS.copyFromLocal(udfjar, hdfs_test_dir) query = "drop function sleep; create function sleep as 'org.apache.hive.udf.generic.GenericUDFSleep' using jar 'hdfs://%s/hive-udfs-0.1.jar';" % hdfs_test_dir exit_code, stdout = Hive.runQuery(query) if exit_code != 0: UpgradePerNode.reportProgress( "[FAILED][Hive][Setup] Long running failed due to exitcode = %d" % exit_code) else: UpgradePerNode.reportProgress( "[PASSED][Hive][Setup] Long running finished successfully")
def ensure_nn_is_active(cls, timeout=11 * 60): from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode active_nn = None curr_time = int(time.time()) end_time = int(time.time()) + timeout while not active_nn and curr_time <= end_time: active_nn = HDFS.getNamenodeByState('active') # we we found active NN return it if active_nn: UpgradePerNode.reportProgress( "[INFO][HDFS] Active Namenode is %s" % active_nn) return # wait for 30s time.sleep(30) curr_time = int(time.time()) if not active_nn: UpgradePerNode.reportProgress("[FAILED][HDFS] No Active Namenode") assert active_nn return
def upgrade_slave(cls, version, node, config=None): ''' Upgrades Yarn slave services : NM :param version: Version to be upgraded to :param node: Slave Node :param config: Config location :return: ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][YARN][Upgrade] Nodemanager Upgrade for %s started " % node) logger.info("**** Beginning upgrade for nodemanager on %s ****" % node) MAPRED.stopTasktrackers(nodes=[node]) from beaver.component.rollingupgrade.ruCommon import hdpSelect hdpSelect.changeVersion("hadoop-yarn-nodemanager", version, node) # BUG-27328 # make sure we set the linux container executor permissions if Hadoop.isSecure() and not Machine.isWindows(): container_exec_cfg = os.path.join( Config.get('hadoop', 'HADOOP_CONF'), 'container-executor.cfg') container_exec = os.path.join( Config.get('hadoop', 'YARN_HOME').replace("client", "nodemanager"), 'bin', 'container-executor') cmd="chown root:hadoop %s %s ; chmod 400 %s ; chmod 6050 %s" % \ (container_exec_cfg, container_exec, container_exec_cfg, container_exec) Machine.runas(Machine.getAdminUser(), cmd, host=node, cwd=None, env=None, logoutput=True, passwd=Machine.getAdminPasswd()) MAPRED.startTasktrackers(config, nodes=[node]) logger.info("**** Ending upgrade for nodemanager on %s ****" % node) UpgradePerNode.reportProgress( "[INFO][YARN][Upgrade] Nodemanager Upgrade on %s Finished " % node)
def validate_correct_tar_used(cls, output): ''' Validates that correct tez tar ball is used :param output: Console output from application :return: True if corret tar ball is used otherwise returns False ''' from beaver.component.rollingupgrade.ruCommon import hdpRelease from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode currVersion = hdpRelease.getCurrentRelease("hadoop-client") pattern = "Using tez.lib.uris value from configuration: /hdp/apps/%s/tez/tez.tar.gz" % currVersion m = re.search(pattern, output) if m: logger.info("Tez application is using correct version") UpgradePerNode.reportProgress( "### [PASSED][Tez][Smoke] Tez smoke test is using correct version. Passed successfully ####" ) return True else: logger.info("Tez application is not using correct version") UpgradePerNode.reportProgress( "### [FAILED][Tez][Smoke] Tez smoke test is not using correct version. Validation Failed ####" ) return False
def background_job_setup(cls, components, runSmokeTestSetup=True, config=None): from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( '#### call for background-job setup for argus admin started ####') if runSmokeTestSetup: # componentList = ["hive","hbase","hdfs","knox","storm"] componentList = [] if Xa.isHdfsInstalled(): componentList.append('hdfs') if Xa.isHiveInstalled(): componentList.append('hive') if Xa.isHBaseInstalled(): componentList.append('hbase') if Xa.isKnoxInstalled(): componentList.append('knox') if Xa.isStormInstalled(): componentList.append('storm') cls.smoke_test_setup(componentList) UpgradePerNode.reportProgress( '#### call for background-job setup for argus admin done ####')
def run_smoke_test(cls, smoketestnumber, config=None): ''' Run smoke test for yarn :param smoketestnumber: Used for unique output log location ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][YARN][Smoke] Starting smoke test for Yarn") numNodes = len(MAPRED.getTasktrackers()) # run sleep job (exitCode, stdout) = HadoopJobHelper.runSleepJob(numOfMaps=numNodes, numOfReduce=numNodes, mapSleepTime=10, reduceSleepTime=10, extraJobArg=cls._jobArgs, runInBackground=False, config=config) # Make sure succeeded ruAssert("YARN", exitCode == 0, "[Smoke] YARN smoke failed") UpgradePerNode.reportProgress( "[INFO][YARN][Smoke] Smoke test for Yarn finished ") cls.run_rest_apis_test()
def background_job_setup(cls, runSmokeTestSetup=True, config=None): ''' Setup for background long running job :param runSmokeTestSetup: Runs smoke test setup if set to true ''' from beaver.component.sqoop import Sqoop from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][SQOOP][BGSetup] Background job set for sqoop is called") dbflavor = Sqoop.getDatabaseFlavor() if dbflavor == "oracle": cls._lrtest_tbl = cls._lrtest_tbl.upper() cls._smtest_tbl = cls._smtest_tbl.upper() logger.info( "*** Creating the test data for Sqoop long running test on dbflavorr= %s ***" % dbflavor) cls.generate_test_data(cls._hdfs_lrtest_dir, cls._test_lrrow_count) create_tbl_query = "create table %s (userid int, age int, primary key (userid));" % cls._lrtest_tbl cls.init_test_database(dbflavor) cls.run_database_query(create_tbl_query, dbflavor) if runSmokeTestSetup: logger.info("**** Running Sqoop Smoke Test Setup ****") cls.smoke_test_setup()
def run_second_background_job(cls, runSmokeTestSetup=False, config=None): ''' Runs long running distributed shell Yarn job after RM upgrade ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][YARN][BGJob2] Starting second background job for Yarn ") Local_Test_dir = os.path.join(Config.getEnv("WORKSPACE"), "tests", "rolling_upgrade", "yarn") Multi_Version_App_Dir = os.path.join(Local_Test_dir, "data") Mapper = os.path.join(Multi_Version_App_Dir, "mvMapper.py") Reducer = os.path.join(Multi_Version_App_Dir, "mvReducer.py") HadoopJobHelper.runSleepJob(numOfMaps=1, numOfReduce=1, mapSleepTime="100000", reduceSleepTime="100", extraJobArg=cls._jobArgs, runInBackground=True, config=config, directoutput=False) #(jobID, STREAMING_APP_ID) = HadoopJobHelper.checkAndRunStreamJob(Mapper, Reducer, cls._hdfs_input, cls._hdfs_output, files = Multi_Version_App_Dir, config = config, forRU=True) logger.info("Second Long Running job for YARN has started") return 1
def write_hbase_site(cls, config=None): ''' Obtain hbase-site.xml from Slider HBase and write to modified config path ''' global new_conf_path from beaver.component.sliderhbase import SliderHBase from beaver.component.slider import Slider tmpHBaseConfFile = os.path.join(Machine.getTempDir(), "hbase-site.xml") if Hadoop.isSecure(): hbasesite = Slider.registry( "sliderhbase", flags="--getconf hbase-site --out " + tmpHBaseConfFile, format="xml", user=cls.HBASE_USER, userFlag=cls.HBASE_USER ) else: hbasesite = Slider.registry( "sliderhbase", flags="--getconf hbase-site --out " + tmpHBaseConfFile, format="xml", user=cls.HBASE_USER ) propertyMap = {'hbase.tmp.dir': '/tmp/hbase-tmp'} generated_hbase_conf = os.path.join(new_conf_path, "hbase-site.xml") # QE-3108 # if the hbase app is created successfully, a /tmp/hbase-site.xml will be # generated from the app. Otherwise if the /tmp/hbase-site.xml is missing # it means the slider app creation fails from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode if os.path.isfile(generated_hbase_conf): util.writePropertiesToConfigXMLFile(tmpHBaseConfFile, generated_hbase_conf, propertyMap) else: UpgradePerNode.reportProgress( "[FAILED][Slider][background] Slider hbase background setup failed due to hbase-site.xml not generated" )
def upgrade_master(cls, version, config=None): ''' Upgrades Master services: :param version: Version to be upgraded to :param config: Config location ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode function_name = inspect.stack()[0][3] from beaver.component.rollingupgrade.ruCommon import hdpSelect from beaver.component.oozie import Oozie cls.background_job_when_master_upgrade() oozieServers = Oozie.getOozieServers() for server in oozieServers: UpgradePerNode.reportProgress("### Oozie %s has started %s ####" % (server, function_name)) Oozie.resetOozie('stop', host=server) time.sleep(60) hdpSelect.changeVersion("oozie-server", version, server) Oozie.resetOozie('start', configDir=config, host=server) time.sleep(60) UpgradePerNode.reportProgress("### Oozie %s has finished %s ####" % (server, function_name)) cls.verify_background_job_when_master_upgrade() cls.background_job_teardown_when_master_upgrade()
def run_background_job(cls, runSmokeTestSetup=True, config=None): ''' Runs background long running Yarn Job :param runSmokeTestSetup: Runs smoke test setup if set to true :param config: expected configuration location :return: Total number of long running jobs started ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][YARN][BGJob] Starting background job for Yarn ") Local_Test_dir = os.path.join(Config.getEnv("WORKSPACE"), "tests", "rolling_upgrade", "yarn") Multi_Version_App_Dir = os.path.join(Local_Test_dir, "data") HDFS.deleteDirectory(cls._hdfs_output, user=Config.get('hadoop', 'HADOOPQA_USER')) Mapper = "data/mvMapper.py" Reducer = "data/mvReducer.py" # Launch job (jobID, STREAMING_APP_ID) = HadoopJobHelper.checkAndRunStreamJob( Mapper, Reducer, cls._hdfs_input, cls._hdfs_output, files=Multi_Version_App_Dir, config=config, forRU=True, extraJobArg=cls._jobArgs, env=None, sleepTimeAfterJobSubmission=60) cls._background_job_appId = STREAMING_APP_ID cls._background_job_jobId = jobID logger.info("Background job started, application ID: " + STREAMING_APP_ID + " job ID: " + jobID) logger.info("Start second long running job for Yarn") num_sec_bkjob = cls.run_second_background_job(config=config) return 2
def runCommand(cls, user, cmd, runInBackground=False): """ Function to run pig cmd as a certain user :param user: :param cmd: :return: """ try: from beaver.component.pig import Pig except ImportError: ## Import fails when Pig is not installed on this machine. Nothing to do return 0, "" if runInBackground: cls._process = Pig.runas(user, cmd, runInBackground=True) else: exit_code, stdout = Pig.runas(user, cmd, runInBackground=False) from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode if exit_code == 0: UpgradePerNode.reportProgress( "### [PASSED][Pig][Smoke] Pig Smoke test passed ####") else: UpgradePerNode.reportProgress( "### [FAILED][Pig][Smoke] Pig Smoke test Failed ####")
def downgrade_master(cls, version, config=None): ''' Downgrade Master services :param version: Version to be downgraded to :param config: Configuration location ''' from beaver.component.rollingupgrade.ruCommon import hdpSelect from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress("### Knox Downgrade started ####") #Stop knox gateway and apacheDS Knox.stopKnox() Knox.stopLdap() node = Config.get('knox', 'KNOX_HOST').split(',')[0] #Switch to the new version hdpSelect.changeVersion("knox-server", version, node) #Start apacheDs and knox gateway service Knox.startLdap() Knox.startKnox() time.sleep(10) UpgradePerNode.reportProgress("### Knox Downgrade Finished ####")
def get_current_release(cls): ''' get current version in action :return: current version in use Like 2.2.0.1-177 ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode argus_root = "/usr/lib/xapolicymgr" linkname = os.path.join(argus_root) UpgradePerNode.reportProgress('#### linkname = %s ####' % linkname) if os.path.islink(linkname): (head, tail) = os.path.split(os.readlink(linkname)) UpgradePerNode.reportProgress('#### head = %s ####' % head) UpgradePerNode.reportProgress('#### tail = %s ####' % tail) argus_version = tail.split('-', 1)[1][6:len(tail.split('-', 1)[1])] return argus_version else: UpgradePerNode.reportProgress( "#### No valid symlink for argus ####") return None
def run_smoke_test(cls, smoketestnumber, config=None): ''' Run MRRSleep tez job :param smoketestnumber: Used for unique output log location ''' logger.info("***** Starting MRRSleep job in Tez ******") from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress("### Starting smoke job for tez ####") numNodes = len(MAPRED.getTasktrackers()) # Post a sleep job (exit_code, stdout) = Tez.runTezExampleJar( 'mrrsleep "-Dmapreduce.framework.name=yarn-tez" "-Dtez.queue.name=%s" -m %d -r %d -ir 3 -irs 5 -mt 10 -rt 10' % (cls._queue, numNodes, numNodes)) # Make sure succeeded if exit_code == 0: UpgradePerNode.reportProgress( "### [PASSED][Tez][Smoke] smoke job for tez passed ####") else: UpgradePerNode.reportProgress( "### [FAILED][Tez][Smoke] smoke job for tez failed ####") time.sleep(10) logger.info("***** Completed MRRSleep job in Tez ******") UpgradePerNode.reportProgress("### smoke test finished for tez ####") cls.validate_correct_tar_used(stdout)
def verifyLongRunningJob(cls, config=None): ''' Verify long running background job after it finishes :return: ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode retVal, checkContent = HDFS.cat(cls._hdfs_output + '/part-00000') #assert retVal == 0 if retVal == 0: UpgradePerNode.reportProgress( "[PASSED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, retVal = 0. Successful check " ) else: UpgradePerNode.reportProgress( "[FAILED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, retVal != 0. Failed check " ) #assert 'true' in checkContent, "Stream job returns false: " + checkContent if 'true' in checkContent: UpgradePerNode.reportProgress( "[PASSED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, true in checkContent. Successful check " ) else: UpgradePerNode.reportProgress( "[FAILED][YARN][BGJobCheck] verifyLongRunning Job for Yarn, true not in checkContent. Failed check " ) #verify application's attempt doesn't increase and no failed tasks. appID = cls._background_job_appId jobID = cls._background_job_jobId # temporarily skipping check #assert YARN.getNumAttemptsForApp(appID) == 1 #YARN.verifyMRTasksCount(jobID, appID, 0, skipAssert=True) from beaver.component.rollingupgrade.ruCommon import hdpRelease Version_Num = hdpRelease.getCurrentRelease("hadoop-client") cls.checkClasspathVersion(Version_Num, config)
def downgrade_master(cls, version, config=None): ''' Downgrade Master services :param version: Version to be downgraded to :param config: Configuration location ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode function_name = inspect.stack()[0][3] UpgradePerNode.reportProgress("### Oozie %s has started ####" % function_name) from beaver.component.rollingupgrade.ruCommon import hdpSelect from beaver.component.oozie import Oozie oozieServers = Oozie.getOozieServers() for server in oozieServers: UpgradePerNode.reportProgress("### Oozie %s has started %s ####" % (server, function_name)) Oozie.resetOozie('stop', host=server) time.sleep(60) hdpSelect.changeVersion("oozie-server", version, server) Oozie.resetOozie('start', configDir=config, host=server) time.sleep(60) UpgradePerNode.reportProgress("### Oozie %s has finished %s ####" % (server, function_name))
def run_smoke_test(cls, smoketestnumber, config=None): ''' Run smoke test for yarn :param smoketestnumber: Used for unique output log location ''' global new_conf_path global buildNo from beaver.component.hbase import HBase # Run slider agent labels funtests from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress("### Slider smoke test started ####") exit_code, stdout = HBase.runShellCmds(cls.SCAN_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path) UpgradePerNode.reportProgress("### Slider smoke test scanned " + cls.TABLE_NAME) hbase_ver = HBase.getVersion(configPath=new_conf_path) if buildNo not in hbase_ver: UpgradePerNode.reportProgress("### smoke test failed: " + buildNo + " not found in " + hbase_ver) else: UpgradePerNode.reportProgress("### smoke test passed: " + buildNo + " found in " + hbase_ver) UpgradePerNode.reportProgress("scanned " + cls.TABLE_NAME) TABLE_NM = cls.id_generator(10) CREATE_TABLE_CMD = ["create '" + TABLE_NM + "', 'family1', 'family2', 'family3'"] exit_code, stdout = HBase.runShellCmds(CREATE_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path) if exit_code == 0: UpgradePerNode.reportProgress("created " + TABLE_NM) else: UpgradePerNode.reportProgress("creation of " + TABLE_NM + "failed") if not cls._LOCAL_WORK_DIR: UpgradePerNode.reportProgress( "[FAILED][Slider][Smoke] Slider smoke test failed due to Slider source code unavailability" ) return exit_code, stdout = Maven.run( "verify -Dslider.conf.dir=%s " "-Dhadoop.version=%s " "-Dprivate.repo.url=%s " "-Dit.test=AppsUpgradeIT" % (cls._LOCAL_CONF_DIR, Hadoop.getVersion(), Maven.getPublicRepoUrl()), cwd=os.path.join(cls._LOCAL_WORK_DIR, 'slider-funtest') ) testresults = {} testResultFiles = [] TEST_RESULT = {} # get a list of all the test result files for name, dirs, files in os.walk(cls._LOCAL_WORK_DIR): if os.path.basename(name) == 'target': # Add in each failsafe-report we find -- this test only runs failsafe reports testResultFiles.extend(util.findMatchingFiles(os.path.join(name, 'failsafe-reports'), 'TEST-*.xml')) for resultFile in testResultFiles: testresults.update(util.parseJUnitXMLResult(resultFile)) for key, value in testresults.items(): TEST_RESULT[key] = value logger.info("=======================") logger.info(TEST_RESULT) logger.info("=======================") TestCases = TEST_RESULT.keys() for testcase in TestCases: result = TEST_RESULT[testcase]['result'] if result == "pass": UpgradePerNode.reportProgress("[PASSED][Slider][Smoke] Slider smoke test passed") else: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider smoke test failed")
def smoke_test_setup(cls): ''' Setup required to run Smoke test ''' from beaver.component.slider import Slider from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode HADOOPQA_USER = Config.get('hadoop', 'HADOOPQA_USER') UpgradePerNode.reportProgress("### Starting set up for Slider smoke test ####") if Hadoop.isSecure(): keytabFile = Machine.getHeadlessUserKeytab(HADOOPQA_USER) kinitloc = Machine.which("kinit", "root") cmd = "%s -k -t %s %s" % (kinitloc, keytabFile, Machine.get_user_principal(HADOOPQA_USER)) exit_code, stdout = Machine.run(cmd) if exit_code != 0: UpgradePerNode.reportProgress("### smoke test setup for Slider failed due to kinit failed ####") # Local directory in artifacts that we'll run tests from # it is possible the download_source() will fail try: cls._LOCAL_WORK_DIR = Slider.download_source(useHDPBaseRepoFile=False, isRUcluster=True) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) except TypeError as err: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source download fail.") return if not cls._LOCAL_WORK_DIR: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source Code missing.") return # Local conf directory with modified conf for tests if not os.path.exists(cls._LOCAL_CONF_DIR): os.makedirs(cls._LOCAL_CONF_DIR) slider_conf = os.path.join(Slider.getSliderHome(), 'conf') logger.info("slider_conf = %s" % slider_conf) logger.info("os path exist slider_conf = %s" % os.path.exists(slider_conf)) if os.path.exists(slider_conf): Machine.copy( os.path.join(slider_conf, 'log4j.properties'), os.path.join(cls._LOCAL_CONF_DIR, 'log4j.properties') ) Machine.copy( os.path.join(slider_conf, 'slider-env.sh'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-env.sh') ) else: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Conf %s missing" % slider_conf) return if Hadoop.isSecure(): util.writePropertiesToConfigXMLFile( os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), { "slider.funtest.enabled": "true", "slider.test.agent.enabled": "true", "HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'), "slider.am.keytab.local.path": Machine.getHeadlessUserKeytab(HADOOPQA_USER), "slider.keytab.principal.name": Machine.get_user_principal(HADOOPQA_USER) } ) else: util.writePropertiesToConfigXMLFile( os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), { "slider.funtest.enabled": "true", "HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'), "slider.test.agent.enabled": "true" } ) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) # Check that the precondition is met and the source is available if not os.path.exists(cls._LOCAL_WORK_DIR) or not os.path.exists(os.path.join(cls._LOCAL_WORK_DIR, 'pom.xml')): logger.info('Slider source does not appear to exist at %s' % (cls._LOCAL_WORK_DIR)) UpgradePerNode.reportProgress( "### Slider source does not appear to exist at %s ####" % (cls._LOCAL_WORK_DIR) ) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) if cls._LOCAL_WORK_DIR == None: logger.info("ERROR: cls._LOCAL_WORK_DIR is None") # Install first so isolated modules can be tested exit_code, stdout = Maven.run( "clean install -DskipTests " "-Dhadoop.version=%s " "-Dprivate.repo.url=%s " % (Hadoop.getVersion(), Maven.getPublicRepoUrl()), cwd=cls._LOCAL_WORK_DIR ) if exit_code != 0: UpgradePerNode.reportProgress("### Error installing Slider source : %d: %s ####" % (exit_code, stdout)) else: UpgradePerNode.reportProgress("### Slider source install passed ####")
def upgrade_argus_admin(cls, latestVersion, config, currVersion): from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress('#### install argus admin. ####') # 'knox.crt' # 'cacertswithknox' node = Xa.getPolicyAdminHost() user = Machine.getAdminUser() xa_admin_stop_cmd = 'sudo service ranger-admin stop' xa_admin_start_cmd = 'sudo service ranger-admin start' ranger_old_version = '/usr/hdp/' + currVersion + '/ranger-admin/' ranger_new_version = '/usr/hdp/' + latestVersion + '/ranger-admin/' localTestWorkDir1 = os.path.join(Config.getEnv('ARTIFACTS_DIR')) knox_cert_files = '{knox.crt,cacertswithknox}' source_files_to_copy = os.path.join(ranger_old_version, knox_cert_files) command_copy_knoxcerts = 'cp -f %s %s' % (source_files_to_copy, localTestWorkDir1) exit_code, stdout = Machine.runas(user, command_copy_knoxcerts, host=node, logoutput=True) logger.info( '*************************** admin copy command_copy_knoxcerts exit_code = ' + str(exit_code)) logger.info( '*************************** admin copy command_copy_knoxcerts stdout = ' + str(stdout)) exit_code, stdout = Machine.runas(user, xa_admin_stop_cmd, host=node, logoutput=True) logger.info('****************** xa admin stop exit_code = ' + str(exit_code)) source_properties_file = os.path.join(ranger_old_version, 'install.properties') destination_properties_file = os.path.join(ranger_new_version, 'install.properties') command = 'cp -f %s %s' % (source_properties_file, destination_properties_file) exit_code, stdout = Machine.runas(user, command, host=node, logoutput=True) logger.info( '*************************** admin copy command exit_code = ' + str(exit_code)) logger.info( '*************************** admin copy command stdout = ' + str(stdout)) command = '(cd %s && export JAVA_HOME=%s && ./setup.sh)' % ( ranger_new_version, Machine.getJavaHome()) UpgradePerNode.reportProgress( '#### ranger-admin: installing new version with command %s ###' % command) exit_code, stdout = Machine.runas(user, command, host=node, logoutput=True) if exit_code == 0 and 'Installation of XASecure PolicyManager Web Application is completed.' in stdout: UpgradePerNode.reportProgress( '#### ranger-admin: installation successful ###') from beaver.component.rollingupgrade.ruCommon import hdpSelect hdpSelect.changeVersion("ranger-admin", latestVersion, node) logger.info( '*************************** ranger-admin: hdp-select to new version done = ' ) source_files_to_copy = os.path.join(localTestWorkDir1, knox_cert_files) command_copy_knoxcerts = 'cp -f %s %s' % (source_files_to_copy, ranger_new_version) exit_code, stdout = Machine.runas(user, command_copy_knoxcerts, host=node, logoutput=True) logger.info( '*************************** admin copy command_copy_knoxcerts back exit_code = ' + str(exit_code)) logger.info( '*************************** admin copy command_copy_knoxcerts back stdout = ' + str(stdout)) UpgradePerNode.reportProgress( '#### ranger-admin: starting new version with command %s ###' % xa_admin_start_cmd) exit_code, stdout = Machine.runas(user, xa_admin_start_cmd, host=node, logoutput=True) logger.info('****************** xa admin start exit_code = ' + str(exit_code)) logger.info('****************** xa admin start stdout = ' + str(stdout)) if exit_code == 0: UpgradePerNode.reportProgress( '#### ranger-admin: new version started successfully ####') else: UpgradePerNode.reportProgress( '#### ranger-admin: failed to start new version! exit_code=%d ####' % exit_code) else: logger.info( '****************** setup.sh script failed for admin ******* ') UpgradePerNode.reportProgress( '#### ranger-admin: installation of new version failed! exit_code=%d ###' % exit_code)
def get_repositories_backup(cls, localTestWorkDir1): from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( '#### saving existing repositories for backup ####') cwObject = XACWTestCase() baseUrl = 'http://' + xa_testenv.getEnv( 'XA_ADMIN_HOST', 'localhost') + ':' + str( xa_testenv.getEnvInt('XA_ADMIN_PORT', 6080)) urlReposUsingApi = baseUrl + '/service/public/api/repository/' username_password = xa_testenv.getEnv( 'XA_ADMIN_USER', 'admin') + ':' + xa_testenv.getEnv( 'XA_ADMIN_PASS', 'admin') headerResponse, response = cwObject.callPyCurlRequest( url=urlReposUsingApi, data=None, method='get', usernamepassword=username_password) UpgradePerNode.reportProgress( '#### get all repositories, headerResponse is : %s ####' % headerResponse) UpgradePerNode.reportProgress( '#### get all repositories, response is : %s ####' % response) if 'HTTP/1.1 200 OK' in headerResponse: file_for_repositories = os.path.join(localTestWorkDir1, 'repositories.bak') UpgradePerNode.reportProgress( '#### saving repositories to path = %s ####' % str(file_for_repositories)) openReposFile = open(file_for_repositories, 'wb') openReposFile.write(response) openReposFile.close() else: UpgradePerNode.reportProgress( '#### unable to save repositories for backup ####') return False UpgradePerNode.reportProgress( '#### repositories back-up file saved successfully ####') return True
def verifySchemaFunctionality(cls): ''' We verify that the system can operate with SCHEMA functionality. ''' from beaver.component.phoenix import Phoenix from beaver.component.hbase import HBase from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode HBASE_HOME = Config.get('hbase', 'HBASE_HOME') HBASE_CONF_DIR = os.path.join(HBASE_HOME, "conf") #We verify the schema functionality. HBASE_CHANGES = {} HBASE_CHANGES['hbase-site.xml'] = {} HBASE_CHANGES['hbase-site.xml'][ 'phoenix.schema.isNamespaceMappingEnabled'] = 'true' HBASE_CHANGES['hbase-site.xml'][ 'phoenix.schema.mapSystemTablesToNamespace'] = 'true' TEST_TABLE_A = 'Table_A' SCHEMA_1 = 'SCHEMA_1' masterNodes = HBase.getAllMasterNodes() regionNodes = HBase.getRegionServers() hbase_allnodes = masterNodes + regionNodes gateway_node = Machine.getfqdn() if gateway_node not in hbase_allnodes: hbase_allnodes.append(gateway_node) HBase.stopHBaseCluster() HBase.modifyConfig(changes=HBASE_CHANGES, nodeSelection={'nodes': hbase_allnodes}) util.copy_back_to_original_config(HBase.getModifiedConfigPath(), HBASE_CONF_DIR, file_list=["hbase-site.xml"], node_list=hbase_allnodes) HBase.startHBaseCluster(HBase.getModifiedConfigPath()) #We grant permissions to all tables. Phoenix.grantPermissionsToSystemTables(schemaFunctionalityEnabled=True) #We check that we can still query the original table. cls.verifyBasicTable() #We check that we can create/query schemas. exit_code, stdout = Phoenix.runSQLLineCmds( 'CREATE SCHEMA IF NOT EXISTS %s;' % SCHEMA_1) if exit_code != 0: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Creation of schema %s failed due to exitcode = %s " % (SCHEMA_1, exit_code)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Schema creation %s succeeded." % (SCHEMA_1)) #we create tables inside that schema primaryKey = {'name': 'ID', 'type': 'BIGINT'} columns = [{ 'name': 'FirstName', 'type': 'VARCHAR(30)' }, { 'name': 'SecondName', 'type': 'VARCHAR(30)' }, { 'name': 'City', 'type': 'VARCHAR(30)' }] exit_code, stdout = Phoenix.createTable(SCHEMA_1 + '.' + TEST_TABLE_A, primaryKey, columns) if exit_code != 0: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Table creation %s on schema %s failed due to exitcode = %s " % (TEST_TABLE_A, SCHEMA_1, exit_code)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Table creation %s on schema %s succeeded." % (TEST_TABLE_A, SCHEMA_1)) #We insert some data into the table through upsert. for i in range(0, 5): exit_code, stdout = Phoenix.runSQLLineCmds( 'UPSERT INTO %s VALUES (%s, "name_%s","secondName_%s","city_%s");' % (SCHEMA_1 + '.' + TEST_TABLE_A, str(i), str(i), str(i), str(i))) if exit_code != 0: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Table UPSERT %s on schema %s failed due to exitcode = %s " % (TEST_TABLE_A, SCHEMA_1, exit_code)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Table UPSERT %s on schema %s succeeded." % (TEST_TABLE_A, SCHEMA_1)) #We verify that the data has been correctly inserted exit_code, stdout = Phoenix.runSQLLineCmds( 'SELECT * FROM %s WHERE ID=3;' % (SCHEMA_1 + '.' + TEST_TABLE_A)) if exit_code != 0: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Table SELECT %s on schema %s failed due to exitcode = %s " % (TEST_TABLE_A, SCHEMA_1, exit_code)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Table SELECT %s on schema %s succeeded." % (TEST_TABLE_A, SCHEMA_1)) if stdout.find('name_3') == -1 or stdout.find( 'secondName_3') == -1 or stdout.find('city_3') == -1: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Table SELECT %s on schema %s returned the wrong results: %s" % (TEST_TABLE_A, SCHEMA_1, stdout)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Table SELECT %s on schema %s succeeded." % (TEST_TABLE_A, SCHEMA_1)) #We verify that we can drop the schemas with tables on it. exit_code, stdout = Phoenix.runSQLLineCmds('DROP SCHEMA %s;' % SCHEMA_1) if exit_code != 0: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Schema drop failed due to exitcode = %s " % (exit_code)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Schema drop succeeded.") #We verify that the schema has been dropped. exit_code, stdout = Phoenix.runSQLLineCmds( 'SELECT TABLE_NAME FROM SYSTEM.CATALOG WHERE SCHEMA = %s' % SCHEMA_1, outputFormat='xmlattr') if exit_code != 0: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Schema drop failed due to exitcode = %s " % (exit_code)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Schema drop succeeded.") if stdout.find(TEST_TABLE_A) != 0: UpgradePerNode.reportProgress( "[FAILED][PHOENIX][Smoke] Table %s did not drop on drop schema command " % (TEST_TABLE_A)) else: UpgradePerNode.reportProgress( "[PASSED][PHOENIX][Smoke] Table %s successfuly dropped." % TEST_TABLE_A)
def prepare_and_start_long_running_jobs(self): ############################Prepare and start long running jobs self.find_components_to_test() import beaver.component.rollingupgrade.ruSetup as ruSetup ruSetup.COMPONENTS_TO_FLIP = self.COMPONENTS_TO_TEST ruSetup.COMPONENTS_AFFECTED = self.COMPONENTS_TO_TEST ruSetup.COMPONENTS_TO_TEST = self.COMPONENTS_TO_TEST ruSetup.COMPONENTS_TO_IMPORT = self.COMPONENTS_TO_TEST from beaver.component.hadoop import HDFS if "slider" in self.COMPONENTS_TO_TEST: from beaver.component.rollingupgrade.ruSlider import ruSlider from beaver.component.rollingupgrade.ruCommon import Rollingupgrade from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode HDFS.createUserDirWithGroup('/user/' + Config.get('hadoop', 'HADOOPQA_USER'), 'hdfs', Config.get('hadoop', 'HADOOPQA_USER'), 'hdfs', diffGroup=True) logger.info(Config.get('hadoop', 'HADOOPQA_USER')) logger.info( "1===========================================================================" ) self.setup_ru_cluster() logger.info( "2===========================================================================" ) logger.info( "3===========================================================================" ) DN = HDFS.getDatanodes() # Find core components (HDFS, YARN, HBase) if exist. #core_components = UpgradePerNode.find_existing_core_components(COMPONENTS_TO_TEST) logger.info( "4===========================================================================" ) #Prepare and save state before upgrade Rollingupgrade.ru_prepare_save_state_for_upgrade( self.COMPONENTS_TO_TEST) logger.info( "5===========================================================================" ) # Run setup for background Jobs for all components Rollingupgrade.background_job_setup(self.COMPONENTS_TO_TEST, config=None) logger.info("Running smoke tests before upgrade") Rollingupgrade.run_smokeTests(self.COMPONENTS_TO_TEST) logger.info( "6===========================================================================" ) # Starts Long running background Jobs for all components numBackgroundJobs = Rollingupgrade.run_longRunning_Application( self.COMPONENTS_TO_TEST, config=None) logger.info( "7===========================================================================" ) logger.info( "Total number of long running background jobs before starting upgrade is %s" % numBackgroundJobs) UpgradePerNode.reportProgress( "### Just started %s background jobs ###" % numBackgroundJobs) UpgradeLogger.reportProgress( "### Just started %s background jobs ###" % numBackgroundJobs, True)
def validate_lr_job(self): #################################################Finsih long running jobs ### Need to stop HDFS Falcon,Yarn long runningJobs #### # create flagFile to kill HDFS background job from beaver.component.hadoop import HDFS, YARN from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode from beaver.component.rollingupgrade.ruCommon import Rollingupgrade TEST_USER = Config.get('hadoop', 'HADOOPQA_USER') createCmd = "dfs -touchz " + UpgradePerNode._HDFS_FLAG_FILE exit_code, output = HDFS.runas(TEST_USER, createCmd) logger.info( "8===========================================================================" ) if self.DO_DOWNGRADE: self.removeComponentFromTest("falcon") if "falcon" in self.COMPONENTS_TO_TEST: from beaver.component.rollingupgrade.ruFalcon import ruFalcon ruFalcon.stopFalconLongRunningJob() if "yarn" in self.COMPONENTS_TO_TEST: from beaver.component.rollingupgrade.ruYarn import ruYARN ruYARN.stopYarnLongRunningJob() # if "hive" in self.COMPONENTS_TO_TEST: # from beaver.component.rollingupgrade.ruHive import ruHive # ruHive.stopHiveLongRunningJob() if "slider" in self.COMPONENTS_TO_TEST: from beaver.component.rollingupgrade.ruSlider import ruSlider ruSlider.stopSliderLongRunningJob() if "storm-slider" in self.COMPONENTS_TO_TEST: from beaver.component.rollingupgrade.ruStorm import ruStorm ruStorm.teardown_storm_slider_app() logger.info( "9===========================================================================" ) ## TODO - wait for long running jobs to finish isZero = YARN.waitForZeroRunningApps() logger.info( "10===========================================================================" ) if isZero: UpgradePerNode.reportProgress("#### None apps are running. ####") UpgradeLogger.reportProgress("#### None apps are running. ####", True) else: UpgradePerNode.reportProgress( "#### Check Failed. some apps are running. ####") UpgradeLogger.reportProgress( "#### Check Failed. some apps are running. ####", False) #assert isZero, "all long running jobs are not finished" ### List down Failed/Killed applications #### Failed_Killed_apps = YARN.getFailedKilledAppList() UpgradePerNode.reportProgress( "### Listing Killed/Failed applications while performing upgrade ####" ) UpgradeLogger.reportProgress( "### Listing Killed/Failed applications while performing upgrade ####", False) for app in Failed_Killed_apps: queue = YARN.getQueueForApp(app) logger.info(" %s running on %s queue Failed/Killed." % (app, queue)) UpgradePerNode.reportProgress( "#### %s running on %s queue Failed/Killed. ####" % (app, queue)) UpgradeLogger.reportProgress( "#### %s running on %s queue Failed/Killed. ####" % (app, queue), False) ## TODO - Validate long running jobs Rollingupgrade.verifyLongRunningJob(self.COMPONENTS_TO_TEST) ## KILL APPLICATIONS #### YARN.killAllApplications(useYarnUser=True) #logger.info("Running smoke tests after upgrade") #Rollingupgrade.run_smokeTests(COMPONENTS_TO_TEST) ## TODO - call Teardown for long running jobs Rollingupgrade.background_job_teardown(self.COMPONENTS_TO_TEST, None) UpgradePerNode.reportProgress("### Completed upgrade ") UpgradeLogger.reportProgress("### Completed upgrade ", True)