def submit_storm_hdfs_topology(cls, tcId, className, args, topologyName, useStandaloneCmd): """ Name: Storm-HDFS Topologies Description: Testing all 4 storm-hdfs topologies in distributed mode 1. SequenceFileTopology 2. HdfsFileTopology 3. TridentFileTopology 4. TridentSequenceTopology Steps to verify: 1. create necessary input/output dir location if needed 2. Run storm topology in distributed mode using "storm <jar> <classname> <args>" 3. Verify expected output from hdfs output dir 4. kill the topology """ exit_code, stdout = Storm.runStormHdfsTopology( TARGET_HDFS_STORM_JAR, className, args, None, logoutput=True, inBackground=False, useStandaloneCmd=useStandaloneCmd) ruAssert("Storm", exit_code == 0, "[StormHDFSSubmit] %s Failed" % (tcId))
def getHiveQueryOutput(cls, query, willRunMR=True, delim=",", useStandaloneCmd=True): from beaver.component.hive import Hive hiveconf = {} if willRunMR: hiveconf = { 'hive.input.format': 'org.apache.hadoop.hive.ql.io.HiveInputFormat', 'hive.vectorized.execution.enabled': 'false', 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DbTxnManager', 'hive.support.concurrency': 'true' } exit_code, stdout, stderr = Hive.runQuery( cls.get_set_queue_cmd(useStandaloneCmd) + query, hiveconf=hiveconf, stderr_as_stdout=False) ruAssert("Storm", exit_code == 0, "[HiveQueryOutput] Failed to run Hive query [%s]" % query) return stdout.replace('\t', delim)
def run_client_smoketest(cls, config=None, env=None): ''' Run wordcount Job passing env variables :param config: Configuration location :param env: Set Environment variables ''' logger.info("**** Running HDFS CLI Test ****") from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][HDFS][ClientSmoke] CLI test for HDFS started ") if not cls._SmokeInputDir: cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput" SmokeOutputDir = cls._base_hdfs_dir + '/smokeHdfsOutput_cli' HDFS.deleteDirectory(SmokeOutputDir, Config.get('hadoop', 'HADOOPQA_USER')) jobCmd = 'jar %s wordcount \"-Dmapreduce.reduce.input.limit=-1\" \"-D%s=%s\" %s %s' % ( Config.get('hadoop', 'HADOOP_EXAMPLES_JAR'), "mapred.job.queue.name", cls._queue, cls._SmokeInputDir, SmokeOutputDir) exit_code, stdout = Hadoop.run(jobCmd, env=env) ruAssert("HDFS", exit_code == 0, "[ClientSmoke] Hdfs smoketest failed") HDFS.deleteDirectory(SmokeOutputDir) ruAssert("HDFS", exit_code == 0, "[ClientSmoke] could not delete: " + SmokeOutputDir) UpgradePerNode.reportProgress( "[INFO][HDFS][ClientSmoke] CLI test for HDFS Finished ")
def runCommand(cls, user, cmd, ignoreError=True): """ Function to run falcon commands cmd as a certain user :param user: :param cmd: :return: """ try: from beaver.component.falcon import Falcon except ImportError: ## Import fails when Falcon is not installed on this machine. Nothing to do return 0, "" exit_code, output = Falcon.runas(user, cmd) if ignoreError: if exit_code != 0: #logger.info("Warning (ignoreError=True): Non-zero exit code when running command " + cmd + " as user " + user) from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[FAILED][Falcon] Warning (ignoreError=True): Non-zero exit code when running command %s as user %s " % (cmd, user)) else: ruAssert( "Falcon", exit_code == 0, "[RunCommand] Could not run command " + cmd + " as user " + user) return exit_code, output
def ru_rollback_state(cls): ''' Saved state is rolled back - upgrade is abandonded NOTE: this command will not return until namenode shuts down :return: ''' logger.info("[INFO][HDFS][Upgrade] HA Journal Node Upgrade Started ") hdfs_user = Config.get('hadoop', 'HDFS_USER') nodes = [] nodes.append(HDFS.getNamenodeByState('standby')) nodes.append(HDFS.getNamenodeByState('active')) logger.info("[INFO][HDFS][Upgrade] HA Namenode Upgrade Started") for node in nodes: HDFS.resetZkfc('stop', hosts=node.split()) HDFS.resetNamenode('stop', host=node) HDFS.resetNamenode('start', config=None, host=nodes[0], option=" -rollingUpgrade rollback") HDFS.resetZkfc('start', hosts=nodes[0].split()) # lets make sure the NN is out of safemode before we proceed to the next namenode HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[0]) command = "sudo su - -c 'hadoop namenode -bootstrapStandby -force' hdfs" (exitcode, stdout) = Machine.runas(Machine.getAdminUser(), command, nodes[1], None, None, "True", Machine.getAdminPasswd()) ruAssert("HDFS", exitcode == 0, "hadoop namenode -bootstrapStandby -force") HDFS.resetNamenode('start', config=None, host=nodes[1], option="") HDFS.resetZkfc('start', hosts=nodes[1].split()) # lets make sure the NN is out of safemode before we proceed to the next namenode HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[1])
def ru_downgrade_state(cls): ''' Downgrades Namenode A downgrade is done - may need to convert state to previous version or state is compatible - again upgrade is being abandoned NOTE: this command will not return until namenode shuts down ''' command = "sudo su - -c 'hadoop namenode -rollingUpgrade downgrade' hdfs" if HDFS.isHAEnabled(): nodes = [] nodes.append(HDFS.getNamenodeByState('standby')) nodes.append(HDFS.getNamenodeByState('active')) for node in nodes: HDFS.resetNamenode('stop', host=node) (exitcode, stdout) = Machine.runas(Machine.getAdminUser(), command, node, None, None, "True", Machine.getAdminPasswd()) ruAssert( "HDFS", exitcode == 0, "[NNDowngrade] hadoop namenode -rollingUpgrade downgrade command failed" ) return HDFS.stopNamenode() node = HDFS.getNamenode() (exitcode, stdout) = Machine.runas(Machine.getAdminUser(), command, node, None, None, "True", Machine.getAdminPasswd()) ruAssert( "HDFS", exitcode == 0, "[NNDowngrade] hadoop namenode -rollingUpgrade downgrade command failed" )
def submit_storm_jdbc_topology(cls, tcId, className, args, topologyName, useStandaloneCmd): """ Name: Storm-JDBC Topology Description: Testing storm-jdbc topology in distributed mode 1. UserPersistanceTopology """ from beaver.dbutil import MySQL MySQL.runAsRoot("CREATE DATABASE test") MySQL.runAsRoot("show databases") exit_code, stdout = Storm.runStormJdbcTopology( TARGET_JDBC_STORM_JAR, className, args, None, logoutput=True, inBackground=False, useStandaloneCmd=useStandaloneCmd) ruAssert("Storm", exit_code == 0, "[StormJDBCSubmit] %s Failed" % (tcId))
def submit_storm_hive_topology(cls, tcId, className, args, useStandaloneCmd): if Hadoop.isSecure(): if Config.hasOption('machine', 'USER_REALM'): user_realm = Config.get('machine', 'USER_REALM', '') else: nnKerbPrincipal = HDFS.getNameNodePrincipal(defaultValue='') atloc = nnKerbPrincipal.find("@") if (atloc != -1): user_realm = nnKerbPrincipal[atloc:] if user_realm != None: args += " " + Machine.getHeadlessUserKeytab( Config.getEnv('USER')) + " " + Config.getEnv( 'USER') + '@' + user_realm exit_code, stdout = Storm.runStormHdfsTopology( TARGET_HIVE_STORM_JAR, className, args, None, logoutput=True, inBackground=False, useStandaloneCmd=useStandaloneCmd) logger.info(exit_code) ruAssert("Storm", exit_code == 0, "[StormHiveSubmit] %s Failed" % (tcId))
def setup_hbase(self): from beaver.component.hbase import HBase from beaver.component.rollingupgrade.RuAssert import ruAssert HBASE_PREUPGRADE_TABLE = "hbase_preUpgrade_table" # HBASE_CONF_DIR = HBase.getModifiedConfigPath() # propertyMap = {'mapred.job.queue.name':'hbase'} # #stopping region servers and master # HBase.stopRegionServers(nodes=HBase.getRegionServers(), skip_slider_check=True) # HBase.stopHBaseMasterNode(HBase.getMasterNode(), wait=15) # HBase.modifyConfig(changes={'hbase-site.xml':propertyMap}, nodeSelection={'services':['all']}) # host = Machine.getfqdn(HBase.getMasterNode()) # lhost = Machine.getfqdn() # if (host != lhost): # Machine.rm(None, None, HBASE_CONF_DIR, isdir=True) # Machine.copyToLocal(None, host, HBASE_CONF_DIR, HBASE_CONF_DIR) # allnodes = HBase.getSelectedNodes({'services':['all']}) # allnodes.append(Machine.getfqdn()) # util.copy_back_to_original_config(HBASE_CONF_DIR, Config.get('hbase', 'HBASE_CONF_DIR'), file_list=["hbase-site.xml"], node_list=allnodes) # #starting region servers and master # HBase.startRegionServers(nodes=HBase.getRegionServers(), skip_slider_check=True) # HBase.startHBaseMasterNode(host=HBase.getMasterNode(), masterStartUpWait=True, wait=15) # time.sleep(120) HBase.dropAndCreateTable(HBASE_PREUPGRADE_TABLE, "cf") exit_code, stdout = HBase.runShellCmds( ["put '%s','row1', 'cf:f1', 'holmes'" % HBASE_PREUPGRADE_TABLE]) ruAssert("HBASE", exit_code == 0) exit_code, stdout = HBase.runShellCmds( ["put '%s','row2', 'cf:f2', 'watson'" % HBASE_PREUPGRADE_TABLE]) ruAssert("HBASE", exit_code == 0)
def setup_storm_hdfs_topology(cls, useStandaloneCmd): storm_version = Storm.getVersion(useStandaloneCmd=True) try: file_obj = open(HDFS_CONFIG_FILE, 'w') if Hadoop.isSecure(): file_obj.write('hdfs.keytab.file: ' + Machine.getHeadlessUserKeytab( user=HADOOPQA_USER) + '\n') file_obj.write('hdfs.kerberos.principal: ' + Machine.get_user_principal(user=HADOOPQA_USER) + '\n') finally: file_obj.close() HDFS.createDirectory("/tmp/mySeqTopology", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/dest", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/dest2", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/foo", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/trident", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/trident-seq", HDFS_USER, "777", False) Machine.copy(JAVA_HDFS_SRC_DIR, LOCAL_HDFS_WORK_DIR, user=None, passwd=None) if not Machine.isWindows(): (exit_code, _) = Maven.run('package', cwd=LOCAL_HDFS_WORK_DIR, env={ HADOOP_VERSION_MAVEN_PARAMETER: HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER: storm_version, HADOOP_CONF_MAVEN_PARAMETER: HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER: HDFS_FILE, HADOOP_CORE_MAVEN_PARAMETER: HADOOP_CONF, CORE_FILE_MAVEN_PARAMETER: CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER: Maven.getPublicRepoUrl() }) else: filepath = os.path.join(MOD_CONF_PATH, "core-site.xml") (exit_code, _) = Maven.run( 'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s' % (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER, storm_version, HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE, HADOOP_CORE_MAVEN_PARAMETER, HADOOP_CONF, CORE_FILE_MAVEN_PARAMETER, CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()), cwd=LOCAL_HDFS_WORK_DIR) ruAssert("Storm", exit_code == 0, "[StormHDFSSetup] maven package command failed")
def checkClasspathVersion(cls, Version_Num, config=None): Local_Test_dir = os.path.join(Config.getEnv("WORKSPACE"), "tests", "rolling_upgrade", "yarn") Multi_Version_App_Dir = os.path.join(Local_Test_dir, "data") Mapper = "data/versionVerifyMapper.py" Reducer = "data/versionVerifyReducer.py" Verify_File_Name = "test.txt" Verify_Test_File = os.path.join(Multi_Version_App_Dir, Verify_File_Name) # Set up env mapred_app_path = MAPRED.getConfigValue( "mapreduce.application.framework.path", None) mapred_classpath = MAPRED.getConfigValue( "mapreduce.application.classpath", None) env = { "mapreduce.application.framework.path": mapred_app_path, "mapreduce.application.classpath": mapred_classpath } verifyInput = cls._hdfs_input + "/verify" HDFS.createDirectory(verifyInput, None, "777", False) # Copy template files for the verifier streaming job templateFile = open(Verify_Test_File, 'w') templateFile.write(Version_Num) templateFile.close() HDFS.copyFromLocal(Verify_Test_File, verifyInput, user=Config.get('hadoop', 'HADOOPQA_USER')) # Submit the special streaming job shortStreamingId = HadoopJobHelper.runStreamJob( Mapper, Reducer, verifyInput, cls._hdfs_output_verify, files=Multi_Version_App_Dir, config=config, extraJobArg=cls._jobArgs, env=env, proposedJobName=cls._shortStreamingName) MAPRED.waitForJobDoneOrTimeout(shortStreamingId, timeoutInSec=180) # Make sure task succeeded #assert YARN.getAppFinalStateFromID(appId) == 'SUCCEEDED' # Check result content retVal, checkContent = HDFS.cat(cls._hdfs_output_verify + '/part-00000') logger.info("CHECK CLASSPATH VERSION OUTPUT") logger.info(retVal) logger.info(checkContent) ruAssert("YARN", retVal == 0) ruAssert("YARN", 'True' in checkContent, "[VersionVerify] Stream job returns false: " + checkContent) #assert retVal == 0 #assert 'True' in checkContent, "Stream job returns false: " + checkContent #assert 'False' not in checkContent, "Stream job returns false: " + checkContent HDFS.deleteDirectory(cls._hdfs_output_verify, user=Config.get('hadoop', 'HADOOPQA_USER'))
def smoke_test_setup(cls): ''' Setup required to run Smoke test ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress("### Running Pig Smoke Test Setup ####") exit_code, stdout = HDFS.copyFromLocal(cls._golden_src_file, cls._hdfs_smoke_input_path) ruAssert("Pig", exit_code == 0, '[SmokeSetup] Smoke Test Data Load failed')
def verifyLongRunningJob(cls): ''' Verify long running background job after it finishes :return: ''' ruAssert( "Tez", YARN.getAppFinalStateFromID( cls._background_job_appId) == 'SUCCEEDED') for output in cls._hdfsOutputList: ruAssert("Tez", HDFS.fileExists(output + '/part*')) logger.info("**** Verified long running job for Tez ****")
def ensure_jns_have_new_txn(cls, nodes, last_tx_id): num_of_jns = len(nodes) actual_tx_ids = {} jns_updated = 0 protocol = 'http' jn_port = '8480' if Hadoop.isEncrypted(): protocol = 'https' jn_port = '8481' # time out of 3 mins time_out = 3 * 60 # stop time for 10s step_time = 10 itr = int(time_out / step_time) for i in range(itr): logger.info( '******************** Check if all Journal Nodes are updated Iteration %s or %s *************************' % (i + 1, itr)) for node in nodes: # if all JNS are updated break if jns_updated == num_of_jns: return try: # if JN is already ahead skip it if actual_tx_ids[node] and int( actual_tx_ids[node]) >= last_tx_id: continue except KeyError: pass # other wise get the data and compare it url = '%s://%s:%s/jmx' % (protocol, node, jn_port) actual_tx_ids[node] = util.getJMXData( url, 'Hadoop:service=JournalNode,name=Journal-', 'LastWrittenTxId') logger.info( '******************** JN: %s LAST TX ID: %s *************************' % (node, last_tx_id)) if int(actual_tx_ids[node]) >= last_tx_id: jns_updated += 1 # if all JNS are updated break if jns_updated == num_of_jns: return time.sleep(step_time) ruAssert("HDFS", jns_updated == num_of_jns)
def runLoadGenerator(cls, numOfNodes=1, elapsedTime=100): from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode flagFile = UpgradePerNode._HDFS_FLAG_FILE # get the jar again as we dont know what version we will be running this job with. HADOOP_TEST_JAR = cls.get_hadoop_test_jar() TEST_USER = Config.get('hadoop', 'HADOOPQA_USER') # load generator jobCmd = 'jar %s NNloadGenerator -Dmapred.job.queue.name=hdfs -mr %s %s -root %s -numOfThreads 5 -maxDelayBetweenOps 1000 -elapsedTime %s -flagFile %s' % ( HADOOP_TEST_JAR, numOfNodes, cls._lgTestOutputDir, cls._lgTestDataDir, elapsedTime, flagFile) exit_code, stdout = Hadoop.run(jobCmd) ruAssert("HDFS", exit_code == 0, "[BGJobSetup] LoadGenerator failed")
def verify_jdbc_topology(cls, topologyName, useStandaloneCmd): """ Verifies the Jdbc topology produced expected output Here, we take row count of the table at two different time stamp and make sure that count2 > count1 """ logger.info("Get Count from User table") first_count = cls.getRowCountFromMySQL("user", "test") logger.info("Wait for 20 seconds and again get count from User table") time.sleep(20) second_count = cls.getRowCountFromMySQL("user", "test") ruAssert("Storm", second_count > first_count, "[StormJDBCVerify] JDBC topology was not in progress")
def setup_storm_hbase_topology(cls, useStandaloneCmd): from beaver.component.hbase import HBase storm_version = Storm.getVersion(useStandaloneCmd=True) Machine.copy(JAVA_HBASE_SRC_DIR, LOCAL_HBASE_WORK_DIR, user=None, passwd=None) if Machine.isWindows(): (_, _) = Maven.run( 'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s' % (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER, storm_version, HBASE_CONF_MAVEN_PARAMETER, HBASE_CONF, HBASE_FILE_MAVEN_PARAMETER, HBASE_FILE, HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE, CORE_FILE_MAVEN_PARAMETER, CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()), cwd=LOCAL_HBASE_WORK_DIR) else: (_, _) = Maven.run('package', cwd=LOCAL_HBASE_WORK_DIR, env={ HADOOP_VERSION_MAVEN_PARAMETER: HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER: storm_version, HBASE_CONF_MAVEN_PARAMETER: HBASE_CONF, HBASE_FILE_MAVEN_PARAMETER: HBASE_FILE, HADOOP_CONF_MAVEN_PARAMETER: HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER: HDFS_FILE, CORE_FILE_MAVEN_PARAMETER: CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER: Maven.getPublicRepoUrl() }) exit_code, stdout = HBase.createTable(HBASE_TABLE_NAME, "cf", True, None) ruAssert("Storm", exit_code == 0) grant_cmd = "grant '%s', 'RWCXA', '%s', 'cf'" % (Storm.getStormUser(), HBASE_TABLE_NAME) exit_code, stdout = HBase.runShellCmds([grant_cmd])
def createState4Rollback2(cls): exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -rm -skipTrash rollback_state1") ruAssert("HDFS", exit_code == 0, "can't get remove file rollback_state1") exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -touchz rollback_state2") ruAssert("HDFS", exit_code == 0, "can't get create file rollback_state2") # truncate the file and validate the truncated size logger.info("**** Truncate file to 1 byte ****") exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -truncate 1 testFileTr") ruAssert("HDFS", exit_code == 0, "can't truncate file testFileTr") if os.path.isfile(cls.localTestFileTr): os.remove(cls.localTestFileTr) logger.info("**** Wait 30 second for file to be recovered ****") time.sleep(30) command = "dfs -copyToLocal testFileTr " + cls.localTestFileTr exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), command) ruAssert("HDFS", exit_code == 0, "can't copy file testFileTr") size = os.path.getsize(cls.localTestFileTr) ruAssert("HDFS", size == 1, "size not 1. Actual size:" + ` size `)
def createState4Rollback1(cls): exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -rm -skipTrash rollback_state1") exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -rm -skipTrash rollback_state2") exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -rm -skipTrash testFileTr") exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), "dfs -touchz rollback_state1") ruAssert("HDFS", exit_code == 0, "can't get create file rollback_state1") command = "dfs -put " + cls.testFileTr + " testFileTr" exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'), command) ruAssert("HDFS", exit_code == 0, "can't upload" + cls.testFileTr)
def run_client_smoketest(cls, config=None, env=None): ''' Run sleep Job passing env variables :param config: Configuration location :param env: Set Environment variables ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][YARN][SmokeClient] Starting CLI test for Yarn ") jobCmd = "jar " + MAPRED.sleepJobJar( ) + " sleep -Dmapred.job.queue.name=%s -m 1 -r 1 -mt 10 -rt 10 " % ( cls._queue) exit_code, stdout = Hadoop.run(jobCmd, env=env) ruAssert("YARN", exit_code == 0, "[SmokeClient] Yarn smoketest failed") UpgradePerNode.reportProgress( "[INFO][YARN][SmokeClient] CLI test for Yarn finished ")
def ru_finalize_state(cls): ''' Upgrade is completed and finalized - save state (if any) can be discarded ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][HDFS][Finalize] Calling finalize for HDFS ") # make sure we do not drop response exit_code, output = HDFS.runas( Config.get('hadoop', 'HDFS_USER'), "dfsadmin -Ddfs.client.test.drop.namenode.response.number=0 -rollingUpgrade finalize" ) ruAssert("HDFS", exit_code == 0, '[Finalize] -rollingUpgrade finalize failed') UpgradePerNode.reportProgress( "[INFO][HDFS][Finalize] Namenode finalized finished ")
def smoke_test_setup(cls): ''' Setup function for HDFS smoke test ''' if not cls._SmokeInputDir: cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput" HDFS.deleteDirectory(cls._SmokeInputDir, Config.get('hadoop', 'HADOOPQA_USER')) jobCmd = 'jar %s randomtextwriter \"-D%s=%s\" \"-D%s=%s\" %s' % ( Config.get('hadoop', 'HADOOP_EXAMPLES_JAR'), "mapreduce.randomtextwriter.totalbytes", "4096", "mapred.job.queue.name", cls._queue, cls._SmokeInputDir) exit_code, stdout = Hadoop.run(jobCmd) ruAssert( "HDFS", exit_code == 0, '[SmokeSetup] Randomtextwriter job failed and could not create data on hdfs' )
def tear_down_hbase_topology(cls, topologyName, useStandaloneCmd): """ tear down hbase topology. """ from beaver.component.hbase import HBase Machine.rm(user=None, host="localhost", filepath=LOCAL_HBASE_WORK_DIR, isdir=True, passwd=None) Storm.killTopology(topologyName, logoutput=True, useStandaloneCmd=useStandaloneCmd) exit_code, stdout = HBase.dropTable(HBASE_TABLE_NAME, True, None) ruAssert("Storm", exit_code == 0)
def submit_storm_hbase_topology(cls, tcId, className, args, useStandaloneCmd): if Hadoop.isSecure(): keytab_file = Machine.getHeadlessUserKeytab(user=HADOOPQA_USER) principal = Machine.get_user_principal(user=HADOOPQA_USER) args = args + " %s %s" % (keytab_file, principal) exit_code, stdout = Storm.runStormHdfsTopology( TARGET_HBASE_STORM_JAR, className, args, None, logoutput=True, inBackground=False, useStandaloneCmd=useStandaloneCmd) logger.info(exit_code) ruAssert("Storm", exit_code == 0, "[StormHBaseSubmit] %s Failed" % (tcId))
def run_JHS_test(cls, config=None): ''' Runs Job History server Test before upgrades starts :param config: expected configuration location ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][YARN][JHSTest] Running JHS test before starting upgrade") logger.info("**** Running JHS test before starting upgrade ****") jobIds = cls.launchMultipleSleepJobs(numJobs=1, config=config) ruAssert("YARN", len(jobIds) == 1, "[JHSTest] More than one history server tests launched!") cls._HS_test_appID = jobIds.pop() logger.info("**** JHS test Finished ****") UpgradePerNode.reportProgress( "[INFO][YARN][JHSTest] Running JHS test finished ")
def verify_hbase_topology(cls, topologyName, lines, useStandaloneCmd): from beaver.component.hbase import HBase #Slider app is killed before log running job verification so disabling topology activation checks. if useStandaloneCmd == True: ruAssert( "Storm", Storm.getTopologyStatus( topologyName, logoutput=True, useStandaloneCmd=useStandaloneCmd) == 'ACTIVE') exit_code, stdout = HBase.runShellCmds(["scan 'WordCount'"]) logger.info(exit_code) logger.info(stdout) for word in lines: ruAssert( "Storm", stdout.find(word) >= 0, "[StormHBaseVerify] %s not found in wordcount table" % word)
def stopFalconLongRunningJob(cls): logger.info("Falcon - Begin function stopFalconLongRunningJob") from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][Falcon][BGJobStop]Stopped/deleting Falcon background jobs started" ) CMD = " instance -type process -name rawEmailIngestProcess " \ " -list -start 2016-07-25T00:00Z -end 2018-07-25T00:00Z " \ " -filterBy STARTEDAFTER:" + cls._job_start_time + " -numResults 1000" exitcode, output = cls.runCommand(cls._job_user, CMD) success_count = output.count("SUCCEEDED") fail_count = output.count("KILLED") ruAssert( "Falcon", success_count > 0 and fail_count == 0, "[BGJobCheck] Long running job failed with " + str(fail_count) + " FAILED instances") try: from beaver.component.falcon import Falcon except ImportError: ## Import fails when Falcon is not installed on this machine. Nothing to do return # Delete entities cls.runCommand( cls._job_user, " entity -type process -delete -name rawEmailIngestProcess") cls.runCommand(cls._job_user, " entity -type feed -delete -name rawEmailFeed") cls.runCommand(cls._job_user, " entity -type cluster -delete -name primaryCluster") cls.runCommand(cls._job_user, " entity -type cluster -delete -name backupCluster") from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][Falcon][BGJobStop]Stopped/deleting Falcon background jobs") logger.info("Falcon - End function stopFalconLongRunningJob") return
def verify_hive_topology(cls, topologyName, rows, useStandaloneCmd): #Slider app is killed before log running job verification so disabling topology activation checks. if useStandaloneCmd == True: ruAssert( "Storm", Storm.getTopologyStatus( topologyName, logoutput=True, useStandaloneCmd=useStandaloneCmd) == 'ACTIVE') verify_table_q = "select distinct id,name,phone,street,city,state from stormdb.userdata order by id;" stdout = cls.getHiveQueryOutput(verify_table_q, willRunMR=True, delim=",", useStandaloneCmd=useStandaloneCmd) logger.info(stdout) for row in rows: ruAssert("Storm", stdout.find(row) >= 0, "[StormHiveVerify] %s not found in userdata table" % row)
def tear_down_hive_topology(cls, topologyName, useStandaloneCmd): """ tear down hbase topology. """ from beaver.component.hive import Hive Machine.rm(user=None, host="localhost", filepath=LOCAL_HIVE_WORK_DIR, isdir=True, passwd=None) Storm.killTopology(topologyName, logoutput=True, useStandaloneCmd=useStandaloneCmd) #Hive.restoreConfig(services=['metastore']) drop_table_q = "use %s; drop table if exists %s; " % (DATABASE_NAME, HIVE_TABLE_NAME) exit_code, stdout = Hive.runQuery( cls.get_set_queue_cmd(useStandaloneCmd) + drop_table_q) ruAssert("Storm", exit_code == 0)
def ru_prepare_save_state_for_upgrade(cls): ''' Prepare Namenode to save State for Upgrade ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][HDFS][Prepare] Preparing state for HDFS upgrade") # BUG-26726: we need to be in safemode only in non HA cluster if not HDFS.isHAEnabled(): exit_code, output = HDFS.runasAdmin("dfsadmin -safemode enter") ruAssert("HDFS", exit_code == 0, '[Preparation] enter safemode failed') exit_code, output = HDFS.runas( Config.get('hadoop', 'HDFS_USER'), "dfsadmin -Ddfs.client.test.drop.namenode.response.number=0 -rollingUpgrade prepare" ) ruAssert("HDFS", exit_code == 0, '[Preparation] -rollingUpgrade prepare failed') if not HDFS.isHAEnabled(): exit_code, output = HDFS.runasAdmin("dfsadmin -safemode leave") ruAssert("HDFS", exit_code == 0, '[Preparation] leave safemode failed') UpgradePerNode.reportProgress( "[INFO][HDFS][Prepare] Preparing state for HDFS upgrade finished ")