def verify_hdfs_topology(cls, topologyName, targetDir, lines, type, useStandaloneCmd): """ Verifies the hdfs topologies produced expected output """ #Slider app is killed before log running job verification so disabling topology activation checks. if useStandaloneCmd == True: ruAssert( "Storm", Storm.getTopologyStatus( topologyName, logoutput=True, useStandaloneCmd=useStandaloneCmd) == 'ACTIVE') exit_code, stdout = HDFS.lsr(targetDir, False, True) hdfsListOutput = stdout.splitlines() #Picking the second last line as the first file might not have enough content and last file gets into transient #HDFS issues. if len(hdfsListOutput) >= 2: fileLine = hdfsListOutput[-2] sampleoutfile = fileLine.split(" ")[-1].strip() # Hecky solution as the test code for trident and core topologies writes under same directory. # if fileLine.endswith(".txt") and type == "cat": # sampleoutfile = fileLine.split(" ")[-1].strip() # if fileLine.endswith(".seq") and type == "text": # sampleoutfile = fileLine.split(" ")[-1].strip() logger.info("Taking sampleoutput file : %s" % (sampleoutfile)) if type == "text": exit_code, stdout = HDFS.text(sampleoutfile, None) else: exit_code, stdout = HDFS.cat(sampleoutfile, None) for line in lines: ruAssert( "Storm", stdout.find(line) >= 0, "[StormHDFSVerify] expected line : %s in %s" % (line, sampleoutfile)) else: ruAssert("Storm", False, "hdfsListOutput must have at least 2 lines")
def verify_hbase_topology(cls, topologyName, lines, useStandaloneCmd): from beaver.component.hbase import HBase #Slider app is killed before log running job verification so disabling topology activation checks. if useStandaloneCmd == True: ruAssert( "Storm", Storm.getTopologyStatus( topologyName, logoutput=True, useStandaloneCmd=useStandaloneCmd) == 'ACTIVE') exit_code, stdout = HBase.runShellCmds(["scan 'WordCount'"]) logger.info(exit_code) logger.info(stdout) for word in lines: ruAssert( "Storm", stdout.find(word) >= 0, "[StormHBaseVerify] %s not found in wordcount table" % word)
def verify_hive_topology(cls, topologyName, rows, useStandaloneCmd): #Slider app is killed before log running job verification so disabling topology activation checks. if useStandaloneCmd == True: ruAssert( "Storm", Storm.getTopologyStatus( topologyName, logoutput=True, useStandaloneCmd=useStandaloneCmd) == 'ACTIVE') verify_table_q = "select distinct id,name,phone,street,city,state from stormdb.userdata order by id;" stdout = cls.getHiveQueryOutput(verify_table_q, willRunMR=True, delim=",", useStandaloneCmd=useStandaloneCmd) logger.info(stdout) for row in rows: ruAssert("Storm", stdout.find(row) >= 0, "[StormHiveVerify] %s not found in userdata table" % row)