def verify_hdfs_topology(cls, topologyName, targetDir, lines, type,
useStandaloneCmd):
"""
Verifies the hdfs topologies produced expected output
"""
#Slider app is killed before log running job verification so disabling topology activation checks.
if useStandaloneCmd == True:
ruAssert(
"Storm",
Storm.getTopologyStatus(
topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd) == 'ACTIVE')
exit_code, stdout = HDFS.lsr(targetDir, False, True)
hdfsListOutput = stdout.splitlines()
#Picking the second last line as the first file might not have enough content and last file gets into transient
#HDFS issues.
if len(hdfsListOutput) >= 2:
fileLine = hdfsListOutput[-2]
sampleoutfile = fileLine.split(" ")[-1].strip()
# Hecky solution as the test code for trident and core topologies writes under same directory.
# if fileLine.endswith(".txt") and type == "cat":
# sampleoutfile = fileLine.split(" ")[-1].strip()
# if fileLine.endswith(".seq") and type == "text":
# sampleoutfile = fileLine.split(" ")[-1].strip()
logger.info("Taking sampleoutput file : %s" % (sampleoutfile))
if type == "text":
exit_code, stdout = HDFS.text(sampleoutfile, None)
else:
exit_code, stdout = HDFS.cat(sampleoutfile, None)
for line in lines:
ruAssert(
"Storm",
stdout.find(line) >= 0,
"[StormHDFSVerify] expected line : %s in %s" %
(line, sampleoutfile))
else:
ruAssert("Storm", False,
"hdfsListOutput must have at least 2 lines")
def verify_hbase_topology(cls, topologyName, lines, useStandaloneCmd):
from beaver.component.hbase import HBase
#Slider app is killed before log running job verification so disabling topology activation checks.
if useStandaloneCmd == True:
ruAssert(
"Storm",
Storm.getTopologyStatus(
topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd) == 'ACTIVE')
exit_code, stdout = HBase.runShellCmds(["scan 'WordCount'"])
logger.info(exit_code)
logger.info(stdout)
for word in lines:
ruAssert(
"Storm",
stdout.find(word) >= 0,
"[StormHBaseVerify] %s not found in wordcount table" % word)
def verify_hive_topology(cls, topologyName, rows, useStandaloneCmd):
#Slider app is killed before log running job verification so disabling topology activation checks.
if useStandaloneCmd == True:
ruAssert(
"Storm",
Storm.getTopologyStatus(
topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd) == 'ACTIVE')
verify_table_q = "select distinct id,name,phone,street,city,state from stormdb.userdata order by id;"
stdout = cls.getHiveQueryOutput(verify_table_q,
willRunMR=True,
delim=",",
useStandaloneCmd=useStandaloneCmd)
logger.info(stdout)
for row in rows:
ruAssert("Storm",
stdout.find(row) >= 0,
"[StormHiveVerify] %s not found in userdata table" % row)
