def submit_storm_hdfs_topology(cls, tcId, className, args, topologyName,
useStandaloneCmd):
"""
Name:
Storm-HDFS Topologies
Description:
Testing all 4 storm-hdfs topologies in distributed mode
1. SequenceFileTopology
2. HdfsFileTopology
3. TridentFileTopology
4. TridentSequenceTopology
Steps to verify:
1. create necessary input/output dir location if needed
2. Run storm topology in distributed mode using "storm <jar> <classname> <args>"
3. Verify expected output from hdfs output dir
4. kill the topology
"""
exit_code, stdout = Storm.runStormHdfsTopology(
TARGET_HDFS_STORM_JAR,
className,
args,
None,
logoutput=True,
inBackground=False,
useStandaloneCmd=useStandaloneCmd)
ruAssert("Storm", exit_code == 0,
"[StormHDFSSubmit] %s Failed" % (tcId))
def getHiveQueryOutput(cls,
query,
willRunMR=True,
delim=",",
useStandaloneCmd=True):
from beaver.component.hive import Hive
hiveconf = {}
if willRunMR:
hiveconf = {
'hive.input.format':
'org.apache.hadoop.hive.ql.io.HiveInputFormat',
'hive.vectorized.execution.enabled': 'false',
'hive.txn.manager':
'org.apache.hadoop.hive.ql.lockmgr.DbTxnManager',
'hive.support.concurrency': 'true'
}
exit_code, stdout, stderr = Hive.runQuery(
cls.get_set_queue_cmd(useStandaloneCmd) + query,
hiveconf=hiveconf,
stderr_as_stdout=False)
ruAssert("Storm", exit_code == 0,
"[HiveQueryOutput] Failed to run Hive query [%s]" % query)
return stdout.replace('\t', delim)
def run_client_smoketest(cls, config=None, env=None):
'''
Run wordcount Job passing env variables
:param config: Configuration location
:param env: Set Environment variables
'''
logger.info("**** Running HDFS CLI Test ****")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][HDFS][ClientSmoke] CLI test for HDFS started ")
if not cls._SmokeInputDir:
cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput"
SmokeOutputDir = cls._base_hdfs_dir + '/smokeHdfsOutput_cli'
HDFS.deleteDirectory(SmokeOutputDir,
Config.get('hadoop', 'HADOOPQA_USER'))
jobCmd = 'jar %s wordcount \"-Dmapreduce.reduce.input.limit=-1\" \"-D%s=%s\" %s %s' % (
Config.get('hadoop',
'HADOOP_EXAMPLES_JAR'), "mapred.job.queue.name",
cls._queue, cls._SmokeInputDir, SmokeOutputDir)
exit_code, stdout = Hadoop.run(jobCmd, env=env)
ruAssert("HDFS", exit_code == 0, "[ClientSmoke] Hdfs smoketest failed")
HDFS.deleteDirectory(SmokeOutputDir)
ruAssert("HDFS", exit_code == 0,
"[ClientSmoke] could not delete: " + SmokeOutputDir)
UpgradePerNode.reportProgress(
"[INFO][HDFS][ClientSmoke] CLI test for HDFS Finished ")
def runCommand(cls, user, cmd, ignoreError=True):
"""
Function to run falcon commands cmd as a certain user
:param user:
:param cmd:
:return:
"""
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return 0, ""
exit_code, output = Falcon.runas(user, cmd)
if ignoreError:
if exit_code != 0:
#logger.info("Warning (ignoreError=True): Non-zero exit code when running command " + cmd + " as user " + user)
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[FAILED][Falcon] Warning (ignoreError=True): Non-zero exit code when running command %s as user %s "
% (cmd, user))
else:
ruAssert(
"Falcon", exit_code == 0,
"[RunCommand] Could not run command " + cmd + " as user " +
user)
return exit_code, output
def ru_rollback_state(cls):
'''
Saved state is rolled back - upgrade is abandonded
NOTE: this command will not return until namenode shuts down
:return:
'''
logger.info("[INFO][HDFS][Upgrade] HA Journal Node Upgrade Started ")
hdfs_user = Config.get('hadoop', 'HDFS_USER')
nodes = []
nodes.append(HDFS.getNamenodeByState('standby'))
nodes.append(HDFS.getNamenodeByState('active'))
logger.info("[INFO][HDFS][Upgrade] HA Namenode Upgrade Started")
for node in nodes:
HDFS.resetZkfc('stop', hosts=node.split())
HDFS.resetNamenode('stop', host=node)
HDFS.resetNamenode('start',
config=None,
host=nodes[0],
option=" -rollingUpgrade rollback")
HDFS.resetZkfc('start', hosts=nodes[0].split())
# lets make sure the NN is out of safemode before we proceed to the next namenode
HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[0])
command = "sudo su - -c 'hadoop namenode -bootstrapStandby -force' hdfs"
(exitcode, stdout) = Machine.runas(Machine.getAdminUser(), command,
nodes[1], None, None, "True",
Machine.getAdminPasswd())
ruAssert("HDFS", exitcode == 0,
"hadoop namenode -bootstrapStandby -force")
HDFS.resetNamenode('start', config=None, host=nodes[1], option="")
HDFS.resetZkfc('start', hosts=nodes[1].split())
# lets make sure the NN is out of safemode before we proceed to the next namenode
HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[1])
def ru_downgrade_state(cls):
'''
Downgrades Namenode
A downgrade is done - may need to convert state to previous version or state is compatible - again upgrade is being abandoned
NOTE: this command will not return until namenode shuts down
'''
command = "sudo su - -c 'hadoop namenode -rollingUpgrade downgrade' hdfs"
if HDFS.isHAEnabled():
nodes = []
nodes.append(HDFS.getNamenodeByState('standby'))
nodes.append(HDFS.getNamenodeByState('active'))
for node in nodes:
HDFS.resetNamenode('stop', host=node)
(exitcode, stdout) = Machine.runas(Machine.getAdminUser(),
command, node, None, None,
"True",
Machine.getAdminPasswd())
ruAssert(
"HDFS", exitcode == 0,
"[NNDowngrade] hadoop namenode -rollingUpgrade downgrade command failed"
)
return
HDFS.stopNamenode()
node = HDFS.getNamenode()
(exitcode, stdout) = Machine.runas(Machine.getAdminUser(), command,
node, None, None, "True",
Machine.getAdminPasswd())
ruAssert(
"HDFS", exitcode == 0,
"[NNDowngrade] hadoop namenode -rollingUpgrade downgrade command failed"
)
def submit_storm_jdbc_topology(cls, tcId, className, args, topologyName,
useStandaloneCmd):
"""
Name:
Storm-JDBC Topology
Description:
Testing storm-jdbc topology in distributed mode
1. UserPersistanceTopology
"""
from beaver.dbutil import MySQL
MySQL.runAsRoot("CREATE DATABASE test")
MySQL.runAsRoot("show databases")
exit_code, stdout = Storm.runStormJdbcTopology(
TARGET_JDBC_STORM_JAR,
className,
args,
None,
logoutput=True,
inBackground=False,
useStandaloneCmd=useStandaloneCmd)
ruAssert("Storm", exit_code == 0,
"[StormJDBCSubmit] %s Failed" % (tcId))
def submit_storm_hive_topology(cls, tcId, className, args,
useStandaloneCmd):
if Hadoop.isSecure():
if Config.hasOption('machine', 'USER_REALM'):
user_realm = Config.get('machine', 'USER_REALM', '')
else:
nnKerbPrincipal = HDFS.getNameNodePrincipal(defaultValue='')
atloc = nnKerbPrincipal.find("@")
if (atloc != -1):
user_realm = nnKerbPrincipal[atloc:]
if user_realm != None:
args += " " + Machine.getHeadlessUserKeytab(
Config.getEnv('USER')) + " " + Config.getEnv(
'USER') + '@' + user_realm
exit_code, stdout = Storm.runStormHdfsTopology(
TARGET_HIVE_STORM_JAR,
className,
args,
None,
logoutput=True,
inBackground=False,
useStandaloneCmd=useStandaloneCmd)
logger.info(exit_code)
ruAssert("Storm", exit_code == 0,
"[StormHiveSubmit] %s Failed" % (tcId))
def setup_hbase(self):
from beaver.component.hbase import HBase
from beaver.component.rollingupgrade.RuAssert import ruAssert
HBASE_PREUPGRADE_TABLE = "hbase_preUpgrade_table"
# HBASE_CONF_DIR = HBase.getModifiedConfigPath()
# propertyMap = {'mapred.job.queue.name':'hbase'}
# #stopping region servers and master
# HBase.stopRegionServers(nodes=HBase.getRegionServers(), skip_slider_check=True)
# HBase.stopHBaseMasterNode(HBase.getMasterNode(), wait=15)
# HBase.modifyConfig(changes={'hbase-site.xml':propertyMap}, nodeSelection={'services':['all']})
# host = Machine.getfqdn(HBase.getMasterNode())
# lhost = Machine.getfqdn()
# if (host != lhost):
# Machine.rm(None, None, HBASE_CONF_DIR, isdir=True)
# Machine.copyToLocal(None, host, HBASE_CONF_DIR, HBASE_CONF_DIR)
# allnodes = HBase.getSelectedNodes({'services':['all']})
# allnodes.append(Machine.getfqdn())
# util.copy_back_to_original_config(HBASE_CONF_DIR, Config.get('hbase', 'HBASE_CONF_DIR'), file_list=["hbase-site.xml"], node_list=allnodes)
# #starting region servers and master
# HBase.startRegionServers(nodes=HBase.getRegionServers(), skip_slider_check=True)
# HBase.startHBaseMasterNode(host=HBase.getMasterNode(), masterStartUpWait=True, wait=15)
# time.sleep(120)
HBase.dropAndCreateTable(HBASE_PREUPGRADE_TABLE, "cf")
exit_code, stdout = HBase.runShellCmds(
["put '%s','row1', 'cf:f1', 'holmes'" % HBASE_PREUPGRADE_TABLE])
ruAssert("HBASE", exit_code == 0)
exit_code, stdout = HBase.runShellCmds(
["put '%s','row2', 'cf:f2', 'watson'" % HBASE_PREUPGRADE_TABLE])
ruAssert("HBASE", exit_code == 0)
def setup_storm_hdfs_topology(cls, useStandaloneCmd):
storm_version = Storm.getVersion(useStandaloneCmd=True)
try:
file_obj = open(HDFS_CONFIG_FILE, 'w')
if Hadoop.isSecure():
file_obj.write('hdfs.keytab.file: ' +
Machine.getHeadlessUserKeytab(
user=HADOOPQA_USER) + '\n')
file_obj.write('hdfs.kerberos.principal: ' +
Machine.get_user_principal(user=HADOOPQA_USER) +
'\n')
finally:
file_obj.close()
HDFS.createDirectory("/tmp/mySeqTopology", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/dest", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/dest2", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/foo", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/trident", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/trident-seq", HDFS_USER, "777", False)
Machine.copy(JAVA_HDFS_SRC_DIR,
LOCAL_HDFS_WORK_DIR,
user=None,
passwd=None)
if not Machine.isWindows():
(exit_code, _) = Maven.run('package',
cwd=LOCAL_HDFS_WORK_DIR,
env={
HADOOP_VERSION_MAVEN_PARAMETER:
HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER:
storm_version,
HADOOP_CONF_MAVEN_PARAMETER:
HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER:
HDFS_FILE,
HADOOP_CORE_MAVEN_PARAMETER:
HADOOP_CONF,
CORE_FILE_MAVEN_PARAMETER:
CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER:
Maven.getPublicRepoUrl()
})
else:
filepath = os.path.join(MOD_CONF_PATH, "core-site.xml")
(exit_code, _) = Maven.run(
'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s'
% (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER, storm_version,
HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE,
HADOOP_CORE_MAVEN_PARAMETER, HADOOP_CONF,
CORE_FILE_MAVEN_PARAMETER, CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()),
cwd=LOCAL_HDFS_WORK_DIR)
ruAssert("Storm", exit_code == 0,
"[StormHDFSSetup] maven package command failed")
def checkClasspathVersion(cls, Version_Num, config=None):
Local_Test_dir = os.path.join(Config.getEnv("WORKSPACE"), "tests",
"rolling_upgrade", "yarn")
Multi_Version_App_Dir = os.path.join(Local_Test_dir, "data")
Mapper = "data/versionVerifyMapper.py"
Reducer = "data/versionVerifyReducer.py"
Verify_File_Name = "test.txt"
Verify_Test_File = os.path.join(Multi_Version_App_Dir,
Verify_File_Name)
# Set up env
mapred_app_path = MAPRED.getConfigValue(
"mapreduce.application.framework.path", None)
mapred_classpath = MAPRED.getConfigValue(
"mapreduce.application.classpath", None)
env = {
"mapreduce.application.framework.path": mapred_app_path,
"mapreduce.application.classpath": mapred_classpath
}
verifyInput = cls._hdfs_input + "/verify"
HDFS.createDirectory(verifyInput, None, "777", False)
# Copy template files for the verifier streaming job
templateFile = open(Verify_Test_File, 'w')
templateFile.write(Version_Num)
templateFile.close()
HDFS.copyFromLocal(Verify_Test_File,
verifyInput,
user=Config.get('hadoop', 'HADOOPQA_USER'))
# Submit the special streaming job
shortStreamingId = HadoopJobHelper.runStreamJob(
Mapper,
Reducer,
verifyInput,
cls._hdfs_output_verify,
files=Multi_Version_App_Dir,
config=config,
extraJobArg=cls._jobArgs,
env=env,
proposedJobName=cls._shortStreamingName)
MAPRED.waitForJobDoneOrTimeout(shortStreamingId, timeoutInSec=180)
# Make sure task succeeded
#assert YARN.getAppFinalStateFromID(appId) == 'SUCCEEDED'
# Check result content
retVal, checkContent = HDFS.cat(cls._hdfs_output_verify +
'/part-00000')
logger.info("CHECK CLASSPATH VERSION OUTPUT")
logger.info(retVal)
logger.info(checkContent)
ruAssert("YARN", retVal == 0)
ruAssert("YARN", 'True' in checkContent,
"[VersionVerify] Stream job returns false: " + checkContent)
#assert retVal == 0
#assert 'True' in checkContent, "Stream job returns false: " + checkContent
#assert 'False' not in checkContent, "Stream job returns false: " + checkContent
HDFS.deleteDirectory(cls._hdfs_output_verify,
user=Config.get('hadoop', 'HADOOPQA_USER'))
def smoke_test_setup(cls):
'''
Setup required to run Smoke test
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress("### Running Pig Smoke Test Setup ####")
exit_code, stdout = HDFS.copyFromLocal(cls._golden_src_file,
cls._hdfs_smoke_input_path)
ruAssert("Pig", exit_code == 0,
'[SmokeSetup] Smoke Test Data Load failed')
def verifyLongRunningJob(cls):
'''
Verify long running background job after it finishes
:return:
'''
ruAssert(
"Tez",
YARN.getAppFinalStateFromID(
cls._background_job_appId) == 'SUCCEEDED')
for output in cls._hdfsOutputList:
ruAssert("Tez", HDFS.fileExists(output + '/part*'))
logger.info("**** Verified long running job for Tez ****")
def ensure_jns_have_new_txn(cls, nodes, last_tx_id):
num_of_jns = len(nodes)
actual_tx_ids = {}
jns_updated = 0
protocol = 'http'
jn_port = '8480'
if Hadoop.isEncrypted():
protocol = 'https'
jn_port = '8481'
# time out of 3 mins
time_out = 3 * 60
# stop time for 10s
step_time = 10
itr = int(time_out / step_time)
for i in range(itr):
logger.info(
'******************** Check if all Journal Nodes are updated Iteration %s or %s *************************'
% (i + 1, itr))
for node in nodes:
# if all JNS are updated break
if jns_updated == num_of_jns:
return
try:
# if JN is already ahead skip it
if actual_tx_ids[node] and int(
actual_tx_ids[node]) >= last_tx_id:
continue
except KeyError:
pass
# other wise get the data and compare it
url = '%s://%s:%s/jmx' % (protocol, node, jn_port)
actual_tx_ids[node] = util.getJMXData(
url, 'Hadoop:service=JournalNode,name=Journal-',
'LastWrittenTxId')
logger.info(
'******************** JN: %s LAST TX ID: %s *************************'
% (node, last_tx_id))
if int(actual_tx_ids[node]) >= last_tx_id:
jns_updated += 1
# if all JNS are updated break
if jns_updated == num_of_jns:
return
time.sleep(step_time)
ruAssert("HDFS", jns_updated == num_of_jns)
def runLoadGenerator(cls, numOfNodes=1, elapsedTime=100):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
flagFile = UpgradePerNode._HDFS_FLAG_FILE
# get the jar again as we dont know what version we will be running this job with.
HADOOP_TEST_JAR = cls.get_hadoop_test_jar()
TEST_USER = Config.get('hadoop', 'HADOOPQA_USER')
# load generator
jobCmd = 'jar %s NNloadGenerator -Dmapred.job.queue.name=hdfs -mr %s %s -root %s -numOfThreads 5 -maxDelayBetweenOps 1000 -elapsedTime %s -flagFile %s' % (
HADOOP_TEST_JAR, numOfNodes, cls._lgTestOutputDir,
cls._lgTestDataDir, elapsedTime, flagFile)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert("HDFS", exit_code == 0, "[BGJobSetup] LoadGenerator failed")
def verify_jdbc_topology(cls, topologyName, useStandaloneCmd):
"""
Verifies the Jdbc topology produced expected output
Here, we take row count of the table at two different time stamp
and make sure that count2 > count1
"""
logger.info("Get Count from User table")
first_count = cls.getRowCountFromMySQL("user", "test")
logger.info("Wait for 20 seconds and again get count from User table")
time.sleep(20)
second_count = cls.getRowCountFromMySQL("user", "test")
ruAssert("Storm", second_count > first_count,
"[StormJDBCVerify] JDBC topology was not in progress")
def setup_storm_hbase_topology(cls, useStandaloneCmd):
from beaver.component.hbase import HBase
storm_version = Storm.getVersion(useStandaloneCmd=True)
Machine.copy(JAVA_HBASE_SRC_DIR,
LOCAL_HBASE_WORK_DIR,
user=None,
passwd=None)
if Machine.isWindows():
(_, _) = Maven.run(
'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s'
% (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER, storm_version,
HBASE_CONF_MAVEN_PARAMETER, HBASE_CONF,
HBASE_FILE_MAVEN_PARAMETER, HBASE_FILE,
HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE,
CORE_FILE_MAVEN_PARAMETER, CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()),
cwd=LOCAL_HBASE_WORK_DIR)
else:
(_, _) = Maven.run('package',
cwd=LOCAL_HBASE_WORK_DIR,
env={
HADOOP_VERSION_MAVEN_PARAMETER:
HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER:
storm_version,
HBASE_CONF_MAVEN_PARAMETER:
HBASE_CONF,
HBASE_FILE_MAVEN_PARAMETER:
HBASE_FILE,
HADOOP_CONF_MAVEN_PARAMETER:
HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER:
HDFS_FILE,
CORE_FILE_MAVEN_PARAMETER:
CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER:
Maven.getPublicRepoUrl()
})
exit_code, stdout = HBase.createTable(HBASE_TABLE_NAME, "cf", True,
None)
ruAssert("Storm", exit_code == 0)
grant_cmd = "grant '%s', 'RWCXA', '%s', 'cf'" % (Storm.getStormUser(),
HBASE_TABLE_NAME)
exit_code, stdout = HBase.runShellCmds([grant_cmd])
def createState4Rollback2(cls):
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -rm -skipTrash rollback_state1")
ruAssert("HDFS", exit_code == 0,
"can't get remove file rollback_state1")
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -touchz rollback_state2")
ruAssert("HDFS", exit_code == 0,
"can't get create file rollback_state2")
# truncate the file and validate the truncated size
logger.info("**** Truncate file to 1 byte ****")
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -truncate 1 testFileTr")
ruAssert("HDFS", exit_code == 0, "can't truncate file testFileTr")
if os.path.isfile(cls.localTestFileTr):
os.remove(cls.localTestFileTr)
logger.info("**** Wait 30 second for file to be recovered ****")
time.sleep(30)
command = "dfs -copyToLocal testFileTr " + cls.localTestFileTr
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
command)
ruAssert("HDFS", exit_code == 0, "can't copy file testFileTr")
size = os.path.getsize(cls.localTestFileTr)
ruAssert("HDFS", size == 1, "size not 1. Actual size:" + ` size `)
def createState4Rollback1(cls):
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -rm -skipTrash rollback_state1")
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -rm -skipTrash rollback_state2")
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -rm -skipTrash testFileTr")
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -touchz rollback_state1")
ruAssert("HDFS", exit_code == 0,
"can't get create file rollback_state1")
command = "dfs -put " + cls.testFileTr + " testFileTr"
exit_code, stdout = HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
command)
ruAssert("HDFS", exit_code == 0, "can't upload" + cls.testFileTr)
def run_client_smoketest(cls, config=None, env=None):
'''
Run sleep Job passing env variables
:param config: Configuration location
:param env: Set Environment variables
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][SmokeClient] Starting CLI test for Yarn ")
jobCmd = "jar " + MAPRED.sleepJobJar(
) + " sleep -Dmapred.job.queue.name=%s -m 1 -r 1 -mt 10 -rt 10 " % (
cls._queue)
exit_code, stdout = Hadoop.run(jobCmd, env=env)
ruAssert("YARN", exit_code == 0, "[SmokeClient] Yarn smoketest failed")
UpgradePerNode.reportProgress(
"[INFO][YARN][SmokeClient] CLI test for Yarn finished ")
def ru_finalize_state(cls):
'''
Upgrade is completed and finalized - save state (if any) can be discarded
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][HDFS][Finalize] Calling finalize for HDFS ")
# make sure we do not drop response
exit_code, output = HDFS.runas(
Config.get('hadoop', 'HDFS_USER'),
"dfsadmin -Ddfs.client.test.drop.namenode.response.number=0 -rollingUpgrade finalize"
)
ruAssert("HDFS", exit_code == 0,
'[Finalize] -rollingUpgrade finalize failed')
UpgradePerNode.reportProgress(
"[INFO][HDFS][Finalize] Namenode finalized finished ")
def smoke_test_setup(cls):
'''
Setup function for HDFS smoke test
'''
if not cls._SmokeInputDir:
cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput"
HDFS.deleteDirectory(cls._SmokeInputDir,
Config.get('hadoop', 'HADOOPQA_USER'))
jobCmd = 'jar %s randomtextwriter \"-D%s=%s\" \"-D%s=%s\" %s' % (
Config.get('hadoop', 'HADOOP_EXAMPLES_JAR'),
"mapreduce.randomtextwriter.totalbytes", "4096",
"mapred.job.queue.name", cls._queue, cls._SmokeInputDir)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert(
"HDFS", exit_code == 0,
'[SmokeSetup] Randomtextwriter job failed and could not create data on hdfs'
)
def tear_down_hbase_topology(cls, topologyName, useStandaloneCmd):
"""
tear down hbase topology.
"""
from beaver.component.hbase import HBase
Machine.rm(user=None,
host="localhost",
filepath=LOCAL_HBASE_WORK_DIR,
isdir=True,
passwd=None)
Storm.killTopology(topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd)
exit_code, stdout = HBase.dropTable(HBASE_TABLE_NAME, True, None)
ruAssert("Storm", exit_code == 0)
def submit_storm_hbase_topology(cls, tcId, className, args,
useStandaloneCmd):
if Hadoop.isSecure():
keytab_file = Machine.getHeadlessUserKeytab(user=HADOOPQA_USER)
principal = Machine.get_user_principal(user=HADOOPQA_USER)
args = args + " %s %s" % (keytab_file, principal)
exit_code, stdout = Storm.runStormHdfsTopology(
TARGET_HBASE_STORM_JAR,
className,
args,
None,
logoutput=True,
inBackground=False,
useStandaloneCmd=useStandaloneCmd)
logger.info(exit_code)
ruAssert("Storm", exit_code == 0,
"[StormHBaseSubmit] %s Failed" % (tcId))
def run_JHS_test(cls, config=None):
'''
Runs Job History server Test before upgrades starts
:param config: expected configuration location
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][JHSTest] Running JHS test before starting upgrade")
logger.info("**** Running JHS test before starting upgrade ****")
jobIds = cls.launchMultipleSleepJobs(numJobs=1, config=config)
ruAssert("YARN",
len(jobIds) == 1,
"[JHSTest] More than one history server tests launched!")
cls._HS_test_appID = jobIds.pop()
logger.info("**** JHS test Finished ****")
UpgradePerNode.reportProgress(
"[INFO][YARN][JHSTest] Running JHS test finished ")
def verify_hbase_topology(cls, topologyName, lines, useStandaloneCmd):
from beaver.component.hbase import HBase
#Slider app is killed before log running job verification so disabling topology activation checks.
if useStandaloneCmd == True:
ruAssert(
"Storm",
Storm.getTopologyStatus(
topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd) == 'ACTIVE')
exit_code, stdout = HBase.runShellCmds(["scan 'WordCount'"])
logger.info(exit_code)
logger.info(stdout)
for word in lines:
ruAssert(
"Storm",
stdout.find(word) >= 0,
"[StormHBaseVerify] %s not found in wordcount table" % word)
def stopFalconLongRunningJob(cls):
logger.info("Falcon - Begin function stopFalconLongRunningJob")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][BGJobStop]Stopped/deleting Falcon background jobs started"
)
CMD = " instance -type process -name rawEmailIngestProcess " \
" -list -start 2016-07-25T00:00Z -end 2018-07-25T00:00Z " \
" -filterBy STARTEDAFTER:" + cls._job_start_time + " -numResults 1000"
exitcode, output = cls.runCommand(cls._job_user, CMD)
success_count = output.count("SUCCEEDED")
fail_count = output.count("KILLED")
ruAssert(
"Falcon", success_count > 0 and fail_count == 0,
"[BGJobCheck] Long running job failed with " + str(fail_count) +
" FAILED instances")
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
# Delete entities
cls.runCommand(
cls._job_user,
" entity -type process -delete -name rawEmailIngestProcess")
cls.runCommand(cls._job_user,
" entity -type feed -delete -name rawEmailFeed")
cls.runCommand(cls._job_user,
" entity -type cluster -delete -name primaryCluster")
cls.runCommand(cls._job_user,
" entity -type cluster -delete -name backupCluster")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][BGJobStop]Stopped/deleting Falcon background jobs")
logger.info("Falcon - End function stopFalconLongRunningJob")
return
def verify_hive_topology(cls, topologyName, rows, useStandaloneCmd):
#Slider app is killed before log running job verification so disabling topology activation checks.
if useStandaloneCmd == True:
ruAssert(
"Storm",
Storm.getTopologyStatus(
topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd) == 'ACTIVE')
verify_table_q = "select distinct id,name,phone,street,city,state from stormdb.userdata order by id;"
stdout = cls.getHiveQueryOutput(verify_table_q,
willRunMR=True,
delim=",",
useStandaloneCmd=useStandaloneCmd)
logger.info(stdout)
for row in rows:
ruAssert("Storm",
stdout.find(row) >= 0,
"[StormHiveVerify] %s not found in userdata table" % row)
def tear_down_hive_topology(cls, topologyName, useStandaloneCmd):
"""
tear down hbase topology.
"""
from beaver.component.hive import Hive
Machine.rm(user=None,
host="localhost",
filepath=LOCAL_HIVE_WORK_DIR,
isdir=True,
passwd=None)
Storm.killTopology(topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd)
#Hive.restoreConfig(services=['metastore'])
drop_table_q = "use %s; drop table if exists %s; " % (DATABASE_NAME,
HIVE_TABLE_NAME)
exit_code, stdout = Hive.runQuery(
cls.get_set_queue_cmd(useStandaloneCmd) + drop_table_q)
ruAssert("Storm", exit_code == 0)
def ru_prepare_save_state_for_upgrade(cls):
'''
Prepare Namenode to save State for Upgrade
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][HDFS][Prepare] Preparing state for HDFS upgrade")
# BUG-26726: we need to be in safemode only in non HA cluster
if not HDFS.isHAEnabled():
exit_code, output = HDFS.runasAdmin("dfsadmin -safemode enter")
ruAssert("HDFS", exit_code == 0,
'[Preparation] enter safemode failed')
exit_code, output = HDFS.runas(
Config.get('hadoop', 'HDFS_USER'),
"dfsadmin -Ddfs.client.test.drop.namenode.response.number=0 -rollingUpgrade prepare"
)
ruAssert("HDFS", exit_code == 0,
'[Preparation] -rollingUpgrade prepare failed')
if not HDFS.isHAEnabled():
exit_code, output = HDFS.runasAdmin("dfsadmin -safemode leave")
ruAssert("HDFS", exit_code == 0,
'[Preparation] leave safemode failed')
UpgradePerNode.reportProgress(
"[INFO][HDFS][Prepare] Preparing state for HDFS upgrade finished ")
