def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Upload Data to HDFS before Upgrade starts
Creates /user/hrt_qa/test_rollingupgrade dir on HDFS
Upload 20 files to /user/hrt_qa/test_rollingupgrade
'''
if not cls._base_hdfs_dir:
cls._base_hdfs_dir = '/user/%s/test_rollingupgrade' % Config.get(
'hadoop', 'HADOOPQA_USER')
exit_code, stdout = HDFS.createDirectory(cls._base_hdfs_dir,
force=True)
ruAssert("HDFS", exit_code == 0,
'[BGJobSetup] could not create dir on hdfs.')
LOCAL_WORK_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
'HDFS_RU_TEST')
localTestWorkDir1 = os.path.join(LOCAL_WORK_DIR, "Temp_data")
HadoopJobHelper.runCustomWordWriter(LOCAL_WORK_DIR, localTestWorkDir1,
20, 40, 1000)
HDFS.copyFromLocal(os.path.join(localTestWorkDir1, "*"),
cls._base_hdfs_dir)
# set up for loadGenerator
cls._lgTestDataDir = cls._base_hdfs_dir + '/testData'
cls._lgTestOutputDir = cls._base_hdfs_dir + '/lg_job'
cls._lgStructureDir = Machine.getTempDir() + "/structure"
# test dir setup
HDFS.deleteDirectory(cls._lgTestDataDir)
HDFS.deleteDirectory(cls._lgTestOutputDir)
command = "rm -rf " + cls._lgStructureDir
exit_code, stdout = Machine.runas(Machine.getAdminUser(), command,
None, None, None, "True",
Machine.getAdminPasswd())
command = "mkdir " + cls._lgStructureDir
stdout = Machine.runas(None, command, None, None, None, "True", None)
Machine.chmod("777", cls._lgStructureDir, "True",
Machine.getAdminUser(), None, Machine.getAdminPasswd())
HADOOP_TEST_JAR = cls.get_hadoop_test_jar()
TEST_USER = Config.get('hadoop', 'HADOOPQA_USER')
# structure generator
jobCmd = 'jar %s NNstructureGenerator -maxDepth 5 -minWidth 2 -maxWidth 5 -numOfFiles 100 -avgFileSize 3 -outDir %s' % (
HADOOP_TEST_JAR, cls._lgStructureDir)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert("HDFS", exit_code == 0,
"[BGJobSetup] StructureGenerator failed")
# data generator
jobCmd = 'jar %s NNdataGenerator -inDir %s -root %s' % (
HADOOP_TEST_JAR, cls._lgStructureDir, cls._lgTestDataDir)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert("HDFS", exit_code == 0, "[BGJobSetup] DataGenerator failed")
if runSmokeTestSetup:
logger.info("**** Running HDFS Smoke Test Setup ****")
cls.smoke_test_setup()
def run_client_smoketest(cls, config=None, env=None):
'''
Run wordcount Job passing env variables
:param config: Configuration location
:param env: Set Environment variables
'''
logger.info("**** Running HDFS CLI Test ****")
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][HDFS][ClientSmoke] CLI test for HDFS started ")
if not cls._SmokeInputDir:
cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput"
SmokeOutputDir = cls._base_hdfs_dir + '/smokeHdfsOutput_cli'
HDFS.deleteDirectory(SmokeOutputDir,
Config.get('hadoop', 'HADOOPQA_USER'))
jobCmd = 'jar %s wordcount \"-Dmapreduce.reduce.input.limit=-1\" \"-D%s=%s\" %s %s' % (
Config.get('hadoop',
'HADOOP_EXAMPLES_JAR'), "mapred.job.queue.name",
cls._queue, cls._SmokeInputDir, SmokeOutputDir)
exit_code, stdout = Hadoop.run(jobCmd, env=env)
ruAssert("HDFS", exit_code == 0, "[ClientSmoke] Hdfs smoketest failed")
HDFS.deleteDirectory(SmokeOutputDir)
ruAssert("HDFS", exit_code == 0,
"[ClientSmoke] could not delete: " + SmokeOutputDir)
UpgradePerNode.reportProgress(
"[INFO][HDFS][ClientSmoke] CLI test for HDFS Finished ")
def getAttemptIdsForJobIdAndStoreInFile(jobId, myTask="map"):
artifactsDir = CommonHadoopEnv.getArtifactsDir()
saveFilePath = os.path.join(artifactsDir,"AttemptIdFile")
listAttemptCmd = " job -list-attempt-ids "+ jobId +" "+ myTask + " running "
out=Hadoop.run(listAttemptCmd)
buf = StringIO.StringIO(out[1])
util.writeToFile(out[1],saveFilePath)
def insertFileIntoHdfs(fileName):
pathFileName = '/user/' + HADOOPQA_USER + '/' + fileName
if (not(HDFS.fileExists(pathFileName))):
sourceFile = DATA_PATH + '/' + fileName
destFile = '/user/' + HADOOPQA_USER + '/' + fileName
putCmd = "dfs -put " + sourceFile + ' ' + destFile
out = Hadoop.run(putCmd)
return out
def runLoadGenerator(cls, numOfNodes=1, elapsedTime=100):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
flagFile = UpgradePerNode._HDFS_FLAG_FILE
# get the jar again as we dont know what version we will be running this job with.
HADOOP_TEST_JAR = cls.get_hadoop_test_jar()
TEST_USER = Config.get('hadoop', 'HADOOPQA_USER')
# load generator
jobCmd = 'jar %s NNloadGenerator -Dmapred.job.queue.name=hdfs -mr %s %s -root %s -numOfThreads 5 -maxDelayBetweenOps 1000 -elapsedTime %s -flagFile %s' % (
HADOOP_TEST_JAR, numOfNodes, cls._lgTestOutputDir,
cls._lgTestDataDir, elapsedTime, flagFile)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert("HDFS", exit_code == 0, "[BGJobSetup] LoadGenerator failed")
def run_client_smoketest(cls, config=None, env=None):
'''
Run sleep Job passing env variables
:param config: Configuration location
:param env: Set Environment variables
'''
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][YARN][SmokeClient] Starting CLI test for Yarn ")
jobCmd = "jar " + MAPRED.sleepJobJar(
) + " sleep -Dmapred.job.queue.name=%s -m 1 -r 1 -mt 10 -rt 10 " % (
cls._queue)
exit_code, stdout = Hadoop.run(jobCmd, env=env)
ruAssert("YARN", exit_code == 0, "[SmokeClient] Yarn smoketest failed")
UpgradePerNode.reportProgress(
"[INFO][YARN][SmokeClient] CLI test for Yarn finished ")
def smoke_test_setup(cls):
'''
Setup function for HDFS smoke test
'''
if not cls._SmokeInputDir:
cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput"
HDFS.deleteDirectory(cls._SmokeInputDir,
Config.get('hadoop', 'HADOOPQA_USER'))
jobCmd = 'jar %s randomtextwriter \"-D%s=%s\" \"-D%s=%s\" %s' % (
Config.get('hadoop', 'HADOOP_EXAMPLES_JAR'),
"mapreduce.randomtextwriter.totalbytes", "4096",
"mapred.job.queue.name", cls._queue, cls._SmokeInputDir)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert(
"HDFS", exit_code == 0,
'[SmokeSetup] Randomtextwriter job failed and could not create data on hdfs'
)
def run_smoke_test(cls, smoketestnumber, config=None):
'''
Run word count as HDFS smoke test
- Create file of 4096 bytes using randomwriter job
- Run wordcount job
'''
logger.info("Running HDFS Smoke test")
# make sure base hdfs dir is set.
if not cls._base_hdfs_dir:
cls._base_hdfs_dir = '/user/%s/test_rollingupgrade' % Config.get(
'hadoop', 'HADOOPQA_USER')
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][HDFS][Smoke] Smoke test for HDFS started ")
HDFS.runas(Config.get('hadoop', 'HADOOPQA_USER'),
"dfs -ls /user/hrt_qa",
env=None,
logoutput=True,
config=None,
host=None,
skipAuth=False)
if not cls._SmokeInputDir:
cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput"
SmokeOutputDir = cls._base_hdfs_dir + '/smokeHdfsOutput' + str(
smoketestnumber)
HDFS.deleteDirectory(SmokeOutputDir,
Config.get('hadoop', 'HADOOPQA_USER'))
jobCmd = 'jar %s wordcount \"-Dmapreduce.reduce.input.limit=-1\" \"-D%s=%s\" %s %s' % (
Config.get('hadoop',
'HADOOP_EXAMPLES_JAR'), "mapred.job.queue.name",
cls._queue, cls._SmokeInputDir, SmokeOutputDir)
exit_code, stdout = Hadoop.run(jobCmd)
ruAssert("HDFS", exit_code == 0, "[Smoke] Hdfs smoketest failed")
HDFS.deleteDirectory(SmokeOutputDir)
ruAssert("HDFS", exit_code == 0,
"[Smoke] could not delete: " + SmokeOutputDir)
UpgradePerNode.reportProgress(
"[INFO][HDFS][Smoke] Smoke test for HDFS Finished ")
