def test_CleanUpOfFilesAfterJobCompletionForFilesWithSymLink():
testCaseDescription="test_CleanUpOfFilesAfterJobCompletionForFilesWithSymLink"
testCaseId = "cleanup04"
util.displayTestCaseMessage(testCaseDescription,testCaseId)
fileCreated = "mysymlink.txt"
out = insertFileIntoHdfs(CREATE_FILE_2)
assert out[0] == 0
time.sleep(15)
logger.info("Try to get Job Tracker")
JOBTRACKER = MAPRED.getJobTracker()
assert JOBTRACKER != None
localFilePath = getFullPathOfFile(CREATE_FILE_2_PATH_IN_LOCAL)
hadoopStreamingCmdFormat = 'jar %s -files %s -input %s -output %s -mapper "python %s" -reducer NONE'
jobJarHadoopStreamingCmd = hadoopStreamingCmdFormat % (HADOOP_STREAMING_JAR, localFilePath, CREATE_FILE_2_PATH_IN_HADOOP, OUT_PATH_IN_HADOOP, CREATE_FILE_2)
logger.info(jobJarHadoopStreamingCmd)
out = Hadoop.runInBackground(jobJarHadoopStreamingCmd)
time.sleep(15)
logger.info("Try to get job id.....")
for i in range(1, 5):
jobId = MAPRED.getJobID()
if (validateJobId(jobId)):
break
time.sleep(10)
assert jobId.startswith('job_') == True
logger.info(" Get JobId: " + jobId + " successfully")
logger.info("Try to get Attempt ID....")
attemptId = MAPRED.getAttemptIdsForJobId(jobId)
assert attemptId.startswith("attempt_") == True
logger.info("Try to get Task Tracker...")
taskTrackersList = Hadoop.getTasktrackers()
taskTracker = taskTrackersList[0].rstrip("\n")
logger.info(" Task Tracker running the map task is " + taskTracker)
time.sleep(40)
logFileDirList = getLocalDirInfo(taskTracker);
logger.info("Log file list: " + logFileDirList)
logFileDirList = logFileDirList.split(',')
isExistedTempFile = False
for logFileDir in logFileDirList:
logger.info("Directory of log file: " + logFileDir)
isExistedTempFile = checkJobCreatedTempFileInTT(logFileDir, HADOOPQA_USER, jobId, attemptId, fileCreated, taskTracker)
if isExistedTempFile == True:
break
assert isExistedTempFile == True
logger.info("Check job is completed or not")
for i in range(1, 10):
isJobCompleted = MAPRED.checkForJobCompletion(jobId)
if isJobCompleted == True:
break
time.sleep(20)
assert isJobCompleted == True
logger.info("Job is completed!")
# Now check for the file to be cleared off after the job is complete
isExistedTempFile = checkJobCreatedTempFileInTT(logFileDir,HADOOPQA_USER,jobId,attemptId,fileCreated,taskTracker)
if isExistedTempFile == True:
logger.info(" The test case to check the files cleared after killing of jobs failed ")
logger.info(" The file created by the job still exists even after the job is successfully killed ")
assert isExistedTempFile==False
def test_CleanUpOfFilesAfterFailedJob():
testCaseDescription="testCleanUpOfFilesAfterFailedJob"
testCaseId = "cleanup03"
util.displayTestCaseMessage(testCaseDescription,testCaseId)
fileCreated="FileCreatedByJob.log"
out=insertFileIntoHdfs(CREATE_FILE)
assert out[0] == 0
logger.info("Try to get Job Tracker")
JOBTRACKER = MAPRED.getJobTracker()
assert JOBTRACKER != None
logger.info( "Submitting a streaming job that will create a file ")
localFilePath = getFullPathOfFile(CREATE_FILE_PATH_IN_LOCAL)
hadoopStreamingCmdFormat = 'jar %s -files %s -input %s -output %s -mapper "python %s" -reducer NONE'
jobJarHadoopStreamingCmd = hadoopStreamingCmdFormat % (HADOOP_STREAMING_JAR, localFilePath, CREATE_FILE_PATH_IN_HADOOP, OUT_PATH_IN_HADOOP, CREATE_FILE)
logger.info(jobJarHadoopStreamingCmd)
out = Hadoop.runInBackground(jobJarHadoopStreamingCmd)
time.sleep(20)
logger.info("Try to get job id.....")
for i in range(1, 5):
jobId = MAPRED.getJobID()
if (validateJobId(jobId)):
break
time.sleep(10)
assert jobId.startswith('job_') == True
logger.info(" Get JobId: " + jobId + " successfully")
logger.info("Try to get Attempt ID....")
attemptId = MAPRED.getAttemptIdsForJobId(jobId)
assert attemptId.startswith("attempt_") == True
logger.info("Try to get Task Tracker...")
taskTrackersList = Hadoop.getTasktrackers()
taskTracker = taskTrackersList[0].rstrip("\n")
logger.info(" Task Tracker running the map task is " + taskTracker)
time.sleep(20)
logFileDirList = getLocalDirInfo(taskTracker);
logger.info("Log file list: " + logFileDirList)
logFileDirList = logFileDirList.split(',')
isExistedTempFile = False
for logFileDir in logFileDirList:
logger.info("Directory of log file: " + logFileDir)
isExistedTempFile = checkJobCreatedTempFileInTT(logFileDir, HADOOPQA_USER, jobId, attemptId, fileCreated, taskTracker)
if isExistedTempFile == True:
break
assert isExistedTempFile == True
# Now fail the job
getAttemptIdsForJobIdAndStoreInFile(jobId)
attemptIdCount=MAPRED.checkForNewAttemptIds(jobId)
assert len(attemptIdCount) != 0
while len(attemptIdCount) != 0:
logger.info(" Since there are attempts ids proceeding to kill them ")
MAPRED.failAttempts(attemptIdCount)
attemptIdCount=MAPRED.checkForNewAttemptIds(jobId)
logger.info("Check job status")
isJobFailed=MAPRED.isJobFailed(jobId)
if isJobFailed==False:
logger.info( " The job could not be failed successfully and unable to proceed with the tests ")
assert isJobFailed==True
isExistedTempFile = checkJobCreatedTempFileInTT(logFileDir,HADOOPQA_USER,jobId,attemptId,fileCreated,taskTracker)
if isExistedTempFile:
logger.info(" The test case to check the files cleared after killing of jobs failed ")
logger.info(" The file created by the job still exists even after the job is successfully killed ")
assert isExistedTempFile==False
