def test6(utils,title):
utils.show_progress(title)
utils.info(title)
fails=0
try:
if utils.OSB_DEST_HOSTNAME=='' or utils.OSB_DEST_USERNAME=='' or utils.OSB_DEST_PASSWORD=='' :
utils.warn("Please set the required variables for OSB node in test's configuration file")
utils.show_progress("Please set the required variables for OSB node in test's configuration file")
return 1
ssh=SSH_utils.open_ssh(utils.OSB_DEST_HOSTNAME,utils.OSB_DEST_USERNAME,utils.OSB_DEST_PASSWORD)
SSH_utils.execute_remote_cmd(ssh,"mkdir -p -m 0777 /tmp/%s/osbbase"%(utils.ID))
target_info,target_ces=utils.get_target_ces()
if len(target_info)==0:
target_info.append("Default Test - Submit to CREAM CE")
target_ces.append("/cream-")
for target in target_ces:
utils.show_progress("%s - %s"%(title,target_info[target_ces.index(target)]))
utils.info("%s - %s"%(title,target_info[target_ces.index(target)]))
utils.run_command_continue_on_error("rm -rf %s/*"%(utils.get_job_output_dir()))
if osbbasetdesturi_test(utils,target,ssh)==1:
fails=fails+1
SSH_utils.close_ssh(ssh)
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
fails=fails+1
def main():
utils = Test_utils.Test_utils(sys.argv[0],"Check glite_wms.conf file")
tests=["Test 1: Check common section"]
tests.append("Test 2: Check JobController section")
tests.append("Test 3: Check NetworkServer section")
tests.append("Test 4: Check LogMonitor section")
tests.append("Test 5: Check WorkloadManager section")
tests.append("Test 6: Check WorkloadManagerProxy section")
tests.append("Test 7: Check ICE section")
tests.append("Test 8: Check if attributes match values of yaim variables")
utils.prepare(sys.argv[1:],tests)
if utils.WMS_USERNAME=='' or utils.WMS_PASSWORD=='':
utils.warn("Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file")
utils.show_progress("Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file")
sys.exit(0)
signal.signal(signal.SIGINT,utils.exit_interrupt)
fails=[]
all_tests=utils.is_all_enabled()
utils.info("Get glite_wms.conf file")
try:
logging.info("Get glite_wms.conf file from remote host %s"%(utils.get_WMS()))
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
target='/etc/glite-wms/glite_wms.conf'
SSH_utils.ssh_get_file(ssh,target,"%s/glite_wms.conf_local"%(utils.get_tmp_dir()))
SSH_utils.close_ssh(ssh)
except (GeneralError) , e:
utils.log_error("Unable to get the glite_wms.conf from remote host: %s"%(utils.get_WMS()))
utils.log_error(e)
utils.exit_failure("Unable to get the glite_wms.conf from remote host: %s %s %s"%(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD))
def set_ISBBase(utils,filename):
utils.log_info("Define a jdl with ISBBaseURI attribute")
FILE=open(filename,"w")
FILE.write("Executable = \"test.sh\";\n")
FILE.write("StdOutput = \"std.out\";\n")
FILE.write("StdError = \"std.err\";\n")
FILE.write("FuzzyRank = true;\n")
FILE.write("InputSandboxBaseURI = \"gsiftp://%s:2811/tmp/%s/isb\";\n"%(utils.ISB_DEST_HOSTNAME,utils.ID))
FILE.write("InputSandbox = {\"test.sh\",\"test1.txt\"};\n")
FILE.write("OutputSandbox = {\"std.out\", \"std.err\"};\n")
FILE.write("DataAccessProtocol = \"gsiftp\";\n")
FILE.write("RetryCount = 1;\n")
FILE.write("ShallowRetryCount = 2;\n")
FILE.close()
utils.log_info("The saved jdl is:\n%s"%(commands.getoutput("cat %s"%(filename))),'DEBUG')
utils.log_info("Create executable script (test.sh) and test1.txt file at remote host: %s"%(utils.ISB_DEST_HOSTNAME))
ssh=SSH_utils.open_ssh(utils.ISB_DEST_HOSTNAME,utils.ISB_DEST_USERNAME,utils.ISB_DEST_PASSWORD,utils)
SSH_utils.execute_remote_cmd(utils,ssh,"rm -f /tmp/%s/isb/*"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"echo \"#!/bin/sh\" > /tmp/%s/isb/test.sh"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"echo $GLITE_WMS_JOBID >> /tmp/%s/isb/test.sh"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"echo \"ls -la\" >> /tmp/%s/isb/test.sh"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"touch /tmp/%s/isb/test1.txt"%(utils.ID))
SSH_utils.close_ssh(ssh,utils)
if all_tests == 1 or utils.check_test_enabled(4) == 1:
if test4(utils, ssh, tests[3]):
fails.append(tests[3])
if all_tests == 1 or utils.check_test_enabled(5) == 1:
if test5(utils, ssh, tests[4]):
fails.append(tests[4])
if all_tests == 1 or utils.check_test_enabled(6) == 1:
if test6(utils, ssh, tests[5]):
fails.append(tests[5])
if all_tests == 1 or utils.check_test_enabled(7) == 1:
if test7(utils, ssh, tests[6]):
fails.append(tests[6])
if all_tests == 1 or utils.check_test_enabled(8) == 1:
if test8(utils, ssh, tests[7]):
fails.append(tests[7])
SSH_utils.close_ssh(ssh)
if len(fails) > 0:
utils.exit_failure("%s test(s) fail(s): %s" % (len(fails), fails))
else:
utils.exit_success()
if __name__ == "__main__":
main()
def test1(utils, title):
utils.show_progress(title)
utils.info(title)
try:
CREAMs=[]
utils.info("Set MaxReplansCount=5; and ReplanGracePeriod=10; EnableReplanner=true to glite_wms.conf at WMS")
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['MaxReplansCount','ReplanGracePeriod','LogLevel','EnableReplanner'],['*','*','*','*'],['5','3','6','true'])
utils.info("Restart workload manager glite-wms-wm")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
utils.set_feedback_jdl(utils.get_jdl_file())
utils.info("Get available CREAM CEs")
CEs=utils.run_command_continue_on_error("glite-wms-job-list-match -a -c %s %s"%(utils.get_config_file(),utils.get_jdl_file())).split("\n")
for CE in CEs:
if CE.find(":8443")!=-1:
CREAMs.append(CE.strip(" -\t\n").split(":")[0])
if len(CREAMs)>1:
utils.set_requirements("RegExp(\"%s*\", other.GlueCEUniqueID) || RegExp(\"%s*\", other.GlueCEUniqueID)"%(CREAMs[0],CREAMs[1]))
utils.info("Submit jobs to trigger feedback mechanism")
JOBIDS=[]
for i in range(0,10):
JOBIDS.append(utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file())))
utils.info("Wait 60 secs")
time.sleep(60)
counter=0
limit=10
find=0
target="%s/workload_manager_events.log"%(SSH_utils.execute_remote_cmd(ssh,"echo $WMS_LOCATION_LOG").strip(" \n\t"))
while counter<limit :
for JOBID in JOBIDS:
utils.info("Check if replan mechanism is triggered for job %s"%(JOBID))
output=SSH_utils.execute_remote_cmd(ssh,"grep \"created replanning request for job %s\" %s"%(JOBID,target))
if output!='':
utils.info("Found in workload_manager_events.log a replanning request for job %s"%(JOBID))
utils.dbg(output)
find=1
break
if find==1:
break
time.sleep(60)
counter=counter+1
if find==0:
utils.error("Timeout reached while checking if replan mechanism is triggered at least for one job")
raise TimeOutError("","Timeout reached while checking if replan mechanism is triggered at least for one job")
utils.info("Check if resubmission event is logged for replan job %s"%(JOBID))
OUTPUT=utils.run_command_continue_on_error("glite-wms-job-logging-info -c %s %s"%(utils.get_config_file(),JOBID)).split("\n")
find=0
for line in OUTPUT:
if line.find("Event: Resubmission")!=-1:
utils.info("Check OK, find resubmission event for job %s"%(JOBID))
find=1
break
if find==0:
utils.error("Test failed, unable to find resubmission event for replan job %s"%(JOBID))
raise GeneralError("Check if resubmission event is logged for replan job %s"%(JOBID),"Unable to find resubmission event for replan job %s"%(JOBID))
utils.info("Check if job is aborted due to the maximum number of allowed replans")
utils.wait_until_job_finishes(JOBID)
utils.job_status(JOBID)
if utils.get_job_status().find("Aborted")==-1:
utils.error("TEST FAILED. Error job's final status is %s and not Aborted"%(utils.get_job_status()))
raise GeneralError("Check if job's status is Aborted","Error job's final status is %s and not Aborted"%(utils.get_job_status()))
else:
OUTPUT=utils.run_command_continue_on_error("glite-wms-job-status -c %s %s"%(utils.get_config_file(),JOBID)).split("\n")
for line in OUTPUT:
if line.find("Status Reason")!=-1:
reason=line.split(":")[1].strip(" \n\t")
if reason.find("hit max number of replans")==-1:
utils.error("TEST FAILED. Aborted reason is '%s' while expected is 'hit max number of replans'"%(reason))
raise GeneralError("Check status reason","Aborted reason is %s while expected is 'hit max number of replans'"%(reason))
else:
utils.info("TEST PASS")
utils.info("Cancel the remaining jobs")
for JOBID in JOBIDS:
if utils.job_is_finished(JOBID)==0:
utils.run_command_continue_on_error("glite-wms-job-cancel -c %s --noint %s"%(utils.get_config_file(),JOBID))
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
return 1
def test1(utils, title):
names,ces=utils.get_target_ces()
fails=0
if len(names)==0:
names.append("Default Test , Submit to CREAM")
ces.append("")
for i in range(len(names)):
utils.show_progress("%s - %s"%(title,names[i]))
utils.info("%s - %s"%(title,names[i]))
try:
utils.info("Set MaxOuputSandboxSize=50M; to glite_wms.conf at WMS")
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['MaxOutputSandboxSize'],['*'],['50M'])
utils.info("Restart workload manager glite-wms-wm")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
utils.info("Submit the job to CE and wait to finish")
set_huge_jdl(utils,utils.get_jdl_file())
if len(ces[i])>0:
utils.set_requirements("%s && %s"%(ces[i],utils.DEFAULTREQ))
else:
utils.set_requirements("%s"%utils.DEFAULTREQ)
utils.run_command_continue_on_error("rm -rf %s/*"%(utils.get_job_output_dir()))
res=Job_utils.submit_only_normal_job(utils,ces[i])
JOBID=res[1]
utils.info("Wait until job finished")
utils.wait_until_job_finishes(JOBID)
utils.info("Check logging info")
result=utils.run_command_continue_on_error("glite-wms-job-logging-info -v 3 --event UserTag %s"%(JOBID))
if result.find("OSB quota exceeded for") == -1:
utils.error("Not found message 'OSB quota exceeded for' at UserTag")
raise GeneralError("Check UserTag event","Not found message 'OSB quota exceeded for' at UserTag")
else:
utils.info("Find message 'OSB quota exceeded for' at UserTag event")
if result.find("Truncated last 52428800 bytes for file") == -1:
utils.error("Not found message 'Truncated last 52428800 bytes for file")
raise GeneralError("Check UserTag event","Not found message 'Truncated last 52428800 bytes for file' at UserTag")
else:
utils.info("Find message 'Truncated last 52428800 bytes for file' at UserTag event")
utils.info("Get job output")
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1 :
utils.remove(utils.get_tmp_file())
utils.info("Retrieve the output")
utils.run_command_continue_on_error ("glite-wms-job-output --nosubdir --noint --dir %s %s >> %s"%(utils.get_job_output_dir(),JOBID,utils.get_tmp_file()))
if os.path.isfile("%s/huge.tail"%(utils.get_job_output_dir())) :
utils.info("Output file (huge.tail) is correctly retrieved")
else:
utils.error("Output file (huge.tail) is not correctly retrieved")
raise GeneralError("Check output file","Output file (huge.tail) is not correctly retrieved")
utils.info("Check the size of the output file")
output=utils.run_command_continue_on_error("ls -l %s/"%(utils.get_job_output_dir()))
if output.find("52428800")!=-1:
utils.info("huge.tail size is 52428800 bytes as expected")
else:
utils.error("huge.tail size is not 52428800 bytes as expected. We get %s"%(ouput))
raise GeneralError("Check the size of the ouput file","huge.tail size is not 52428800 bytes as expected")
else:
utils.error("Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
raise GeneralError("Check final job status","Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
fails=fails+1
def test2(utils, title):
utils.show_progress(title)
utils.info(title)
try:
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
set_test2_jdl(utils,utils.get_jdl_file())
utils.info("Submit a job")
JOBID=utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.info("Set WMS in draining mode")
SSH_utils.execute_remote_cmd(ssh,"touch /var/.drain")
utils.info("Try to submit jobs")
output=utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()),1)
utils.info("Check error message")
if output.find("Unable to find any endpoint where to perform service request")==-1:
utils.error("Failed reason is not 'Unable to find any endpoint where to perform service request'")
raise GeneralError("Check error message","Failed reason is not 'Unable to find any endpoint where to perform service request'")
else:
utils.info("Failed reason is 'Unable to find any endpoint where to perform service request' as expected")
utils.info("Try list-match")
output=utils.run_command_continue_on_error("glite-wms-job-list-match %s --config %s %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()),1)
if output.find("Unable to find any endpoint where to perform service request")==-1:
utils.error("Failed reason is not 'Unable to find any endpoint where to perform service request'")
raise GeneralError("Check error message","Failed reason is not 'Unable to find any endpoint where to perform service request'")
else:
utils.info("Failed reason is 'Unable to find any endpoint where to perform service request' as expected")
utils.info("Check status of previously submitted job")
utils.wait_until_job_finishes(JOBID)
utils.info("Get job output")
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1 :
utils.remove(utils.get_tmp_file())
utils.info("Retrieve the output")
utils.run_command_continue_on_error ("glite-wms-job-output --nosubdir --noint --dir %s %s >> %s"%(utils.get_job_output_dir(),JOBID,utils.get_tmp_file()))
utils.info("Check if the output files are correctly retrieved")
if os.path.isfile("%s/std.out"%(utils.get_job_output_dir())) & os.path.isfile("%s/std.err"%(utils.get_job_output_dir())) :
utils.info("Output files are correctly retrieved")
else:
utils.error("Output files are not correctly retrieved")
raise GeneralError("Check output files","Output files are not correctly retrieved")
else:
utils.error("Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
raise GeneralError("Check final job status","Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
utils.info("Unset WMS from draining mode")
SSH_utils.execute_remote_cmd(ssh,"rm -f /var/.drain")
SSH_utils.close_ssh(ssh)
utils.info("TEST PASS")
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
utils.info("Unset WMS from draining mode")
SSH_utils.execute_remote_cmd(ssh,"rm -f /var/.drain")
SSH_utils.close_ssh(ssh)
return 1
def test3(utils, title):
utils.show_progress(title)
utils.info(title)
try:
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
utils.info("Get last access date for script /opt/lcg/sbin/grid_monitor.sh before submission")
output=SSH_utils.execute_remote_cmd(ssh, "ls -lu --time-style=full-iso `locate grid_mon`")
before_access=''
for line in output.split("\n"):
if line.find("/opt/lcg/sbin/grid_monitor.sh")!=-1 and line.find("/usr/sbin/grid_monitor.sh")==-1:
before_access=line
if len(before_access)==0:
utils.error("Unable to find script /opt/lcg/sbin/grid_monitor.sh")
raise GeneralError("Check if script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission","Unable to find script /opt/lcg/sbin/grid_monitor.sh")
utils.info("Wait 10 secs")
time.sleep(10)
utils.info("Submit a job to GRAM CE")
utils.set_jdl(utils.get_jdl_file())
utils.set_destination_ce(utils.get_jdl_file(),"2119/jobmanager")
JOBID=utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.job_status(JOBID)
while utils.get_job_status().find("Ready")!=-1 or utils.get_job_status().find("Waiting")!=-1:
utils.info("Wait 30 secs")
time.sleep(30)
utils.job_status(JOBID)
utils.info("Get last access date for script /opt/lcg/sbin/grid_monitor.sh after submission")
output=SSH_utils.execute_remote_cmd(ssh, "ls -lu --time-style=full-iso `locate grid_mon`")
SSH_utils.close_ssh(ssh)
after_access=''
for line in output.split("\n"):
if line.find("/opt/lcg/sbin/grid_monitor.sh")!=-1 and line.find("/usr/sbin/grid_monitor.sh")==-1:
after_access=line
for value in before_access.split(" "):
if value.find(":")!=-1:
before_time=value.split(".")[0]
if value.find("-")!=-1 and value.find("->")==-1:
before_date=value
for value in after_access.split(" "):
if value.find(":")!=-1:
after_time=value.split(".")[0]
if value.find("-")!=-1 and value.find("->")==-1:
after_date=value
before=time.mktime(time.strptime("%s %s"%(before_date,before_time),"%Y-%m-%d %H:%M:%S"))
after=time.mktime(time.strptime("%s %s"%(after_date,after_time),"%Y-%m-%d %H:%M:%S"))
utils.info("Check if script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission")
if after>before:
utils.info("Check OK, script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission")
else:
utils.error("Test failed, script /opt/lcg/sbin/grid_monitor.sh hasn't been used during the job submission")
utils.error("Access details before submission: %s %s"%(before_date,before_time))
utils.error("Access details after submission: %s %s"%(after_date,after_time))
raise GeneralError("Check if script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission","Test failed, script /opt/lcg/sbin/grid_monitor.sh hasn't been used during the job submission")
utils.info("TEST PASS")
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
SSH_utils.close_ssh(ssh)
return 1
