def set_limiter_values(utils,ssh,limit_values):
try:
job_submit="\"${WMS_LOCATION_SBIN}/glite_wms_wmproxy_load_monitor --oper jobSubmit %s\""%(limit_values)
job_register="\"${WMS_LOCATION_SBIN}/glite_wms_wmproxy_load_monitor --oper jobRegister %s\""%(limit_values)
utils.log_info("Set jobSubmit = %s; and jobRegister = %s; to glite_wms.conf at WMS"%(job_submit,job_register))
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['jobSubmit','jobRegister'],['*','*'], [job_submit,job_register])
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_info('ERROR - Command: %s'%(e.expression))
utils.log_info('ERROR - Message: %s'%(e.message))
restore_configuration_file(utils,ssh)
def run_test(utils,ssh):
CREAMs=[]
utils.log_info("Set MaxReplansCount=5; and ReplanGracePeriod=10; to glite_wms.conf at WMS")
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['MaxReplansCount','ReplanGracePeriod','LogLevel'],['*','*','*'],['5','10','6'])
utils.log_info("Restart workload manager glite-wms-wm")
SSH_utils.execute_remote_cmd(utils,ssh,"/etc/init.d/glite-wms-wm restart")
utils.set_feedback_jdl(utils.get_jdl_file())
utils.log_info("Get available CREAM CEs")
CEs=utils.run_command("glite-wms-job-list-match -a -c %s %s"%(utils.get_config_file(),utils.get_jdl_file())).split("\n")
for CE in CEs:
if CE.find(":8443")!=-1:
CREAMs.append(CE.strip(" -\t\n").split(":")[0])
if len(CREAMs)>1:
utils.set_requirements("RegExp(\"%s*\", other.GlueCEUniqueID) || RegExp(\"%s*\", other.GlueCEUniqueID)"%(CREAMs[0],CREAMs[1]))
utils.log_info("Submit jobs to trigger feedback mechanism")
JOBIDS=[]
for i in range(0,10):
JOBIDS.append(utils.run_command("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file())))
utils.log_info("Wait 60 secs")
time.sleep(60)
counter=0
limit=10
find=0
target="%s/workload_manager_events.log"%(SSH_utils.execute_remote_cmd(utils,ssh,"echo $WMS_LOCATION_LOG").strip(" \n\t"))
while counter<limit :
for JOBID in JOBIDS:
utils.log_info("Check if replan mechanism is triggered for job %s"%(JOBID))
output=SSH_utils.execute_remote_cmd(utils,ssh,"grep \"created replanning request for job %s\" %s"%(JOBID,target))
if output!='':
utils.log_info("Found in workload_manager_events.log a replanning request for job %s"%(JOBID))
utils.log_info(output,'DEBUG')
find=1
break
if find==1:
break
time.sleep(60)
counter=counter+1
if find==0:
utils.log_info("ERROR: Timeout reached while checking if replan mechanism is triggered at least for one job")
raise TimeOutError("","Timeout reached while checking if replan mechanism is triggered at least for one job")
utils.log_info("Check if resubmission event is logged for replan job %s"%(JOBID))
OUTPUT=utils.run_command("glite-wms-job-logging-info -c %s %s"%(utils.get_config_file(),JOBID)).split("\n")
find=0
for line in OUTPUT:
if line.find("Event: Resubmission")!=-1:
utils.log_info("Check OK, find resubmission event for job %s"%(JOBID))
find=1
break
if find==0:
utils.log_info("ERROR: Test failed, unable to find resubmission event for replan job %s"%(JOBID))
raise GeneralError("Check if resubmission event is logged for replan job %s"%(JOBID),"Unable to find resubmission event for replan job %s"%(JOBID))
utils.log_info("Check if job is aborted due to the maximum number of allowed replans")
utils.wait_until_job_finishes(JOBID)
status=utils.get_job_status(JOBID)
if status.find("Aborted")==-1:
utils.log_info("ERROR: TEST FAILED. Error job's final status is %s and not Aborted"%(status))
raise GeneralError("Check if job's status is Aborted","Error job's final status is %s and not Aborted"%(status))
else:
OUTPUT=utils.run_command("glite-wms-job-status -c %s %s"%(utils.get_config_file(),JOBID)).split("\n")
for line in OUTPUT:
if line.find("Status Reason")!=-1:
reason=line.split(":")[1].strip(" \n\t")
if reason.find("hit max number of replans")==-1:
utils.log_info("TEST FAILED. Aborted reason is '%s' while expected is 'hit max number of replans'"%(reason))
raise GeneralError("Check status reason","Aborted reason is %s while expected is 'hit max number of replans'"%(reason))
utils.log_info("Cancel the remaining jobs")
for JOBID in JOBIDS:
if utils.job_is_finished(JOBID)==0:
utils.run_command("glite-wms-job-cancel -c %s --noint %s"%(utils.get_config_file(),JOBID))
def set_MaxOSB(utils,ssh):
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf",["MaxOutputSandboxSize"],["*"],["50M"])
def test1(utils, title):
names,ces=utils.get_target_ces()
fails=0
if len(names)==0:
names.append("Default Test , Submit to CREAM")
ces.append("")
for i in range(len(names)):
utils.show_progress("%s - %s"%(title,names[i]))
utils.info("%s - %s"%(title,names[i]))
try:
utils.info("Set MaxOuputSandboxSize=50M; to glite_wms.conf at WMS")
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['MaxOutputSandboxSize'],['*'],['50M'])
utils.info("Restart workload manager glite-wms-wm")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
utils.info("Submit the job to CE and wait to finish")
set_huge_jdl(utils,utils.get_jdl_file())
if len(ces[i])>0:
utils.set_requirements("%s && %s"%(ces[i],utils.DEFAULTREQ))
else:
utils.set_requirements("%s"%utils.DEFAULTREQ)
utils.run_command_continue_on_error("rm -rf %s/*"%(utils.get_job_output_dir()))
res=Job_utils.submit_only_normal_job(utils,ces[i])
JOBID=res[1]
utils.info("Wait until job finished")
utils.wait_until_job_finishes(JOBID)
utils.info("Check logging info")
result=utils.run_command_continue_on_error("glite-wms-job-logging-info -v 3 --event UserTag %s"%(JOBID))
if result.find("OSB quota exceeded for") == -1:
utils.error("Not found message 'OSB quota exceeded for' at UserTag")
raise GeneralError("Check UserTag event","Not found message 'OSB quota exceeded for' at UserTag")
else:
utils.info("Find message 'OSB quota exceeded for' at UserTag event")
if result.find("Truncated last 52428800 bytes for file") == -1:
utils.error("Not found message 'Truncated last 52428800 bytes for file")
raise GeneralError("Check UserTag event","Not found message 'Truncated last 52428800 bytes for file' at UserTag")
else:
utils.info("Find message 'Truncated last 52428800 bytes for file' at UserTag event")
utils.info("Get job output")
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1 :
utils.remove(utils.get_tmp_file())
utils.info("Retrieve the output")
utils.run_command_continue_on_error ("glite-wms-job-output --nosubdir --noint --dir %s %s >> %s"%(utils.get_job_output_dir(),JOBID,utils.get_tmp_file()))
if os.path.isfile("%s/huge.tail"%(utils.get_job_output_dir())) :
utils.info("Output file (huge.tail) is correctly retrieved")
else:
utils.error("Output file (huge.tail) is not correctly retrieved")
raise GeneralError("Check output file","Output file (huge.tail) is not correctly retrieved")
utils.info("Check the size of the output file")
output=utils.run_command_continue_on_error("ls -l %s/"%(utils.get_job_output_dir()))
if output.find("52428800")!=-1:
utils.info("huge.tail size is 52428800 bytes as expected")
else:
utils.error("huge.tail size is not 52428800 bytes as expected. We get %s"%(ouput))
raise GeneralError("Check the size of the ouput file","huge.tail size is not 52428800 bytes as expected")
else:
utils.error("Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
raise GeneralError("Check final job status","Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
fails=fails+1
def test2(utils,ssh,title):
utils.show_progress(title)
utils.info(title)
try:
limit_values="--load1 22 --load5 0.01 --load15 18 --memusage 99 --diskusage 95 --fdnum 1000 --jdnum 150000 --ftpconn 300"
job_submit="\"${WMS_LOCATION_SBIN}/glite_wms_wmproxy_load_monitor --oper jobSubmit %s\""%(limit_values)
job_register="\"${WMS_LOCATION_SBIN}/glite_wms_wmproxy_load_monitor --oper jobRegister %s\""%(limit_values)
utils.info("Set jobSubmit = %s; and jobRegister = %s; to glite_wms.conf at WMS"%(job_submit,job_register))
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['jobSubmit','jobRegister'],['*','*'],[job_submit,job_register])
utils.info("Restart workload manager proxy glite-wms-wmproxy")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wmproxy restart")
utils.set_isb_jdl(utils.get_jdl_file())
utils.info("Submit a job , submission should be failed")
OUTPUT=utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()),1).split("\n")
utils.info("Check failed message")
find=0
for line in OUTPUT:
if line.find("System load is too high")!=-1:
utils.info("Check OK, the detected failure reason is: 'System load is too high'")
find=1
break
if find==0:
utils.error("Faulure reason is not 'System load is too high' as expected.")
raise GeneralError("Check faulure reason","Failure reason is not 'System load is too high' as expected")
utils.info("Check if the detected threshold is for parameter --load5, 'Load Average(5 min)'")
find=0
for line in OUTPUT:
if line.find("Threshold for Load Average(5 min):")!=-1:
utils.info("Check OK, detected threshold is for parameter --load5 'Load Average5 min)' as expected")
find=1
break
if find==0:
utils.error("Detected threshold is not for parameter load5 ('Load Average(5 min)') as expected")
raise GeneralError("Check the parameter of the limiter mechanism","Detected threshold is not for parameter load5 ('Load Average(5 min)') as expected ")
for line in OUTPUT:
if line.find("Method")!=-1:
utils.info("Get Method: %s"%(line))
break
utils.info("Test OK")
utils.info("Restore initial version of glite-wms.conf file")
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wmproxy restart")
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
utils.info("Restore initial version of glite-wms.conf file")
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wmproxy restart")
return 1
