def check_for_expired_proxies(utils,ssh,wms_location_var):
proxies=SSH_utils.execute_remote_cmd(utils,ssh,"find %s/proxycache/ -name \*.pem"%(wms_location_var)).split("\n")
for proxy in proxies:
if proxy.strip()!='':
expiry_date=SSH_utils.execute_remote_cmd(utils,ssh,"openssl x509 -in %s -noout -enddate"%(proxy.strip())).split("=")[1].strip("\n")
ddate_str=time.strptime(expiry_date,"%b %d %H:%M:%S %Y %Z")[0:8]
dt = datetime.datetime(ddate_str[0],ddate_str[1],ddate_str[2],ddate_str[3],ddate_str[4],ddate_str[5],ddate_str[6])
now_dt = datetime.datetime.utcnow()
diff=now_dt-dt
minutes, seconds = divmod(diff.seconds, 60)
hours, minutes = divmod(minutes, 60)
if diff.days>=0:
#Maybe it is necessary to check and minutes
if hours>=6:
utils.log_info("ERROR: Find expired proxy for more than 6 hours. Proxy is %s"%(proxy))
raise GeneralError("Check for expired proxies for more than 6 hours","Find expired proxy for more than 6 hours. Proxy is %s"%(proxy))
def set_limiter_values(utils,ssh,limit_values):
try:
job_submit="\"${WMS_LOCATION_SBIN}/glite_wms_wmproxy_load_monitor --oper jobSubmit %s\""%(limit_values)
job_register="\"${WMS_LOCATION_SBIN}/glite_wms_wmproxy_load_monitor --oper jobRegister %s\""%(limit_values)
utils.log_info("Set jobSubmit = %s; and jobRegister = %s; to glite_wms.conf at WMS"%(job_submit,job_register))
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['jobSubmit','jobRegister'],['*','*'], [job_submit,job_register])
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_info('ERROR - Command: %s'%(e.expression))
utils.log_info('ERROR - Message: %s'%(e.message))
restore_configuration_file(utils,ssh)
def test6(utils,title):
utils.show_progress(title)
utils.info(title)
fails=0
try:
if utils.OSB_DEST_HOSTNAME=='' or utils.OSB_DEST_USERNAME=='' or utils.OSB_DEST_PASSWORD=='' :
utils.warn("Please set the required variables for OSB node in test's configuration file")
utils.show_progress("Please set the required variables for OSB node in test's configuration file")
return 1
ssh=SSH_utils.open_ssh(utils.OSB_DEST_HOSTNAME,utils.OSB_DEST_USERNAME,utils.OSB_DEST_PASSWORD)
SSH_utils.execute_remote_cmd(ssh,"mkdir -p -m 0777 /tmp/%s/osbbase"%(utils.ID))
target_info,target_ces=utils.get_target_ces()
if len(target_info)==0:
target_info.append("Default Test - Submit to CREAM CE")
target_ces.append("/cream-")
for target in target_ces:
utils.show_progress("%s - %s"%(title,target_info[target_ces.index(target)]))
utils.info("%s - %s"%(title,target_info[target_ces.index(target)]))
utils.run_command_continue_on_error("rm -rf %s/*"%(utils.get_job_output_dir()))
if osbbasetdesturi_test(utils,target,ssh)==1:
fails=fails+1
SSH_utils.close_ssh(ssh)
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
fails=fails+1
def check_for_empty_directories(utils,ssh,wms_location_var):
dirs = SSH_utils.execute_remote_cmd(utils,ssh,"ls /%s/proxycache"%(wms_location_var)).split(" ")
for dir in dirs:
dir=dir.strip(" \n\t")
if dir.find("cache")==-1:
utils.log_info("Check directory: %s"%(dir))
subdirs=SSH_utils.execute_remote_cmd(utils,ssh,"find /%s/proxycache/%s -name userproxy.pem | wc -l"%(wms_location_var,dir))
total_subdirs=SSH_utils.execute_remote_cmd(utils,ssh,"ls -l /%s/proxycache/%s | grep glite | wc -l"%(wms_location_var,dir))
if total_subdirs == subdirs :
utils.log_info("Check OK, there are no empty directories in %s"%(dir))
else:
utils.log_info("ERROR: Test Failed. There are empty directories in %s"%(dir))
raise GeneralError("Check for emptry directories","Test Failed. There are empty directories in %s"%(dir))
def main():
utils = Test_utils.Test_utils(sys.argv[0],"Check glite_wms.conf file")
tests=["Test 1: Check common section"]
tests.append("Test 2: Check JobController section")
tests.append("Test 3: Check NetworkServer section")
tests.append("Test 4: Check LogMonitor section")
tests.append("Test 5: Check WorkloadManager section")
tests.append("Test 6: Check WorkloadManagerProxy section")
tests.append("Test 7: Check ICE section")
tests.append("Test 8: Check if attributes match values of yaim variables")
utils.prepare(sys.argv[1:],tests)
if utils.WMS_USERNAME=='' or utils.WMS_PASSWORD=='':
utils.warn("Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file")
utils.show_progress("Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file")
sys.exit(0)
signal.signal(signal.SIGINT,utils.exit_interrupt)
fails=[]
all_tests=utils.is_all_enabled()
utils.info("Get glite_wms.conf file")
try:
logging.info("Get glite_wms.conf file from remote host %s"%(utils.get_WMS()))
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
target='/etc/glite-wms/glite_wms.conf'
SSH_utils.ssh_get_file(ssh,target,"%s/glite_wms.conf_local"%(utils.get_tmp_dir()))
SSH_utils.close_ssh(ssh)
except (GeneralError) , e:
utils.log_error("Unable to get the glite_wms.conf from remote host: %s"%(utils.get_WMS()))
utils.log_error(e)
utils.exit_failure("Unable to get the glite_wms.conf from remote host: %s %s %s"%(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD))
def test7_target(utils,ssh,wms_location_var):
proxies=SSH_utils.execute_remote_cmd(utils,ssh,"find %s/SandboxDir/ -name user.proxy"%(wms_location_var)).split("\n")
has_expired=0
for proxy in proxies:
if proxy.strip()!='':
expiry_date=SSH_utils.execute_remote_cmd(utils,ssh,"openssl x509 -in %s -noout -enddate"%(proxy.strip())).split("=")[1].strip("\n")
ddate_str=time.strptime(expiry_date,"%b %d %H:%M:%S %Y %Z")[0:8]
dt = datetime.datetime(ddate_str[0],ddate_str[1],ddate_str[2],ddate_str[3],ddate_str[4],ddate_str[5],ddate_str[6])
now_dt = datetime.datetime.utcnow()
diff=now_dt-dt
minutes, seconds = divmod(diff.seconds, 60)
hours, minutes = divmod(minutes, 60)
if diff.days>=0:
if minutes>0 or hours>0:
target_proxy=proxy
has_expired=1
break
if has_expired==0:
utils.log_info("Skip test: Unable to find any expired proxy in %s/SandboxDir"%(wms_location_var))
raise GeneralError("","Skip test: Unable to find any expired proxy in %s/SandboxDir"%(wms_location_var))
target_dir=target_proxy.split("/https")[0]
return target_dir
def check_nodes_info(utils,ssh,nodes_prefix,nodes_dir):
x=0
for ndprefix in nodes_prefix:
output=SSH_utils.execute_remote_cmd_failed(utils,ssh,"ls -l /var/SandboxDir/%s/%s"%(ndprefix,nodes_dir[x]))
if output.find("No such file or directory")==-1:
utils.log_info("Command failed for some other reason. Expected reason 'No such file or directory'")
raise GeneralError("Execute command ls -l /var/SandboxDir/%s/%s"%(ndprefix,nodes_dir[x]),"Command failed for some other reason. Expected reasaon 'No such file or directory'")
x=x+1
def get_nodes_dir(utils,ssh,jobid):
output=utils.run_command("glite-wms-job-status -c %s %s"%(utils.get_config_file(),jobid)).split("\n")
nodes_dir=[]
for line in output :
if line.find("https://")!=-1 and line.find(jobid)==-1:
id=line.split("Status info for the Job : ")[1]
ndprefix=id.split(":9000/")[1][0:2]
nodes_dir.append(SSH_utils.execute_remote_cmd(utils,ssh,"ls /var/SandboxDir/%s"%(ndprefix)))
return nodes_dir
def main():
fails=[]
tests=["Test 1: Test option --load1 threshold for load average (1 min)"]
tests.append("Test 2: Test option --load5 threshold for load average (5 min)")
tests.append("Test 3: Test option --load15 threshold for load average (15 min)")
tests.append("Test 4: Test option --memusage threshold for memory usage (%)")
tests.append("Test 5: Test option --swapusage threshold for swap usage (%)")
tests.append("Test 6: Test option --ftpconn threshold for number of FTP connections")
tests.append("Test 7: Test option --diskusage threshold for disk usage (%)")
tests.append("Test 8: Test option --fdnum threshold for used file descriptor")
tests.append("Test 9: Test option --jdnum threshold for number of unprocessed jobs (for jobdir)")
tests.append("Test 10: Test option --jdsize threshold for input jobdir size (KB)")
#tests.append("Test 11: Test option --flnum threshold for number of unprocessed jobs (for filelist)")
#tests.append("Test 12: Test option --flsize threshold for input filelist size (KB)")
utils = Test_utils.Test_utils(sys.argv[0],"WMS Limiter Mechanism")
utils.prepare(sys.argv[1:],tests)
signal.signal(signal.SIGINT,utils.exit_interrupt)
if utils.WMS_USERNAME=='' or utils.WMS_PASSWORD=='':
utils.warn("Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file")
utils.show_progress("Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file")
sys.exit(0)
utils.info("WMS Limiter Mechanism Testing")
all_tests=utils.is_all_enabled()
try:
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
except (GeneralError) , e:
utils.log_error("Unable to connect to remote host: %s"%(utils.get_WMS()))
utils.log_error(e)
utils.exit_failure("Unable to connect to remote host: %s"%(utils.get_WMS()))
def main():
fails = []
tests = ["Test 1: Test purge , normal job cycle ( Submit to LCG CE )"]
tests.append("Test 2: Test purge, normal job cycle ( Submit to CREAM CE )")
tests.append("Test 3: Test proxy cache purging")
tests.append("Test 4: SandBoxDir cron purger ( Submit a job and force its purge before it finishes )")
tests.append("Test 5: SandBoxDir cron purger ( Purge a DONE job )")
tests.append("Test 6: SandBoxDir cron purger ( Purge only jobs older than 12 hours )")
tests.append("Test 7: SandBoxDir cron purger ( Purge a job which proxy is already expired )")
tests.append("Test 8: SandBoxDir cron purger ( Purge a DAG job )")
utils = Test_utils.Test_utils(sys.argv[0], "WMS Purge Mechanism")
utils.prepare(sys.argv[1:], tests)
signal.signal(signal.SIGINT, utils.exit_interrupt)
if utils.WMS_USERNAME == "" or utils.WMS_PASSWORD == "":
utils.warn("Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file")
utils.show_progress(
"Please set the required variables WMS_USERNAME , WMS_PASSWORD in test's configuration file"
)
sys.exit(0)
utils.info("WMS Purge Mechanism Testing")
all_tests = utils.is_all_enabled()
try:
ssh = SSH_utils.open_ssh(utils.get_WMS(), utils.WMS_USERNAME, utils.WMS_PASSWORD)
except (GeneralError), e:
utils.log_error("Unable to connect to remote host: %s" % (utils.get_WMS()))
utils.log_error(e)
utils.exit_failure("Unable to connect to remote host %s" % (utils.get_WMS()))
def test5(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
Job_utils.prepare_normal_job(utils, utils.get_jdl_file())
utils.info("Submit a job to a CREAM CE")
JOBID = Job_utils.submit_only_normal_job(utils)[1]
prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2]
utils.info("Wait until job finishes")
utils.wait_until_job_finishes(JOBID)
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1:
utils.info("Purge a done job")
# WARNING EXECUTE COMMAND AS root INSTEAD OF glite USER
utils.info(
"Execute '/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s' on remote host" % (prefix)
)
output = SSH_utils.execute_remote_cmd(
ssh, "/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s" % (prefix)
)
utils.info("Check glite-wms-purgeStorage.sh output")
if output.find("%s: removed DONE job" % (JOBID)) != -1:
utils.info("glite-wms-purgeStorage.sh successfully remove the DONE job %s" % (JOBID))
else:
utils.error("glite-wms-purgeStorage.sh didn't remove successfully the DONE job %s" % (JOBID))
raise GeneralError(
"Check glite-wms-purgeStorage.sh",
"glite-wms-purgeStorage.sh didn't remove successfully the DONE job %s" % (JOBID),
)
utils.info("Check again job's status")
utils.job_status(JOBID)
if utils.get_job_status().find("Cleared") != -1:
utils.info("Job's final status after purge is Cleared")
utils.info("Check job's status reason")
status = utils.get_job_status_reason(JOBID)
if status.find("timed out, resource purge forced") != -1:
utils.info("Status reason is 'timed out, resource purge forced' as expected")
else:
utils.error(
"Status reason is %s while the expected is 'timed out, resource purge forced'" % (status)
)
raise GeneralError(
"Check job's status reason",
"Status reason is %s while the expected is 'timed out, resource purge forced'" % (status),
)
else:
utils.error(
"Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status())
)
raise GeneralError(
"Check job final status after purge",
"Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()),
)
else:
utils.error("Job finishes with status: %s while expected 'Done'" % (utils.get_job_status()))
raise GeneralError(
"Check job final status",
"Job finishes with status: %s while expected 'Done'" % (utils.get_job_status()),
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def test3(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
wms_location_var = SSH_utils.execute_remote_cmd(ssh, "echo $WMS_LOCATION_VAR")[:-1]
utils.info("Check inside $WMS_LOCATION_VAR/proxycache for expired proxies for more than 6 hours")
proxies = SSH_utils.execute_remote_cmd(ssh, "find %s/proxycache/ -name \*.pem" % (wms_location_var)).split("\n")
for proxy in proxies:
if proxy.strip() != "":
expiry_date = (
SSH_utils.execute_remote_cmd(ssh, "openssl x509 -in %s -noout -enddate" % (proxy.strip()))
.split("=")[1]
.strip("\n")
)
ddate_str = time.strptime(expiry_date, "%b %d %H:%M:%S %Y %Z")[0:8]
dt = datetime.datetime(
ddate_str[0], ddate_str[1], ddate_str[2], ddate_str[3], ddate_str[4], ddate_str[5], ddate_str[6]
)
now_dt = datetime.datetime.utcnow()
diff = now_dt - dt
minutes, seconds = divmod(diff.seconds, 60)
hours, minutes = divmod(minutes, 60)
if diff.days >= 0:
# Maybe it is necessary to check and minutes
if hours >= 6:
utils.error("Find expired proxy for more than 6 hours. Proxy is %s" % (proxy))
raise GeneralError(
"Check for expired proxies for more than 6 hours",
"Find expired proxy for more than 6 hours. Proxy is %s" % (proxy),
)
utils.info("Check if there are empty directories")
dirs = SSH_utils.execute_remote_cmd(ssh, "ls /%s/proxycache" % (wms_location_var)).split(" ")
for dir in dirs:
dir = dir.strip(" \n\t")
if dir.find("cache") == -1:
utils.info("Check directory: %s" % (dir))
subdirs = SSH_utils.execute_remote_cmd(
ssh, "find /var/proxycache/%s -name userproxy.pem | wc -l" % (dir)
)
total_subdirs = SSH_utils.execute_remote_cmd(
ssh, "ls -l /var/proxycache/%s | grep glite | wc -l" % (dir)
)
if total_subdirs == subdirs:
utils.info("Check OK, there are no empty directories in %s" % (dir))
else:
utils.error("Test Failed. There are empty directories in %s" % (dir))
raise GeneralError(
"Check for emptry directories", "Test Failed. There are empty directories in %s" % (dir)
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def test4(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
utils.set_long_jdl(utils.get_jdl_file())
utils.info("Submit a job")
JOBID = utils.run_command_continue_on_error(
"glite-wms-job-submit %s --config %s --nomsg %s"
% (utils.get_delegation_options(), utils.get_config_file(), utils.get_jdl_file())
)
utils.info("Job submitted successfuly. Returned JOBID: %s" % (JOBID))
prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2]
utils.job_status(JOBID)
while utils.get_job_status().find("Waiting") != -1:
utils.info("Wait 60 secs. Job's status is Waiting")
time.sleep(60)
utils.job_status(JOBID)
utils.info("Purge the job")
# WARNING EXECUTE COMMAND AS root INSTEAD OF glite USER
utils.info("Execute '/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s' on remote host" % (prefix))
output = SSH_utils.execute_remote_cmd(
ssh, "/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s" % (prefix)
)
utils.info("Check glite-wms-purgeStorage.sh output")
if output.find("%s: removed" % (JOBID)) != -1:
utils.info("glite-wms-purgeStorage.sh successfully remove the job %s" % (JOBID))
else:
utils.error("glite-wms-purgeStorage.sh didn't remove successfully the job %s" % (JOBID))
raise GeneralError(
"Check glite-wms-purgeStorage.sh",
"glite-wms-purgeStorage.sh didn't remove successfully the job %s" % (JOBID),
)
utils.info("Wait 60 secs")
time.sleep(60)
utils.info("Check again job's status")
utils.job_status(JOBID)
if utils.get_job_status().find("Cleared") != -1:
utils.info("Job's final status after purge is Cleared")
utils.info("Check job's status reason")
status = utils.get_job_status_reason(JOBID)
if status.find("timed out, resource purge forced") != -1:
utils.info("Status reason is 'timed out, resource purge forced' as expected")
else:
utils.error("Status reason is %s while the expected is 'timed out, resource purge forced'" % (status))
raise GeneralError(
"Check job's status reason",
"Status reason is %s while the expected is 'timed out, resource purge forced'" % (status),
)
else:
utils.error("Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()))
raise GeneralError(
"Check job final status after purge",
"Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()),
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
if all_tests == 1 or utils.check_test_enabled(4) == 1:
if test4(utils, ssh, tests[3]):
fails.append(tests[3])
if all_tests == 1 or utils.check_test_enabled(5) == 1:
if test5(utils, ssh, tests[4]):
fails.append(tests[4])
if all_tests == 1 or utils.check_test_enabled(6) == 1:
if test6(utils, ssh, tests[5]):
fails.append(tests[5])
if all_tests == 1 or utils.check_test_enabled(7) == 1:
if test7(utils, ssh, tests[6]):
fails.append(tests[6])
if all_tests == 1 or utils.check_test_enabled(8) == 1:
if test8(utils, ssh, tests[7]):
fails.append(tests[7])
SSH_utils.close_ssh(ssh)
if len(fails) > 0:
utils.exit_failure("%s test(s) fail(s): %s" % (len(fails), fails))
else:
utils.exit_success()
if __name__ == "__main__":
main()
def test1(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
Job_utils.prepare_normal_job(utils, utils.get_jdl_file(), "2119/jobmanager")
utils.info("Submit a job to a LCG CE")
JOBID = Job_utils.submit_only_normal_job(utils, "2119/jobmanager")[1]
utils.info("Check the SBD and the others file used by the services")
prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2]
dir2 = "/var/jobcontrol/condorio/%s" % (prefix)
utils.info("Check files at %s" % (dir2))
condorio = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir2))
for line in condorio.split("\n"):
if line.find(JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1]) != -1:
target = "https_%s" % (line.lstrip().split("https_")[1])
dir1 = "/var/SandboxDir/%s/%s" % (prefix, target)
utils.info("Check files at %s" % (dir1))
SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir1))
dir3 = "/var/jobcontrol/submit/%s/" % (prefix)
utils.info("Check files at %s" % (dir3))
SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir3))
utils.info("Wait until job finishes")
utils.wait_until_job_finishes(JOBID)
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1:
utils.info("Check on WMS auxiliary files should be removed")
utils.info("Check directory %s" % (dir3))
output = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir3))
if output.find(target) != 1:
utils.error("Auxiliary files are not removed from directory %s" % (dir3))
raise GeneralError(
"Check for auxiliary files at %s" % (dir3),
"Auxiliary files are not removed from directory %s" % (dir3),
)
else:
utils.info("Auxiliary files are removed as expected from directory %s" % (dir3))
utils.info("Check directory %s" % (dir2))
output = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir2))
if output.find(target) != 1:
utils.error("Auxiliary files are not removed from directory %s" % (dir2))
raise GeneralError(
"Check for auxiliary files at %s" % (dir2),
"Auxiliary files are not removed from directory %s" % (dir2),
)
else:
utils.info("Auxiliary files are removed as expected from directory %s" % (dir2))
utils.info("Retrieve job output")
utils.run_command_continue_on_error(
"glite-wms-job-output --nosubdir --noint --dir %s %s" % (utils.get_job_output_dir(), JOBID)
)
utils.info("Check that also the SBD has been removed on WMS")
utils.info("Check directory /var/SandboxDir/%s/" % (prefix))
output = SSH_utils.execute_remote_cmd(ssh, "ls -l /var/SandboxDir/%s/" % (prefix))
if output.find(target) != 1:
utils.error("SBD not removed from directory /var/SandboxDir/%s/" % (prefix))
raise GeneralError(
"Check for SBD at /var/SandboxDir/%s/" % (prefix),
"Auxiliary files are not removed from directory %s" % (dir2),
)
else:
utils.info("SBD has been removed as expected from directory /var/SandboxDir/%s/" % (prefix))
else:
utils.error("Job finishes with status: %s cannot retrieve output" % (utils.get_job_status()))
raise GeneralError(
"Check job final status",
"Job finishes with status: %s cannot retrieve output" % (utils.get_job_status()),
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def test8(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
utils.set_dag_jdl(utils.get_jdl_file())
utils.info("Submit a DAG job")
JOBID = utils.run_command_continue_on_error(
"glite-wms-job-submit %s --config %s --nomsg %s"
% (utils.get_delegation_options(), utils.get_config_file(), utils.get_jdl_file())
)
utils.info("Job submitted successfuly. Returned JOBID: %s" % (JOBID))
output = utils.run_command_continue_on_error(
"glite-wms-job-status -c %s %s" % (utils.get_config_file(), JOBID)
).split("\n")
nodes_prefix = []
nodes_dir = []
for line in output:
if line.find("https://") != -1 and line.find(JOBID) == -1:
id = line.split("Status info for the Job : ")[1]
ndprefix = id.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2]
nodes_prefix.append(ndprefix)
nodes_dir.append(SSH_utils.execute_remote_cmd(ssh, "ls /var/SandboxDir/%s" % (ndprefix)))
prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2]
utils.info("Purge a DAG job")
# WARNING EXECUTE COMMAND AS root INSTEAD OF glite USER
utils.info("Execute '/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s' on remote host" % (prefix))
output = SSH_utils.execute_remote_cmd(
ssh, "/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s" % (prefix)
)
utils.info("Check glite-wms-purgeStorage.sh output")
if output.find("%s: 3/3 nodes removed" % (JOBID)) != -1:
utils.info("glite-wms-purgeStorage.sh successfully remove all the nodes of job %s" % (JOBID))
else:
utils.error("glite-wms-purgeStorage.sh didn't remove successfully all the nodes of job %s" % (JOBID))
raise GeneralError(
"Check glite-wms-purgeStorage.sh",
"glite-wms-purgeStorage.sh didn't remove successfully all the nodes of job %s" % (JOBID),
)
if output.find("%s: removed" % (JOBID)) != -1:
utils.info("glite-wms-purgeStorage.sh successfully remove the job %s" % (JOBID))
else:
utils.error("glite-wms-purgeStorage.sh didn't remove successfully the job %s" % (JOBID))
raise GeneralError(
"Check glite-wms-purgeStorage.sh",
"glite-wms-purgeStorage.sh didn't remove successfully the job %s" % (JOBID),
)
utils.info("Check the SandBoxDir of nodes")
x = 0
for ndprefix in nodes_prefix:
output = SSH_utils.execute_remote_cmd_failed(ssh, "ls -l /var/SandboxDir/%s/%s1" % (ndprefix, nodes_dir[x]))
if output.find("No such file or directory") == -1:
utils.error("Command failed for some other reason. Expected reasaon 'No such file or directory'")
raise GeneralError(
"Execute command ls -l /var/SandboxDir/%s/%s" % (ndprefix, nodes_dir[x]),
"Command failed for some other reason. Expected reasaon 'No such file or directory'",
)
x = x + 1
utils.info("Check job's final status")
utils.job_status(JOBID)
if utils.get_job_status().find("Cleared") != -1:
utils.info("Job's final status after purge is Cleared")
else:
utils.error("Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()))
raise GeneralError(
"Check job final status after purge",
"Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()),
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def test1(utils, title):
names,ces=utils.get_target_ces()
fails=0
if len(names)==0:
names.append("Default Test , Submit to CREAM")
ces.append("")
for i in range(len(names)):
utils.show_progress("%s - %s"%(title,names[i]))
utils.info("%s - %s"%(title,names[i]))
try:
utils.info("Set MaxOuputSandboxSize=50M; to glite_wms.conf at WMS")
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['MaxOutputSandboxSize'],['*'],['50M'])
utils.info("Restart workload manager glite-wms-wm")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
utils.info("Submit the job to CE and wait to finish")
set_huge_jdl(utils,utils.get_jdl_file())
if len(ces[i])>0:
utils.set_requirements("%s && %s"%(ces[i],utils.DEFAULTREQ))
else:
utils.set_requirements("%s"%utils.DEFAULTREQ)
utils.run_command_continue_on_error("rm -rf %s/*"%(utils.get_job_output_dir()))
res=Job_utils.submit_only_normal_job(utils,ces[i])
JOBID=res[1]
utils.info("Wait until job finished")
utils.wait_until_job_finishes(JOBID)
utils.info("Check logging info")
result=utils.run_command_continue_on_error("glite-wms-job-logging-info -v 3 --event UserTag %s"%(JOBID))
if result.find("OSB quota exceeded for") == -1:
utils.error("Not found message 'OSB quota exceeded for' at UserTag")
raise GeneralError("Check UserTag event","Not found message 'OSB quota exceeded for' at UserTag")
else:
utils.info("Find message 'OSB quota exceeded for' at UserTag event")
if result.find("Truncated last 52428800 bytes for file") == -1:
utils.error("Not found message 'Truncated last 52428800 bytes for file")
raise GeneralError("Check UserTag event","Not found message 'Truncated last 52428800 bytes for file' at UserTag")
else:
utils.info("Find message 'Truncated last 52428800 bytes for file' at UserTag event")
utils.info("Get job output")
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1 :
utils.remove(utils.get_tmp_file())
utils.info("Retrieve the output")
utils.run_command_continue_on_error ("glite-wms-job-output --nosubdir --noint --dir %s %s >> %s"%(utils.get_job_output_dir(),JOBID,utils.get_tmp_file()))
if os.path.isfile("%s/huge.tail"%(utils.get_job_output_dir())) :
utils.info("Output file (huge.tail) is correctly retrieved")
else:
utils.error("Output file (huge.tail) is not correctly retrieved")
raise GeneralError("Check output file","Output file (huge.tail) is not correctly retrieved")
utils.info("Check the size of the output file")
output=utils.run_command_continue_on_error("ls -l %s/"%(utils.get_job_output_dir()))
if output.find("52428800")!=-1:
utils.info("huge.tail size is 52428800 bytes as expected")
else:
utils.error("huge.tail size is not 52428800 bytes as expected. We get %s"%(ouput))
raise GeneralError("Check the size of the ouput file","huge.tail size is not 52428800 bytes as expected")
else:
utils.error("Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
raise GeneralError("Check final job status","Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
fails=fails+1
def set_drain(utils,ssh):
utils.log_info("Set WMS in draining mode")
SSH_utils.execute_remote_cmd(utils,ssh,"touch /var/.drain")
def unset_drain(utils,ssh):
utils.log_info("Unset WMS from draining mode")
SSH_utils.execute_remote_cmd(utils,ssh,"rm -f /var/.drain")
def get_location_var(utils,ssh): return SSH_utils.execute_remote_cmd(utils,ssh,"echo $WMS_LOCATION_VAR")[:-1]
def set_MaxOSB(utils,ssh):
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf",["MaxOutputSandboxSize"],["*"],["50M"])
def set_ISBBase(utils,filename):
utils.log_info("Define a jdl with ISBBaseURI attribute")
FILE=open(filename,"w")
FILE.write("Executable = \"test.sh\";\n")
FILE.write("StdOutput = \"std.out\";\n")
FILE.write("StdError = \"std.err\";\n")
FILE.write("FuzzyRank = true;\n")
FILE.write("InputSandboxBaseURI = \"gsiftp://%s:2811/tmp/%s/isb\";\n"%(utils.ISB_DEST_HOSTNAME,utils.ID))
FILE.write("InputSandbox = {\"test.sh\",\"test1.txt\"};\n")
FILE.write("OutputSandbox = {\"std.out\", \"std.err\"};\n")
FILE.write("DataAccessProtocol = \"gsiftp\";\n")
FILE.write("RetryCount = 1;\n")
FILE.write("ShallowRetryCount = 2;\n")
FILE.close()
utils.log_info("The saved jdl is:\n%s"%(commands.getoutput("cat %s"%(filename))),'DEBUG')
utils.log_info("Create executable script (test.sh) and test1.txt file at remote host: %s"%(utils.ISB_DEST_HOSTNAME))
ssh=SSH_utils.open_ssh(utils.ISB_DEST_HOSTNAME,utils.ISB_DEST_USERNAME,utils.ISB_DEST_PASSWORD,utils)
SSH_utils.execute_remote_cmd(utils,ssh,"rm -f /tmp/%s/isb/*"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"echo \"#!/bin/sh\" > /tmp/%s/isb/test.sh"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"echo $GLITE_WMS_JOBID >> /tmp/%s/isb/test.sh"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"echo \"ls -la\" >> /tmp/%s/isb/test.sh"%(utils.ID))
SSH_utils.execute_remote_cmd(utils,ssh,"touch /tmp/%s/isb/test1.txt"%(utils.ID))
SSH_utils.close_ssh(ssh,utils)
def osbdesturi_test(utils,target_ce,ssh):
SSH_utils.execute_remote_cmd(utils,ssh,"rm -f /tmp/%s/osbdest/*"%(utils.ID))
set_OSBDest(utils,utils.get_jdl_file())
if utils.EXTERNAL_REQUIREMENTS==0:
utils.set_requirements("%s"%utils.DEFAULTREQ)
else:
utils.set_requirements("%s && %s"%(target_ce,utils.DEFAULTREQ))
utils.log_info("Submit job")
JOBID=utils.run_command("glite-wms-job-submit %s -c %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.log_info("Wait until job finishes")
utils.wait_until_job_finishes(JOBID)
utils.log_info("Check job's output")
status=utils.get_job_status(JOBID)
if status.find("Done") != -1 :
utils.remove(utils.get_tmp_file())
utils.log_info("Retrieve the output")
utils.run_command("glite-wms-job-output --nosubdir --noint --dir %s %s >> %s"%(utils.get_job_output_dir(),JOBID, utils.get_tmp_file()))
utils.log_info("Check UI for std.err file")
if len(os.listdir(utils.get_job_output_dir()))==1:
utils.log_info("Find 1 file as expected , check its name")
if os.path.isfile("%s/std.err"%(utils.get_job_output_dir())):
utils.log_info("OK for std.err file")
else:
utils.log_info("ERROR: One file transferred to UI but not the expected (std.err)")
raise GeneralError("","One file has been transferred to UI but not the expected (std.err)")
else:
utils.log_info("ERROR: Wrong number of files have been transferred to UI")
raise GeneralError("Check UI for std.err file","Wrong number of files have been transferred to UI")
utils.log_info("Check if file std.out has been transferred to remote host %s"%(utils.OSB_DEST_HOSTNAME))
files=SSH_utils.execute_remote_cmd(utils,ssh,"ls /tmp/%s/osbdest/"%(utils.ID)).split(" ")
files = map(lambda s: s.strip(), files)
expected=['std.out']
if len(files)!=len(expected):
utils.log_info("ERROR: Wrong number of output files. Find %s while expected only %s"%(len(files),len(expected)))
raise GeneralError("Check number of ouput files","Wrong number of output files. Find %s while expected only %s"%(len(files),len(expected)))
if len(set(files)&set(expected))!=len(expected):
utils.log_info("ERROR: Unable to find all the expected output files. Find: %s , expected : %s"%(','.join(files),','.join(expected)))
raise GeneralError("Check output files","Unable to find all the expected output files. Find: %s , expected : %s"%(','.join(files),','.join(expected)))
utils.log_info("Check content of std.out file")
output=SSH_utils.execute_remote_cmd(utils,ssh,"cat /tmp/%s/osbdest/std.out"%(utils.ID))
values=[JOBID,'std.out','std.err','test.sh']
for value in values:
if output.find(value)!=-1:
utils.log_info("Find: %s"%(value))
else:
utils.log_info("ERROR: Unable to find '%s' in std.out"%(value))
raise GeneralError("Check content of std.out file","Unable to find '%s' in std.out"%(value))
else:
utils.log_info("ERROR: Job finishes with status: %s cannot retrieve output"%(status))
raise GeneralError("Check job status","Job finishes with status: %s cannot retrieve output"%(status))
def test3(utils, title):
utils.show_progress(title)
utils.info(title)
try:
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
utils.info("Get last access date for script /opt/lcg/sbin/grid_monitor.sh before submission")
output=SSH_utils.execute_remote_cmd(ssh, "ls -lu --time-style=full-iso `locate grid_mon`")
before_access=''
for line in output.split("\n"):
if line.find("/opt/lcg/sbin/grid_monitor.sh")!=-1 and line.find("/usr/sbin/grid_monitor.sh")==-1:
before_access=line
if len(before_access)==0:
utils.error("Unable to find script /opt/lcg/sbin/grid_monitor.sh")
raise GeneralError("Check if script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission","Unable to find script /opt/lcg/sbin/grid_monitor.sh")
utils.info("Wait 10 secs")
time.sleep(10)
utils.info("Submit a job to GRAM CE")
utils.set_jdl(utils.get_jdl_file())
utils.set_destination_ce(utils.get_jdl_file(),"2119/jobmanager")
JOBID=utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.job_status(JOBID)
while utils.get_job_status().find("Ready")!=-1 or utils.get_job_status().find("Waiting")!=-1:
utils.info("Wait 30 secs")
time.sleep(30)
utils.job_status(JOBID)
utils.info("Get last access date for script /opt/lcg/sbin/grid_monitor.sh after submission")
output=SSH_utils.execute_remote_cmd(ssh, "ls -lu --time-style=full-iso `locate grid_mon`")
SSH_utils.close_ssh(ssh)
after_access=''
for line in output.split("\n"):
if line.find("/opt/lcg/sbin/grid_monitor.sh")!=-1 and line.find("/usr/sbin/grid_monitor.sh")==-1:
after_access=line
for value in before_access.split(" "):
if value.find(":")!=-1:
before_time=value.split(".")[0]
if value.find("-")!=-1 and value.find("->")==-1:
before_date=value
for value in after_access.split(" "):
if value.find(":")!=-1:
after_time=value.split(".")[0]
if value.find("-")!=-1 and value.find("->")==-1:
after_date=value
before=time.mktime(time.strptime("%s %s"%(before_date,before_time),"%Y-%m-%d %H:%M:%S"))
after=time.mktime(time.strptime("%s %s"%(after_date,after_time),"%Y-%m-%d %H:%M:%S"))
utils.info("Check if script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission")
if after>before:
utils.info("Check OK, script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission")
else:
utils.error("Test failed, script /opt/lcg/sbin/grid_monitor.sh hasn't been used during the job submission")
utils.error("Access details before submission: %s %s"%(before_date,before_time))
utils.error("Access details after submission: %s %s"%(after_date,after_time))
raise GeneralError("Check if script /opt/lcg/sbin/grid_monitor.sh has been used during the job submission","Test failed, script /opt/lcg/sbin/grid_monitor.sh hasn't been used during the job submission")
utils.info("TEST PASS")
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
SSH_utils.close_ssh(ssh)
return 1
def test6(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
wms_location_var = SSH_utils.execute_remote_cmd(ssh, "echo $WMS_LOCATION_VAR")[:-1]
utils.info("Check inside $WMS_LOCATION_VAR/SandboxDir for uncleared jobs")
available_jobs = SSH_utils.execute_remote_cmd(
ssh, "find %s/SandboxDir/ -name https*" % (wms_location_var)
).split("\n")
last_update = []
threshold_overcome = 0
threshold = 432
for job_dir in available_jobs:
if job_dir.strip() != "":
utils.info("Parse job id from directory %s" % (job_dir))
JOBID = convert_to_job(job_dir)
utils.info("Get the timestamp of the latest event for the job %s" % (JOBID))
timestamps = []
output = utils.run_command_continue_on_error("glite-wms-job-logging-info -v 2 %s" % (JOBID)).split(
"\n"
)
for line in output:
if line.find("- Timestamp") != -1:
timestamps.append(line.split("=")[1].strip(" \t\n"))
last_update.append(timestamps[len(timestamps) - 1])
utils.info("Find jobs which are older than 12 hours")
for update in last_update:
ddate_str = time.strptime(update, "%a %b %d %H:%M:%S %Y %Z")[0:8]
dt = int(
datetime.datetime(
ddate_str[0], ddate_str[1], ddate_str[2], ddate_str[3], ddate_str[4], ddate_str[5], ddate_str[6]
).strftime("%s")
)
now_dt = int(datetime.datetime.now().strftime("%s"))
if now_dt - dt > threshold:
threshold_overcome = threshold_overcome + 1
if threshold_overcome == 0:
utils.error("Test is skipped. There are no jobs older than 12 hours")
raise GeneralError(
"Purge jobs older than 12 hours", "Test is skipped. There are no jobs older than 12 hours"
)
utils.info("Purge jobs older than 12 hours")
# WARNING EXECUTE COMMAND AS root INSTEAD OF glite USER
utils.info(
"Execute '/usr/sbin/glite-wms-purgeStorage.sh -p %s/SandboxDir -t %s' on remote host"
% (wms_location_var, threshold)
)
output = SSH_utils.execute_remote_cmd(
ssh, "/usr/sbin/glite-wms-purgeStorage.sh -p %s/SandboxDir -t %s" % (wms_location_var, threshold)
).split("\n")
utils.info("Check the number of purged jobs")
removed = 0
for line in output:
if line.find(": removed") != -1:
removed = removed + 1
if removed != threshold_overcome:
utils.error("Removed %s jobs while expected %s" % (removed, threshold_overcome))
raise GeneralError(
"Check the number of purged jobs", "Removed %s jobs while expected %s" % (removed, threshold_overcome)
)
utils.info("Check again the %s/SandboxDir directory after job purging")
remaining_jobs = SSH_utils.execute_remote_cmd(
ssh, "find %s/SandboxDir/ -name https*" % (wms_location_var)
).split("\n")
if len(available_jobs) - removed != len(remaining_jobs):
utils.error("It seems not to be removed all the jobs which are older than 12 hours")
raise GeneralError(
"Check again the %s/SandboxDir directory after job purging",
"It seems not to be removed all the jobs which are older than 12 hours",
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def run_test(utils,ssh):
CREAMs=[]
utils.log_info("Set MaxReplansCount=5; and ReplanGracePeriod=10; to glite_wms.conf at WMS")
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['MaxReplansCount','ReplanGracePeriod','LogLevel'],['*','*','*'],['5','10','6'])
utils.log_info("Restart workload manager glite-wms-wm")
SSH_utils.execute_remote_cmd(utils,ssh,"/etc/init.d/glite-wms-wm restart")
utils.set_feedback_jdl(utils.get_jdl_file())
utils.log_info("Get available CREAM CEs")
CEs=utils.run_command("glite-wms-job-list-match -a -c %s %s"%(utils.get_config_file(),utils.get_jdl_file())).split("\n")
for CE in CEs:
if CE.find(":8443")!=-1:
CREAMs.append(CE.strip(" -\t\n").split(":")[0])
if len(CREAMs)>1:
utils.set_requirements("RegExp(\"%s*\", other.GlueCEUniqueID) || RegExp(\"%s*\", other.GlueCEUniqueID)"%(CREAMs[0],CREAMs[1]))
utils.log_info("Submit jobs to trigger feedback mechanism")
JOBIDS=[]
for i in range(0,10):
JOBIDS.append(utils.run_command("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file())))
utils.log_info("Wait 60 secs")
time.sleep(60)
counter=0
limit=10
find=0
target="%s/workload_manager_events.log"%(SSH_utils.execute_remote_cmd(utils,ssh,"echo $WMS_LOCATION_LOG").strip(" \n\t"))
while counter<limit :
for JOBID in JOBIDS:
utils.log_info("Check if replan mechanism is triggered for job %s"%(JOBID))
output=SSH_utils.execute_remote_cmd(utils,ssh,"grep \"created replanning request for job %s\" %s"%(JOBID,target))
if output!='':
utils.log_info("Found in workload_manager_events.log a replanning request for job %s"%(JOBID))
utils.log_info(output,'DEBUG')
find=1
break
if find==1:
break
time.sleep(60)
counter=counter+1
if find==0:
utils.log_info("ERROR: Timeout reached while checking if replan mechanism is triggered at least for one job")
raise TimeOutError("","Timeout reached while checking if replan mechanism is triggered at least for one job")
utils.log_info("Check if resubmission event is logged for replan job %s"%(JOBID))
OUTPUT=utils.run_command("glite-wms-job-logging-info -c %s %s"%(utils.get_config_file(),JOBID)).split("\n")
find=0
for line in OUTPUT:
if line.find("Event: Resubmission")!=-1:
utils.log_info("Check OK, find resubmission event for job %s"%(JOBID))
find=1
break
if find==0:
utils.log_info("ERROR: Test failed, unable to find resubmission event for replan job %s"%(JOBID))
raise GeneralError("Check if resubmission event is logged for replan job %s"%(JOBID),"Unable to find resubmission event for replan job %s"%(JOBID))
utils.log_info("Check if job is aborted due to the maximum number of allowed replans")
utils.wait_until_job_finishes(JOBID)
status=utils.get_job_status(JOBID)
if status.find("Aborted")==-1:
utils.log_info("ERROR: TEST FAILED. Error job's final status is %s and not Aborted"%(status))
raise GeneralError("Check if job's status is Aborted","Error job's final status is %s and not Aborted"%(status))
else:
OUTPUT=utils.run_command("glite-wms-job-status -c %s %s"%(utils.get_config_file(),JOBID)).split("\n")
for line in OUTPUT:
if line.find("Status Reason")!=-1:
reason=line.split(":")[1].strip(" \n\t")
if reason.find("hit max number of replans")==-1:
utils.log_info("TEST FAILED. Aborted reason is '%s' while expected is 'hit max number of replans'"%(reason))
raise GeneralError("Check status reason","Aborted reason is %s while expected is 'hit max number of replans'"%(reason))
utils.log_info("Cancel the remaining jobs")
for JOBID in JOBIDS:
if utils.job_is_finished(JOBID)==0:
utils.run_command("glite-wms-job-cancel -c %s --noint %s"%(utils.get_config_file(),JOBID))
def test7(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
wms_location_var = SSH_utils.execute_remote_cmd(ssh, "echo $WMS_LOCATION_VAR")[:-1]
utils.info("Check inside $WMS_LOCATION_VAR/SandboxDir for jobs with expired proxy")
proxies = SSH_utils.execute_remote_cmd(ssh, "find %s/SandboxDir/ -name user.proxy" % (wms_location_var)).split(
"\n"
)
has_expired = 0
for proxy in proxies:
if proxy.strip() != "":
expiry_date = (
SSH_utils.execute_remote_cmd(ssh, "openssl x509 -in %s -noout -enddate" % (proxy.strip()))
.split("=")[1]
.strip("\n")
)
ddate_str = time.strptime(expiry_date, "%b %d %H:%M:%S %Y %Z")[0:8]
dt = datetime.datetime(
ddate_str[0], ddate_str[1], ddate_str[2], ddate_str[3], ddate_str[4], ddate_str[5], ddate_str[6]
)
now_dt = datetime.datetime.utcnow()
diff = now_dt - dt
minutes, seconds = divmod(diff.seconds, 60)
hours, minutes = divmod(minutes, 60)
if diff.days >= 0:
if minutes > 0 or hours > 0:
target_proxy = proxy
has_expired = 1
break
if has_expired == 0:
utils.error("Skip test: Unable to find any expired proxy in %s/SandboxDir" % (wms_location_var))
raise GeneralError(
title, "Skip test: Unable to find any expired proxy in %s/SandboxDir" % (wms_location_var)
)
target_dir = target_proxy.split("/https")[0]
utils.info("Purge a job with expired proxy")
# WARNING EXECUTE COMMAND AS root INSTEAD OF glite USER
utils.info("Execute '/usr/sbin/glite-wms-purgeStorage.sh -p %s -s' on remote host" % (target_dir))
output = SSH_utils.execute_remote_cmd(ssh, "/usr/sbin/glite-wms-purgeStorage.sh -p %s -s" % (target_dir))
utils.info("Get job id ")
JOBID = output.split(": ")[2]
utils.info("Check job's status")
utils.job_status(JOBID)
if utils.get_job_status().find("Cleared") != -1:
utils.info("Job's final status after purge is Cleared")
utils.info("Check job's status reason")
status = utils.get_job_status_reason(JOBID)
if status.find("timed out, resource purge forced") != -1:
utils.info("Status reason is 'timed out, resource purge forced' as expected")
else:
utils.error("Status reason is %s while the expected is 'timed out, resource purge forced'" % (status))
raise GeneralError(
"Check job's status reason",
"Status reason is %s while the expected is 'timed out, resource purge forced'" % (status),
)
else:
utils.error("Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()))
raise GeneralError(
"Check job final status after purge",
"Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()),
)
utils.info("Check the user Clear event")
output = utils.run_command_continue_on_error(
"glite-wms-job-logging-info --event Clear -v 2 %s" % (JOBID)
).split("\n")
host = ""
user = ""
for line in output:
if line.find("- Host") != -1:
host = line.split(" = ")[1].strip(" \t\n")
if line.find("- User") != -1:
user = line.split(" = ")[1].strip(" \t\t")
if user.find(host) == -1:
utils.error("Unable to find the host proxy as user of the Clear event")
raise GeneralError(
"Check the user of the Clear event", "Unable to find the host proxy as user of the Clear event"
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def restore_configuration_file(utils,ssh):
utils.log_info("Restore initial version of glite-wms.conf file")
SSH_utils.execute_remote_cmd(utils,ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(utils,ssh,"/etc/init.d/glite-wms-wmproxy restart")
def restore_configuration(utils,ssh):
SSH_utils.execute_remote_cmd(utils,ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(utils,ssh,"/etc/init.d/glite-wms-wm restart")
def test2(utils, title):
utils.show_progress(title)
utils.info(title)
try:
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
set_test2_jdl(utils,utils.get_jdl_file())
utils.info("Submit a job")
JOBID=utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.info("Set WMS in draining mode")
SSH_utils.execute_remote_cmd(ssh,"touch /var/.drain")
utils.info("Try to submit jobs")
output=utils.run_command_continue_on_error("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()),1)
utils.info("Check error message")
if output.find("Unable to find any endpoint where to perform service request")==-1:
utils.error("Failed reason is not 'Unable to find any endpoint where to perform service request'")
raise GeneralError("Check error message","Failed reason is not 'Unable to find any endpoint where to perform service request'")
else:
utils.info("Failed reason is 'Unable to find any endpoint where to perform service request' as expected")
utils.info("Try list-match")
output=utils.run_command_continue_on_error("glite-wms-job-list-match %s --config %s %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()),1)
if output.find("Unable to find any endpoint where to perform service request")==-1:
utils.error("Failed reason is not 'Unable to find any endpoint where to perform service request'")
raise GeneralError("Check error message","Failed reason is not 'Unable to find any endpoint where to perform service request'")
else:
utils.info("Failed reason is 'Unable to find any endpoint where to perform service request' as expected")
utils.info("Check status of previously submitted job")
utils.wait_until_job_finishes(JOBID)
utils.info("Get job output")
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1 :
utils.remove(utils.get_tmp_file())
utils.info("Retrieve the output")
utils.run_command_continue_on_error ("glite-wms-job-output --nosubdir --noint --dir %s %s >> %s"%(utils.get_job_output_dir(),JOBID,utils.get_tmp_file()))
utils.info("Check if the output files are correctly retrieved")
if os.path.isfile("%s/std.out"%(utils.get_job_output_dir())) & os.path.isfile("%s/std.err"%(utils.get_job_output_dir())) :
utils.info("Output files are correctly retrieved")
else:
utils.error("Output files are not correctly retrieved")
raise GeneralError("Check output files","Output files are not correctly retrieved")
else:
utils.error("Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
raise GeneralError("Check final job status","Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
utils.info("Unset WMS from draining mode")
SSH_utils.execute_remote_cmd(ssh,"rm -f /var/.drain")
SSH_utils.close_ssh(ssh)
utils.info("TEST PASS")
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
utils.info("Unset WMS from draining mode")
SSH_utils.execute_remote_cmd(ssh,"rm -f /var/.drain")
SSH_utils.close_ssh(ssh)
return 1