def bulk_submit_with_single_jdl(utils, title):
fails=0
names,ces=utils.get_target_ces()
if len(names)==0:
names=["Submit to LCG-CE","Submit to CREAM CE","Submit without restrictions"]
ces=["2119/jobmanager","/cream-",""]
for i in range(len(names)):
utils.show_progress("%s - %s"%(title,names[i]))
utils.info("%s - %s"%(title,names[i]))
Job_utils.prepare_single_jdl_for_collection_job(utils,utils.get_jdl_file(),ces[i])
result=Job_utils.submit_collection_job(utils,ces[i],True)
if result[0] == 1 :
utils.info(result[1])
fails=fails+1
else:
utils.dbg("Clean job output directory")
os.system("rm -rf %s"%(utils.get_job_output_dir()))
return fails
def test5(utils, title):
# Cancel node of a collection job
utils.show_progress(title)
utils.info(title)
try:
utils.set_long_jdl(utils.get_jdl_file())
Job_utils.prepare_collection_job(utils, utils.get_jdl_file(), "/cream-")
utils.dbg("Submit a collection job")
JOBID = utils.run_command_continue_on_error(
"glite-wms-job-submit %s --config %s --nomsg --output %s --collection %s/collection_jdls"
% (utils.get_delegation_options(), utils.get_config_file(), utils.get_output_file(), utils.get_tmp_dir())
)
utils.info("Job submitted successfuly. Returned JOBID: %s" % (JOBID))
utils.dbg("Wait 60 secs")
time.sleep(60)
# Get nodes' ids
output = commands.getoutput("glite-wms-job-status %s" % (JOBID))
IDS = []
for line in output.splitlines():
if line.split(":", 1)[0].strip() == "Status info for the Job":
IDS.append(line.split(":", 1)[1].strip())
utils.dbg("Next node's id is: %s" % (IDS[-1]))
utils.info("Now try to cancel the first node")
# IDS[0] is the collection ID
utils.run_command_continue_on_error(
"glite-wms-job-cancel --config %s --noint %s" % (utils.get_config_file(), IDS[1])
)
if not check_status(utils, IDS[1]):
utils.info("TEST PASS")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def dag_submit(utils, title):
# Dag jobs can be submitted only to LCG-CE
utils.show_progress(title)
utils.info(title)
Job_utils.prepare_dag_job(utils,utils.get_jdl_file())
result=Job_utils.submit_dag_job(utils)
if result[0] == 1 :
utils.info(result[1])
return 1
else:
utils.dbg("Clean job output directory")
os.system("rm -rf %s"%(utils.get_job_output_dir()))
return 0
def test3(utils,title):
names,ces=utils.get_target_ces()
fails=0
if len(names)==0:
names.append("Default Test")
ces.append("")
for i in range(len(names)):
utils.show_progress("%s - %s"%(title,names[i]))
utils.info("%s - %s"%(title,names[i]))
try:
###Create jdl with ShortDeadlineJob attribute enabled
utils.set_isb_jdl(utils.get_jdl_file())
utils.add_jdl_general_attribute("RetryCount",1)
utils.add_jdl_general_attribute("ShortDeadlineJob","true")
if len(ces[i])>0:
utils.set_requirements("%s && %s"%(ces[i],utils.DEFAULTREQ))
else:
utils.set_requirements("%s"%utils.DEFAULTREQ)
output=utils.run_command_continue_on_error("glite-wms-job-list-match %s -c %s %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
if output.find("No Computing Element matching your job requirements has been found!")!=-1:
utils.error("No Computing Element matching your job requirements has been found!")
raise GeneralError("","No Computing Element matching your job requirements has been found!")
else:
result=Job_utils.submit_normal_job(utils,"")
if result[0] == 1 :
utils.error(result[1])
raise GeneralError("Job Submission",result[1])
else:
utils.dbg("Clean job output directory")
os.system("rm -rf %s"%(utils.get_job_output_dir()))
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
fails=fails+1
def test5(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
Job_utils.prepare_normal_job(utils, utils.get_jdl_file())
utils.info("Submit a job to a CREAM CE")
JOBID = Job_utils.submit_only_normal_job(utils)[1]
prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2]
utils.info("Wait until job finishes")
utils.wait_until_job_finishes(JOBID)
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1:
utils.info("Purge a done job")
# WARNING EXECUTE COMMAND AS root INSTEAD OF glite USER
utils.info(
"Execute '/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s' on remote host" % (prefix)
)
output = SSH_utils.execute_remote_cmd(
ssh, "/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s" % (prefix)
)
utils.info("Check glite-wms-purgeStorage.sh output")
if output.find("%s: removed DONE job" % (JOBID)) != -1:
utils.info("glite-wms-purgeStorage.sh successfully remove the DONE job %s" % (JOBID))
else:
utils.error("glite-wms-purgeStorage.sh didn't remove successfully the DONE job %s" % (JOBID))
raise GeneralError(
"Check glite-wms-purgeStorage.sh",
"glite-wms-purgeStorage.sh didn't remove successfully the DONE job %s" % (JOBID),
)
utils.info("Check again job's status")
utils.job_status(JOBID)
if utils.get_job_status().find("Cleared") != -1:
utils.info("Job's final status after purge is Cleared")
utils.info("Check job's status reason")
status = utils.get_job_status_reason(JOBID)
if status.find("timed out, resource purge forced") != -1:
utils.info("Status reason is 'timed out, resource purge forced' as expected")
else:
utils.error(
"Status reason is %s while the expected is 'timed out, resource purge forced'" % (status)
)
raise GeneralError(
"Check job's status reason",
"Status reason is %s while the expected is 'timed out, resource purge forced'" % (status),
)
else:
utils.error(
"Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status())
)
raise GeneralError(
"Check job final status after purge",
"Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()),
)
else:
utils.error("Job finishes with status: %s while expected 'Done'" % (utils.get_job_status()))
raise GeneralError(
"Check job final status",
"Job finishes with status: %s while expected 'Done'" % (utils.get_job_status()),
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def test1(utils, ssh, title):
utils.show_progress(title)
utils.info(title)
try:
Job_utils.prepare_normal_job(utils, utils.get_jdl_file(), "2119/jobmanager")
utils.info("Submit a job to a LCG CE")
JOBID = Job_utils.submit_only_normal_job(utils, "2119/jobmanager")[1]
utils.info("Check the SBD and the others file used by the services")
prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2]
dir2 = "/var/jobcontrol/condorio/%s" % (prefix)
utils.info("Check files at %s" % (dir2))
condorio = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir2))
for line in condorio.split("\n"):
if line.find(JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1]) != -1:
target = "https_%s" % (line.lstrip().split("https_")[1])
dir1 = "/var/SandboxDir/%s/%s" % (prefix, target)
utils.info("Check files at %s" % (dir1))
SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir1))
dir3 = "/var/jobcontrol/submit/%s/" % (prefix)
utils.info("Check files at %s" % (dir3))
SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir3))
utils.info("Wait until job finishes")
utils.wait_until_job_finishes(JOBID)
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1:
utils.info("Check on WMS auxiliary files should be removed")
utils.info("Check directory %s" % (dir3))
output = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir3))
if output.find(target) != 1:
utils.error("Auxiliary files are not removed from directory %s" % (dir3))
raise GeneralError(
"Check for auxiliary files at %s" % (dir3),
"Auxiliary files are not removed from directory %s" % (dir3),
)
else:
utils.info("Auxiliary files are removed as expected from directory %s" % (dir3))
utils.info("Check directory %s" % (dir2))
output = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir2))
if output.find(target) != 1:
utils.error("Auxiliary files are not removed from directory %s" % (dir2))
raise GeneralError(
"Check for auxiliary files at %s" % (dir2),
"Auxiliary files are not removed from directory %s" % (dir2),
)
else:
utils.info("Auxiliary files are removed as expected from directory %s" % (dir2))
utils.info("Retrieve job output")
utils.run_command_continue_on_error(
"glite-wms-job-output --nosubdir --noint --dir %s %s" % (utils.get_job_output_dir(), JOBID)
)
utils.info("Check that also the SBD has been removed on WMS")
utils.info("Check directory /var/SandboxDir/%s/" % (prefix))
output = SSH_utils.execute_remote_cmd(ssh, "ls -l /var/SandboxDir/%s/" % (prefix))
if output.find(target) != 1:
utils.error("SBD not removed from directory /var/SandboxDir/%s/" % (prefix))
raise GeneralError(
"Check for SBD at /var/SandboxDir/%s/" % (prefix),
"Auxiliary files are not removed from directory %s" % (dir2),
)
else:
utils.info("SBD has been removed as expected from directory /var/SandboxDir/%s/" % (prefix))
else:
utils.error("Job finishes with status: %s cannot retrieve output" % (utils.get_job_status()))
raise GeneralError(
"Check job final status",
"Job finishes with status: %s cannot retrieve output" % (utils.get_job_status()),
)
utils.info("TEST OK")
except (RunCommandError, GeneralError, TimeOutError), e:
utils.log_error("%s" % (utils.get_current_test()))
utils.log_error("Command: %s" % (e.expression))
utils.log_error("Message: %s" % (e.message))
utils.log_traceback("%s" % (utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
return 1
def main():
utils = Test_utils.Test_utils(sys.argv[0],"Test a complete job cycle: from submission to get output")
tests=["Set 1: Submit a normal job"]
tests.append("Set 2: Submit a bulk of jobs")
tests.append("Set 3: Submit a parametric job")
tests.append("Set 4: Submit a DAG job")
tests.append("Set 5: Submit a MPI job")
tests.append("Set 6: Submit a perusal job")
tests.append("Set 7: Submit a bulk of jobs using a single jdl with al the jdls of nodes")
tests.append("Set 8: Testing forwarding parameters for parallel jobs")
utils.prepare(sys.argv[1:],tests)
utils.info("Test a complete job cycle: from submission to get output")
signal.signal(signal.SIGINT,utils.exit_interrupt)
fails=[]
# This test is for direct submission to a given CE
if utils.get_user_CE() !='' :
utils.info ("Submit a normal job to %s"%(utils.get_user_CE()))
Job_utils.prepare_normal_job(utils,utils.get_jdl_file(),utils.get_user_CE())
result=Job_utils.submit_normal_job(utils,utils.get_user_CE())
if result[0] == 1 :
utils.warn(result[1])
utils.log_error("Submit a normal job to %s"%(utils.get_user_CE()))
utils.log_error(result[1])
utils.exit_failure(result[1])
utils.message("Clean job output directory")
os.system("rm -rf %s"%(utils.get_job_output_dir()))
else:
all_tests=utils.is_all_enabled()
if all_tests==1 or utils.check_test_enabled(1)==1 :
if normal_submit(utils, tests[0]):
fails.append(tests[0])
if all_tests==1 or utils.check_test_enabled(2)==1 :
if bulk_submit(utils, tests[1]):
fails.append(tests[1])
if all_tests==1 or utils.check_test_enabled(3)==1 :
if parametric_submit(utils, tests[2]):
fails.append(tests[2])
if all_tests==1 or utils.check_test_enabled(4)==1 :
if dag_submit(utils, tests[3]):
fails.append(tests[3])
if all_tests==1 or utils.check_test_enabled(5)==1 :
if parallel_submit(utils, tests[4]):
fails.append(tests[4])
if all_tests==1 or utils.check_test_enabled(6)==1 :
if perusal_submit(utils, tests[5]):
fails.append(tests[5])
if all_tests==1 or utils.check_test_enabled(7)==1 :
if bulk_submit_with_single_jdl(utils, tests[6]):
fails.append(tests[6])
if all_tests==1 or utils.check_test_enabled(8)==1 :
if forward_parameters_parallel_jobs(utils, tests[7]):
fails.append(tests[7])
if len(fails) > 0 :
utils.exit_failure("%s test(s) fail(s): %s"%(len(fails), fails))
else:
utils.exit_success()
def forward_parameters_parallel_jobs(utils, title):
fails=0
names,ces=utils.get_target_ces()
if len(names)==0:
names=["Submit to CREAM CE"]
ces=["/cream-"]
for i in range(len(names)):
try:
utils.show_progress("%s Case 1 - %s"%(title,names[i]))
utils.info("\t%s Case 1 - %s"%(title,names[i]))
Job_utils.prepare_mpi_job(utils,utils.get_jdl_file(),ces[i])
utils.change_jdl_attribute("CpuNumber","2")
utils.add_jdl_general_attribute("WholeNodes","true")
utils.add_jdl_general_attribute("SMPGranularity","2")
utils.add_jdl_general_attribute("Hostnumber","1")
check=['CpuNumber = 2','WholeNodes = true','SMPGranularity = 2','Hostnumber = 1']
errors=[]
utils.info("Submit MPI job")
JOBID=utils.run_command_continue_on_error ("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.info("Job submitted successfuly. Returned JOBID: %s"%(JOBID))
utils.info("Wait until job transefered to CE")
utils.wait_until_job_transfered(JOBID)
cream_jobid=utils.get_cream_jobid(JOBID)
utils.info("Get the resulting cream jdl")
cream_jdl=utils.get_cream_jdl(cream_jobid)
utils.info("Check the cream jdl for the forwarding parameters")
for attribute in check:
if cream_jdl.find(attribute)==-1:
errors.append(attribute)
if len(errors)>0:
msg=' , '.join(errors)
utils.error("Problem with the following parameters: %s"%(msg))
fails=fails+1
utils.show_progress("%s Case 2 - %s"%(title,names[i]))
utils.info("\t%s Case 2 - %s"%(title,names[i]))
Job_utils.prepare_mpi_job(utils,utils.get_jdl_file(),ces[i])
utils.change_jdl_attribute("CpuNumber","1")
utils.add_jdl_general_attribute("WholeNodes","true")
utils.add_jdl_general_attribute("SMPGranularity","2")
check=['CpuNumber = 1','WholeNodes = true','SMPGranularity = 2']
errors=[]
utils.info("Submit MPI job")
JOBID=utils.run_command_continue_on_error ("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.info("Job submitted successfuly. Returned JOBID: %s"%(JOBID))
utils.info("Wait until job transefered to CREAM CE")
utils.wait_until_job_transfered(JOBID)
cream_jobid=utils.get_cream_jobid(JOBID)
utils.info("Get the resulting cream jdl")
cream_jdl=utils.get_cream_jdl(cream_jobid)
utils.info("Check the cream jdl for the forwarding parameters")
for attribute in check:
if cream_jdl.find(attribute)==-1:
errors.append(attribute)
if len(errors)>0:
msg=' , '.join(errors)
utils.error("Problem with the following parameters: %s"%(msg))
fails=fails+1
utils.show_progress("%s Case 3 - %s"%(title,names[i]))
utils.info("\t%s Case 3 - %s"%(title,names[i]))
Job_utils.prepare_mpi_job(utils,utils.get_jdl_file(),ces[i])
utils.change_jdl_attribute("CpuNumber","3")
utils.add_jdl_general_attribute("WholeNodes","false")
utils.add_jdl_general_attribute("SMPGranularity","3")
utils.add_jdl_general_attribute("Hostnumber","1")
utils.info("Submit MPI job")
message=utils.run_command_continue_on_error ("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()),1)
utils.info("Check the error message")
if message.find("SMPGranularity and HostNumber are mutually exclusive when WholeNodes allocation is not requested: wrong combination of values")==-1:
utils.error("Job failed reason: %s. Expected reason: SMPGranularity and HostNumber are mutually exclusive when WholeNodes allocation is not requested: wrong combination of values"%(message))
fails=fails+1
utils.show_progress("%s Case 4 - %s"%(title,names[i]))
utils.info("\t%s Case 4 - %s"%(title,names[i]))
Job_utils.prepare_mpi_job(utils,utils.get_jdl_file(),ces[i])
utils.change_jdl_attribute("CpuNumber","3")
utils.add_jdl_general_attribute("WholeNodes","false")
utils.add_jdl_general_attribute("SMPGranularity","3")
check=['CpuNumber = 3','WholeNodes = false','SMPGranularity = 3']
errors=[]
utils.info("Submit MPI job")
JOBID=utils.run_command_continue_on_error ("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.info("Job submitted successfuly. Returned JOBID: %s"%(JOBID))
utils.info("Wait until job transefered to CREAM CE")
utils.wait_until_job_transfered(JOBID)
cream_jobid=utils.get_cream_jobid(JOBID)
utils.info("Get the resulting cream jdl")
cream_jdl=utils.get_cream_jdl(cream_jobid)
utils.info("Check the cream jdl for the forwarding parameters")
for attribute in check:
if cream_jdl.find(attribute)==-1:
errors.append(attribute)
if len(errors)>0:
msg=' , '.join(errors)
utils.error("Problem with the following parameters: %s"%(msg))
fails=fails+1
utils.show_progress("%s Case 5 - %s"%(title,names[i]))
utils.info("\t%s Case 5 - %s"%(title,names[i]))
Job_utils.prepare_mpi_job(utils,utils.get_jdl_file(),ces[i])
utils.change_jdl_attribute("CpuNumber","8")
utils.add_jdl_general_attribute("WholeNodes","true")
utils.add_jdl_general_attribute("SMPGranularity","8")
utils.add_jdl_general_attribute("Hostnumber","2")
check=['CpuNumber = 8','WholeNodes = true','SMPGranularity = 8','Hostnumber = 2']
errors=[]
utils.info("Submit MPI job")
JOBID=utils.run_command_continue_on_error ("glite-wms-job-submit %s --config %s --nomsg %s"%(utils.get_delegation_options(),utils.get_config_file(),utils.get_jdl_file()))
utils.info("Job submitted successfuly. Returned JOBID: %s"%(JOBID))
utils.info("Wait until job transefered to CREAM CE")
utils.wait_until_job_transfered(JOBID)
cream_jobid=utils.get_cream_jobid(JOBID)
utils.info("Get the resulting cream jdl")
cream_jdl=utils.get_cream_jdl(cream_jobid)
utils.info("Check the cream jdl for the forwarding parameters")
for attribute in check:
if cream_jdl.find(attribute)==-1:
errors.append(attribute)
if len(errors)>0:
msg=' , '.join(errors)
utils.error("Problem with the following parameters: %s"%(msg))
fails=fails+1
except (GeneralError,RunCommandError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
fails=fails+1
def test1(utils, title):
names,ces=utils.get_target_ces()
fails=0
if len(names)==0:
names.append("Default Test , Submit to CREAM")
ces.append("")
for i in range(len(names)):
utils.show_progress("%s - %s"%(title,names[i]))
utils.info("%s - %s"%(title,names[i]))
try:
utils.info("Set MaxOuputSandboxSize=50M; to glite_wms.conf at WMS")
ssh=SSH_utils.open_ssh(utils.get_WMS(),utils.WMS_USERNAME,utils.WMS_PASSWORD)
SSH_utils.change_remote_file(utils,ssh,"/etc/glite-wms/glite_wms.conf", ['MaxOutputSandboxSize'],['*'],['50M'])
utils.info("Restart workload manager glite-wms-wm")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
utils.info("Submit the job to CE and wait to finish")
set_huge_jdl(utils,utils.get_jdl_file())
if len(ces[i])>0:
utils.set_requirements("%s && %s"%(ces[i],utils.DEFAULTREQ))
else:
utils.set_requirements("%s"%utils.DEFAULTREQ)
utils.run_command_continue_on_error("rm -rf %s/*"%(utils.get_job_output_dir()))
res=Job_utils.submit_only_normal_job(utils,ces[i])
JOBID=res[1]
utils.info("Wait until job finished")
utils.wait_until_job_finishes(JOBID)
utils.info("Check logging info")
result=utils.run_command_continue_on_error("glite-wms-job-logging-info -v 3 --event UserTag %s"%(JOBID))
if result.find("OSB quota exceeded for") == -1:
utils.error("Not found message 'OSB quota exceeded for' at UserTag")
raise GeneralError("Check UserTag event","Not found message 'OSB quota exceeded for' at UserTag")
else:
utils.info("Find message 'OSB quota exceeded for' at UserTag event")
if result.find("Truncated last 52428800 bytes for file") == -1:
utils.error("Not found message 'Truncated last 52428800 bytes for file")
raise GeneralError("Check UserTag event","Not found message 'Truncated last 52428800 bytes for file' at UserTag")
else:
utils.info("Find message 'Truncated last 52428800 bytes for file' at UserTag event")
utils.info("Get job output")
utils.job_status(JOBID)
if utils.get_job_status().find("Done") != -1 :
utils.remove(utils.get_tmp_file())
utils.info("Retrieve the output")
utils.run_command_continue_on_error ("glite-wms-job-output --nosubdir --noint --dir %s %s >> %s"%(utils.get_job_output_dir(),JOBID,utils.get_tmp_file()))
if os.path.isfile("%s/huge.tail"%(utils.get_job_output_dir())) :
utils.info("Output file (huge.tail) is correctly retrieved")
else:
utils.error("Output file (huge.tail) is not correctly retrieved")
raise GeneralError("Check output file","Output file (huge.tail) is not correctly retrieved")
utils.info("Check the size of the output file")
output=utils.run_command_continue_on_error("ls -l %s/"%(utils.get_job_output_dir()))
if output.find("52428800")!=-1:
utils.info("huge.tail size is 52428800 bytes as expected")
else:
utils.error("huge.tail size is not 52428800 bytes as expected. We get %s"%(ouput))
raise GeneralError("Check the size of the ouput file","huge.tail size is not 52428800 bytes as expected")
else:
utils.error("Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
raise GeneralError("Check final job status","Job finishes with status: %s cannot retrieve output"%(utils.get_job_status()))
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
except (RunCommandError,GeneralError,TimeOutError) , e :
utils.log_error("%s"%(utils.get_current_test()))
utils.log_error("Command: %s"%(e.expression))
utils.log_error("Message: %s"%(e.message))
utils.log_traceback("%s"%(utils.get_current_test()))
utils.log_traceback(traceback.format_exc())
SSH_utils.execute_remote_cmd(ssh, "cp -f /etc/glite-wms/glite_wms.conf.bak /etc/glite-wms/glite_wms.conf")
SSH_utils.execute_remote_cmd(ssh,"/etc/init.d/glite-wms-wm restart")
SSH_utils.close_ssh(ssh)
fails=fails+1
