def normal_submit(utils, title): fails=0 names,ces=utils.get_target_ces() if len(names)==0: names=["Submit to LCG-CE","Submit to CREAM CE","Submit without restrictions"] ces=["2119/jobmanager","/cream-",""] for i in range(len(names)): utils.show_progress("%s - %s"%(title,names[i])) utils.info("%s - %s"%(title,names[i])) Job_utils.prepare_normal_job(utils,utils.get_jdl_file(),ces[i]) result=Job_utils.submit_normal_job(utils,ces[i]) if result[0] == 1 : utils.info(result[1]) fails=fails+1 else: utils.dbg("Clean job output directory") os.system("rm -rf %s"%(utils.get_job_output_dir())) return fails
def test5(utils, ssh, title): utils.show_progress(title) utils.info(title) try: Job_utils.prepare_normal_job(utils, utils.get_jdl_file()) utils.info("Submit a job to a CREAM CE") JOBID = Job_utils.submit_only_normal_job(utils)[1] prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2] utils.info("Wait until job finishes") utils.wait_until_job_finishes(JOBID) utils.job_status(JOBID) if utils.get_job_status().find("Done") != -1: utils.info("Purge a done job") # WARNING EXECUTE COMMAND AS root INSTEAD OF glite USER utils.info( "Execute '/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s' on remote host" % (prefix) ) output = SSH_utils.execute_remote_cmd( ssh, "/usr/sbin/glite-wms-purgeStorage.sh -p /var/SandboxDir/%s -s" % (prefix) ) utils.info("Check glite-wms-purgeStorage.sh output") if output.find("%s: removed DONE job" % (JOBID)) != -1: utils.info("glite-wms-purgeStorage.sh successfully remove the DONE job %s" % (JOBID)) else: utils.error("glite-wms-purgeStorage.sh didn't remove successfully the DONE job %s" % (JOBID)) raise GeneralError( "Check glite-wms-purgeStorage.sh", "glite-wms-purgeStorage.sh didn't remove successfully the DONE job %s" % (JOBID), ) utils.info("Check again job's status") utils.job_status(JOBID) if utils.get_job_status().find("Cleared") != -1: utils.info("Job's final status after purge is Cleared") utils.info("Check job's status reason") status = utils.get_job_status_reason(JOBID) if status.find("timed out, resource purge forced") != -1: utils.info("Status reason is 'timed out, resource purge forced' as expected") else: utils.error( "Status reason is %s while the expected is 'timed out, resource purge forced'" % (status) ) raise GeneralError( "Check job's status reason", "Status reason is %s while the expected is 'timed out, resource purge forced'" % (status), ) else: utils.error( "Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()) ) raise GeneralError( "Check job final status after purge", "Job's final status after purge is not Cleared , instead we get %s" % (utils.get_job_status()), ) else: utils.error("Job finishes with status: %s while expected 'Done'" % (utils.get_job_status())) raise GeneralError( "Check job final status", "Job finishes with status: %s while expected 'Done'" % (utils.get_job_status()), ) utils.info("TEST OK") except (RunCommandError, GeneralError, TimeOutError), e: utils.log_error("%s" % (utils.get_current_test())) utils.log_error("Command: %s" % (e.expression)) utils.log_error("Message: %s" % (e.message)) utils.log_traceback("%s" % (utils.get_current_test())) utils.log_traceback(traceback.format_exc()) return 1
def test1(utils, ssh, title): utils.show_progress(title) utils.info(title) try: Job_utils.prepare_normal_job(utils, utils.get_jdl_file(), "2119/jobmanager") utils.info("Submit a job to a LCG CE") JOBID = Job_utils.submit_only_normal_job(utils, "2119/jobmanager")[1] utils.info("Check the SBD and the others file used by the services") prefix = JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1][0:2] dir2 = "/var/jobcontrol/condorio/%s" % (prefix) utils.info("Check files at %s" % (dir2)) condorio = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir2)) for line in condorio.split("\n"): if line.find(JOBID.split("https://%s:9000/" % (utils.get_WMS()))[1]) != -1: target = "https_%s" % (line.lstrip().split("https_")[1]) dir1 = "/var/SandboxDir/%s/%s" % (prefix, target) utils.info("Check files at %s" % (dir1)) SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir1)) dir3 = "/var/jobcontrol/submit/%s/" % (prefix) utils.info("Check files at %s" % (dir3)) SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir3)) utils.info("Wait until job finishes") utils.wait_until_job_finishes(JOBID) utils.job_status(JOBID) if utils.get_job_status().find("Done") != -1: utils.info("Check on WMS auxiliary files should be removed") utils.info("Check directory %s" % (dir3)) output = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir3)) if output.find(target) != 1: utils.error("Auxiliary files are not removed from directory %s" % (dir3)) raise GeneralError( "Check for auxiliary files at %s" % (dir3), "Auxiliary files are not removed from directory %s" % (dir3), ) else: utils.info("Auxiliary files are removed as expected from directory %s" % (dir3)) utils.info("Check directory %s" % (dir2)) output = SSH_utils.execute_remote_cmd(ssh, "ls -l %s" % (dir2)) if output.find(target) != 1: utils.error("Auxiliary files are not removed from directory %s" % (dir2)) raise GeneralError( "Check for auxiliary files at %s" % (dir2), "Auxiliary files are not removed from directory %s" % (dir2), ) else: utils.info("Auxiliary files are removed as expected from directory %s" % (dir2)) utils.info("Retrieve job output") utils.run_command_continue_on_error( "glite-wms-job-output --nosubdir --noint --dir %s %s" % (utils.get_job_output_dir(), JOBID) ) utils.info("Check that also the SBD has been removed on WMS") utils.info("Check directory /var/SandboxDir/%s/" % (prefix)) output = SSH_utils.execute_remote_cmd(ssh, "ls -l /var/SandboxDir/%s/" % (prefix)) if output.find(target) != 1: utils.error("SBD not removed from directory /var/SandboxDir/%s/" % (prefix)) raise GeneralError( "Check for SBD at /var/SandboxDir/%s/" % (prefix), "Auxiliary files are not removed from directory %s" % (dir2), ) else: utils.info("SBD has been removed as expected from directory /var/SandboxDir/%s/" % (prefix)) else: utils.error("Job finishes with status: %s cannot retrieve output" % (utils.get_job_status())) raise GeneralError( "Check job final status", "Job finishes with status: %s cannot retrieve output" % (utils.get_job_status()), ) utils.info("TEST OK") except (RunCommandError, GeneralError, TimeOutError), e: utils.log_error("%s" % (utils.get_current_test())) utils.log_error("Command: %s" % (e.expression)) utils.log_error("Message: %s" % (e.message)) utils.log_traceback("%s" % (utils.get_current_test())) utils.log_traceback(traceback.format_exc()) return 1
def main(): utils = Test_utils.Test_utils(sys.argv[0],"Test a complete job cycle: from submission to get output") tests=["Set 1: Submit a normal job"] tests.append("Set 2: Submit a bulk of jobs") tests.append("Set 3: Submit a parametric job") tests.append("Set 4: Submit a DAG job") tests.append("Set 5: Submit a MPI job") tests.append("Set 6: Submit a perusal job") tests.append("Set 7: Submit a bulk of jobs using a single jdl with al the jdls of nodes") tests.append("Set 8: Testing forwarding parameters for parallel jobs") utils.prepare(sys.argv[1:],tests) utils.info("Test a complete job cycle: from submission to get output") signal.signal(signal.SIGINT,utils.exit_interrupt) fails=[] # This test is for direct submission to a given CE if utils.get_user_CE() !='' : utils.info ("Submit a normal job to %s"%(utils.get_user_CE())) Job_utils.prepare_normal_job(utils,utils.get_jdl_file(),utils.get_user_CE()) result=Job_utils.submit_normal_job(utils,utils.get_user_CE()) if result[0] == 1 : utils.warn(result[1]) utils.log_error("Submit a normal job to %s"%(utils.get_user_CE())) utils.log_error(result[1]) utils.exit_failure(result[1]) utils.message("Clean job output directory") os.system("rm -rf %s"%(utils.get_job_output_dir())) else: all_tests=utils.is_all_enabled() if all_tests==1 or utils.check_test_enabled(1)==1 : if normal_submit(utils, tests[0]): fails.append(tests[0]) if all_tests==1 or utils.check_test_enabled(2)==1 : if bulk_submit(utils, tests[1]): fails.append(tests[1]) if all_tests==1 or utils.check_test_enabled(3)==1 : if parametric_submit(utils, tests[2]): fails.append(tests[2]) if all_tests==1 or utils.check_test_enabled(4)==1 : if dag_submit(utils, tests[3]): fails.append(tests[3]) if all_tests==1 or utils.check_test_enabled(5)==1 : if parallel_submit(utils, tests[4]): fails.append(tests[4]) if all_tests==1 or utils.check_test_enabled(6)==1 : if perusal_submit(utils, tests[5]): fails.append(tests[5]) if all_tests==1 or utils.check_test_enabled(7)==1 : if bulk_submit_with_single_jdl(utils, tests[6]): fails.append(tests[6]) if all_tests==1 or utils.check_test_enabled(8)==1 : if forward_parameters_parallel_jobs(utils, tests[7]): fails.append(tests[7]) if len(fails) > 0 : utils.exit_failure("%s test(s) fail(s): %s"%(len(fails), fails)) else: utils.exit_success()