def test_notifications_for_cancelled_jobs():
print "Creating proxy ..."
cream_testing.create_proxy("sarabINFN","dteam")
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl("dteam","300", "/tmp")
print "Submitting job " + jdl_fname
cream_job_id = cream_testing.submit_job(jdl_fname, "cream-06.pd.infn.it:8443/cream-pbs-cert" )
print cream_job_id
time.sleep(60)
print "Verifying status of job " + cream_job_id
cream_regression.job_status_should_be_in(cream_job_id, ['PENDING', 'RUNNING', 'REALLY-RUNNING'])
print "Cancelling job " + cream_job_id
cream_testing.cancel_job(cream_job_id)
print "Sleeping 200sec"
time.sleep(200)
job_num = blah_testing.get_job_num_from_jid(cream_job_id)
#job_num = "985342878"
blah_parser_log_file_name = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(blah_parser_log_file_name, "/tmp")
#local_blah_parser_log_file = "/tmp/local_copy_of_a_cream_file"
time.sleep(200)
notifications_list = blah_testing.get_notifications_in_blah_parser_log(local_blah_parser_log_file, job_num)
print notifications_list
print blah_testing.check_notifications_for_cancelled(notifications_list)
def test_notifications_for_normally_finished_jobs():
# Create_proxy
cream_testing.create_proxy("sarabINFN","dteam")
# sleep_jdl(100)
jdl_fname = cream_testing.sleep_jdl("dteam","100", "/tmp")
#submit()
cream_job_id = cream_testing.submit_job(jdl_fname, "cream-06.pd.infn.it:8443/cream-pbs-cert" )
print cream_job_id
time.sleep(5)
#get_final_status()
print "Getting final job status ... "
final_job_status = cream_testing.get_final_status(cream_job_id)
print "Final job status = " + final_job_status
time.sleep(200)
job_num = blah_testing.get_job_num_from_jid(cream_job_id)
blah_parser_log_file_name = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(blah_parser_log_file_name, "/tmp")
notifications_list = blah_testing.get_notifications_in_blah_parser_log(local_blah_parser_log_file, job_num)
print notifications_list
print blah_testing.check_notifications_for_normally_finished(notifications_list)
def test_notifications_for_normally_finished_jobs():
# Create_proxy
cream_testing.create_proxy("sarabINFN", "dteam")
# sleep_jdl(100)
jdl_fname = cream_testing.sleep_jdl("dteam", "100", "/tmp")
#submit()
cream_job_id = cream_testing.submit_job(
jdl_fname, "cream-06.pd.infn.it:8443/cream-pbs-cert")
print cream_job_id
time.sleep(5)
#get_final_status()
print "Getting final job status ... "
final_job_status = cream_testing.get_final_status(cream_job_id)
print "Final job status = " + final_job_status
time.sleep(200)
job_num = blah_testing.get_job_num_from_jid(cream_job_id)
blah_parser_log_file_name = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(
blah_parser_log_file_name, "/tmp")
notifications_list = blah_testing.get_notifications_in_blah_parser_log(
local_blah_parser_log_file, job_num)
print notifications_list
print blah_testing.check_notifications_for_normally_finished(
notifications_list)
def test_notifications_for_cancelled_jobs():
print "Creating proxy ..."
cream_testing.create_proxy("sarabINFN","dteam")
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl("dteam","300", "/tmp")
print "Submitting job " + jdl_fname
cream_job_id = cream_testing.submit_job(jdl_fname, "cream-06.pd.infn.it:8443/cream-pbs-cert" )
print cream_job_id
time.sleep(60)
print "Verifying status of job " + cream_job_id
cream_regression.job_status_should_be_in(cream_job_id, ['PENDING', 'RUNNING', 'REALLY-RUNNING'])
print "Cancelling job " + cream_job_id
cream_testing.cancel_job(cream_job_id)
print "Sleeping 200sec"
time.sleep(200)
job_num = blah_testing.get_job_num_from_jid(cream_job_id)
blah_parser_log_file_name = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(blah_parser_log_file_name, "/tmp")
time.sleep(200)
notifications_list = blah_testing.get_notifications_in_blah_parser_log(local_blah_parser_log_file, job_num)
print notifications_list
print blah_testing.check_notifications_for_cancelled(notifications_list)
def check_for_bug_94414(jobs_list):
'''
| Description: | Given a dictionary of couples cream_job_id:job_status |
| | check if status is DONE-OK and if in blparser log file |
| | notifications for suspended and resumed jobs are present |
| Arguments: | jobs_statuses | dictionary of couples |
| | cream_job_id:job_status |
| Returns: | SUCCESS/FAILED |
| Exceptions: | |
'''
ret_val = ['CHECK SUCCESSFUL', 'CHECK FAILED']
final_ret_val = ret_val[0]
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
output_dir = my_conf.getParam('testsuite_behaviour', 'tmp_dir')
print "Getting final jobs status ... "
jobs_final_states, failed_jobs = cream_regression.get_n_job_status(
jobs_list, "DONE-OK", 200)
keys = jobs_final_states.keys()
for index in keys:
print "Job " + str(index) + " --- " + jobs_final_states[index]
if len(failed_jobs) == 0:
print "Statuses check successful"
final_ret_val = ret_val[0]
else:
print "Statuses check failed"
final_ret_val = ret_val[1]
# Check also blparser notifications because cream has a mechanism to detect job done independently by blparser
blparser_log = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(
blparser_log, output_dir)
for cream_job_id in keys:
batch_job_id = blah_testing.get_job_num_from_jid(cream_job_id)
notifications_list = blah_testing.get_notifications_in_blah_parser_log(
local_blah_parser_log_file, batch_job_id)
result = blah_testing.check_notifications_for_normally_finished(
notifications_list)
if result == 'NOTIFICATIONS OK':
print "Notifications check successful for job " + cream_job_id
else:
print "Notifications failure for job " + cream_job_id
final_ret_val = ret_val[1]
return final_ret_val
def check_for_bug_94414(jobs_list):
'''
| Description: | Given a dictionary of couples cream_job_id:job_status |
| | check if status is DONE-OK and if in blparser log file |
| | notifications for suspended and resumed jobs are present |
| Arguments: | jobs_statuses | dictionary of couples |
| | cream_job_id:job_status |
| Returns: | SUCCESS/FAILED |
| Exceptions: | |
'''
ret_val = ['CHECK SUCCESSFUL', 'CHECK FAILED']
final_ret_val = ret_val[0]
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
output_dir = my_conf.getParam('testsuite_behaviour','tmp_dir')
print "Getting final jobs status ... "
jobs_final_states, failed_jobs = cream_regression.get_n_job_status(jobs_list, "DONE-OK", 200)
keys = jobs_final_states.keys()
for index in keys:
print "Job " + str(index) + " --- " + jobs_final_states[index]
if len(failed_jobs) == 0:
print "Statuses check successful"
final_ret_val = ret_val[0]
else:
print "Statuses check failed"
final_ret_val = ret_val[1]
# Check also blparser notifications because cream has a mechanism to detect job done independently by blparser
blparser_log = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(blparser_log, output_dir)
for cream_job_id in keys:
batch_job_id = blah_testing.get_job_num_from_jid(cream_job_id)
notifications_list = blah_testing.get_notifications_in_blah_parser_log(local_blah_parser_log_file, batch_job_id)
result = blah_testing.check_notifications_for_normally_finished(notifications_list)
if result == 'NOTIFICATIONS OK':
print "Notifications check successful for job " + cream_job_id
else:
print "Notifications failure for job " + cream_job_id
final_ret_val = ret_val[1]
return final_ret_val
# kill the budater and bnotifier processes.
# Wait for 1 minute: you should see that the bupdater and bnotifier have been restarted.
import cream_regression, blah_regression, cream_testing
import re, time, sys
blparser_with_updater_and_notifier = cream_regression.get_yaim_param("BLPARSER_WITH_UPDATER_NOTIFIER")
print "blparser_with_updater_and_notifier = " + blparser_with_updater_and_notifier
if not blparser_with_updater_and_notifier:
print "Changing configuration parameter ... "
cream_regression.change_conf_param_in_file("/root/sarab_devel/siteinfo/services/glite-creamce", "BLPARSER_WITH_UPDATER_NOTIFIER", "true")
config_file = cream_regression.get_file_from_ce("/etc/blah.config", "/tmp")
is_initialized = cream_regression.check_parameter(config_file, "blah_check_children_interval")
if is_initialized != "INITIALIZED":
print "KO - parameter blah_check_children_interval not initialized in /etc/blah.config"
sys.exit()
print "Create proxy"
cream_testing.create_proxy("sarabINFN","dteam")
print "Clear the ce deleting al jobs"
cream_testing.cancel_all_jobs("cream-06.pd.infn.it:8443")
print "Re-configure the ce"
cream_regression.configure_ce_by_yaim("/root/sarab_devel/siteinfo/site-info.def")
print "Find bupdater and bnotifier pids in `cat /var/blah/blah_bupdater.pid` and `cat /var/blah/blah_bnotifier.pid`"
#!/usr/bin/python
import cream_regression, regression_vars
import datetime, os, shutil, re
print "Get yaim_conf_param CREAM_CONCURRENCY_LEVEL value"
cream_concurrency_level_old = cream_regression.get_cream_concurrency_level_from_yaim_conf()
print "CREAM_CONCURRENCY_LEVEL = " + cream_concurrency_level_old
#cream_concurrency_level_old = 50
Save a local copy of services
now = datetime.datetime.now()
suffix = now.strftime("%Y%m%d_%M%S")
os.mkdir("/tmp/tmp_" + suffix)
local_copy_of_services = cream_regression.get_file_from_ce("/root/siteinfo/services/glite-creamce", "/tmp/tmp_" + suffix)
print "Saving services file on a local copy _save"
cream_regression.exec_local_command("cp " + local_copy_of_services + " " + local_copy_of_services + "_save")
# Reconfigure the CE with a different value of CREAM_CONCURRENCY_LEVEL
print "Changing configuration parameter ... "
cream_concurrency_level_new = int(cream_concurrency_level_old) + 50
cream_regression.change_conf_param_in_file("/root/siteinfo/services/glite-creamce", "CREAM_CONCURRENCY_LEVEL", str(cream_concurrency_level_new))
print "Re-configure running yaim"
cream_regression.configure_ce_by_yaim("/root/siteinfo/site-info.def")
#cream_concurrency_level_new = 100
print "Check CREAM_CONCURRENCY_LEVEL in configuration file"
local_copy_config_file = cream_regression.get_file_from_ce("/etc/glite-ce-cream/cream-config.xml", "/tmp")
def test_notifications_for_suspended_resumed_jobs():
print "Creating proxy ..."
cream_testing.create_proxy("sarabINFN","dteam")
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl("dteam","300", "/tmp")
# With PBS and SGE it is possible to suspend only IDLE (i.e. not yet running) jobs.
# To perform the test, first saturate the batch system, second submit the job which
# willbe under test.
# At the end of the test cancel all submitted job to free the batch system
print "Saturating batch system"
job_ids_list = blah_testing.saturate_batch_system()
print "Submitting job " + jdl_fname
cream_job_id = cream_testing.submit_job(jdl_fname, "cream-06.pd.infn.it:8443/cream-pbs-cert" )
print cream_job_id
print "Sleeping 1min"
time.sleep(60)
print "Verifying status of job " + cream_job_id
cream_regression.job_status_should_be_in(cream_job_id, ['IDLE'])
print "Suspending job ..."
cream_testing.suspend_job(cream_job_id)
print "Sleeping 10sec"
time.sleep(10)
#check if job suspended on batch system. It is enough check status and verify it is 'HELD'
print "Verifying status of job " + cream_job_id + ". Should be in 'HELD'"
cream_regression.job_status_should_be_in(cream_job_id, ['HELD'])
#count = int(len(job_ids_list)/4)
#print "Sleep " + str(count) + "sec"
#time.sleep(count)
print "Cancel job saturating batch system"
blah_testing.cancel_list_of_jobs(job_ids_list)
print "Sleeping 5 min ..."
time.sleep(300)
print "Resuming job ..."
cream_testing.resume_job(cream_job_id)
print "Getting final job status ... "
final_job_status = cream_testing.get_final_status(cream_job_id)
print "Final job status = " + final_job_status
print "Sleeping 30sec"
time.sleep(30)
job_num = blah_testing.get_job_num_from_jid(cream_job_id)
blah_parser_log_file_name = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(blah_parser_log_file_name, "/tmp")
print "Sleeping 6min."
time.sleep(360)
print "Get notifications list in blah parser log file local copy"
notifications_list = blah_testing.get_notifications_in_blah_parser_log(local_blah_parser_log_file, job_num)
print notifications_list
print blah_testing.check_notifications_for_resumed(notifications_list)
print "Cancelling all jobs..."
cream_testing.cancel_all_jobs("cream-06.pd.infn.it:8443")
def test_notifications_for_suspended_resumed_jobs():
print "Creating proxy ..."
cream_testing.create_proxy("sarabINFN", "dteam")
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl("dteam", "300", "/tmp")
# With PBS and SGE it is possible to suspend only IDLE (i.e. not yet running) jobs.
# To perform the test, first saturate the batch system, second submit the job which
# willbe under test.
# At the end of the test cancel all submitted job to free the batch system
print "Saturating batch system"
job_ids_list = blah_testing.saturate_batch_system()
print "Submitting job " + jdl_fname
cream_job_id = cream_testing.submit_job(
jdl_fname, "cream-06.pd.infn.it:8443/cream-pbs-cert")
print cream_job_id
print "Sleeping 1min"
time.sleep(60)
print "Verifying status of job " + cream_job_id
cream_regression.job_status_should_be_in(cream_job_id, ['IDLE'])
print "Suspending job ..."
cream_testing.suspend_job(cream_job_id)
print "Sleeping 10sec"
time.sleep(10)
#check if job suspended on batch system. It is enough check status and verify it is 'HELD'
print "Verifying status of job " + cream_job_id + ". Should be in 'HELD'"
cream_regression.job_status_should_be_in(cream_job_id, ['HELD'])
#count = int(len(job_ids_list)/4)
#print "Sleep " + str(count) + "sec"
#time.sleep(count)
print "Cancel job saturating batch system"
blah_testing.cancel_list_of_jobs(job_ids_list)
print "Sleeping 5 min ..."
time.sleep(300)
print "Resuming job ..."
cream_testing.resume_job(cream_job_id)
print "Getting final job status ... "
final_job_status = cream_testing.get_final_status(cream_job_id)
print "Final job status = " + final_job_status
print "Sleeping 30sec"
time.sleep(30)
job_num = blah_testing.get_job_num_from_jid(cream_job_id)
blah_parser_log_file_name = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(
blah_parser_log_file_name, "/tmp")
print "Sleeping 6min."
time.sleep(360)
print "Get notifications list in blah parser log file local copy"
notifications_list = blah_testing.get_notifications_in_blah_parser_log(
local_blah_parser_log_file, job_num)
print notifications_list
print blah_testing.check_notifications_for_resumed(notifications_list)
print "Cancelling all jobs..."
cream_testing.cancel_all_jobs("cream-06.pd.infn.it:8443")
# Get yaim_conf_param CREAM_SANDBOX_PATH_old
sandbox_path_value_old = cream_regression.get_cream_sandbox_from_yaim_conf()
print "sandbox_path_value_old = " + sandbox_path_value_old
# Verify that the job is finished (it should be in a terminal state)
print "Getting final job status ... "
final_job_status = cream_testing.get_final_status(cream_job_id)
print "Final job status = " + final_job_status
cream_regression.exec_remote_command("service tomcat5 stop")
now = datetime.datetime.now()
suffix = now.strftime("%Y%m%d_%M%S")
os.mkdir("/tmp/tmp_" + suffix)
local_copy_of_services = cream_regression.get_file_from_ce(
"/root/sarab_devel/siteinfo/services/glite-creamce", "/tmp/tmp_" + suffix)
print "Saving services file on a local copy _save"
cream_regression.exec_local_command("cp " + local_copy_of_services + " " +
local_copy_of_services + "_save")
# Reconfigure the CE with a different value of CREAM_SANDBOX_PATH
print "Changing configuration parameter ... "
cream_regression.change_conf_param_in_file(
"/root/sarab_devel/siteinfo/services/glite-creamce", "CREAM_SANDBOX_PATH",
"/tmp/cream_sanbox_x")
print "Reconfiguring cream ce ... "
cream_regression.run_yaim_func("/root/sarab_devel/siteinfo/site-info.def",
"config_cream_ce")
# Get yaim_conf_param CREAM_SANDBOX_PATH_old
sandbox_path_value_old = cream_regression.get_cream_sandbox_from_yaim_conf()
print "sandbox_path_value_old = " + sandbox_path_value_old
# Verify that the job is finished (it should be in a terminal state)
print "Getting final job status ... "
final_job_status = cream_testing.get_final_status(cream_job_id)
print "Final job status = " + final_job_status
cream_regression.exec_remote_command("service tomcat5 stop")
now = datetime.datetime.now()
suffix = now.strftime("%Y%m%d_%M%S")
os.mkdir("/tmp/tmp_" + suffix)
local_copy_of_services = cream_regression.get_file_from_ce("/root/sarab_devel/siteinfo/services/glite-creamce", "/tmp/tmp_" + suffix)
print "Saving services file on a local copy _save"
cream_regression.exec_local_command("cp " + local_copy_of_services + " " + local_copy_of_services + "_save")
# Reconfigure the CE with a different value of CREAM_SANDBOX_PATH
print "Changing configuration parameter ... "
cream_regression.change_conf_param_in_file("/root/sarab_devel/siteinfo/services/glite-creamce", "CREAM_SANDBOX_PATH", "/tmp/cream_sanbox_x")
print "Reconfiguring cream ce ... "
cream_regression.run_yaim_func("/root/sarab_devel/siteinfo/site-info.def", "config_cream_ce")
# Get db username and password
print "Get db username and password from CE"
db_usr_name, db_usr_pass = cream_regression.get_cream_db_user_password_from_config()
import cream_testing, blah_testing, cream_regression
import time
local_copy_of_blah_config = cream_regression.get_file_from_ce(
"/etc/blah.config", "/tmp")
#ret_val = INITIALIZED, NOT_PRESENT, or NOT_INITIALIZED
ret_val = cream_regression.check_parameter(local_copy_of_blah_config,
"bupdater_use_bhist_for_killed")
print "bupdater_use_bhist_for_killed = " + ret_val + " Should be INITIALIZED"
print "Creating proxy ..."
cream_testing.create_proxy("sarabINFN", "dteam")
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl("dteam", "300", "/tmp")
print "Submitting job " + jdl_fname
cream_job_id = cream_testing.submit_job(
jdl_fname, "cream-48.pd.infn.it:8443/cream-lsf-cert")
print cream_job_id
time.sleep(60)
print "Verifying status of job " + cream_job_id
cream_regression.job_status_should_be_in(
cream_job_id, ['PENDING', 'RUNNING', 'REALLY-RUNNING'])
print "Cancelling job " + cream_job_id
cream_testing.cancel_job(cream_job_id)
print "Sleeping 200sec"
import paramiko, os, datetime
import testsuite_utils, cream_regression
ce_host = "cream-48.pd.infn.it"
admin_name = 'root'
tester_home = os.environ['HOME']
my_utils = testsuite_utils.Utils()
cmd_mng = testsuite_utils.CommandMng()
print "++++++++ Get local copy of file and save it"
now = datetime.datetime.now()
suffix = now.strftime("%Y%m%d_%M%S")
os.mkdir("/tmp/tmp_" + suffix)
local_copy_of_ComputingShare_ldif = cream_regression.get_file_from_ce("/var/lib/bdii/gip/ldif/ComputingShare.ldif", "/tmp/tmp_" + suffix)
print "++++++++ Saving services file on a local copy _save"
cream_regression.exec_local_command("cp " + local_copy_of_ComputingShare_ldif + " " + local_copy_of_ComputingShare_ldif + "_save")
print "++++++++ Modify the file"
with open(local_copy_of_ComputingShare_ldif, "a") as myfile:
myfile.write("GLUE2PolicyRule:\n")
myfile.close()
print "++++++++ Put modified file on ce"
my_utils.put_file_on_ce(local_copy_of_ComputingShare_ldif, "/var/lib/bdii/gip/ldif/ComputingShare.ldif")
print "++++++++ Exec the test"
cmd = "/var/lib/bdii/gip/plugin/glite-info-dynamic-scheduler-wrapper"
import cream_testing, blah_testing, cream_regression
import time
local_copy_of_blah_config = cream_regression.get_file_from_ce("/etc/blah.config", "/tmp")
#ret_val = INITIALIZED, NOT_PRESENT, or NOT_INITIALIZED
ret_val = cream_regression.check_parameter(local_copy_of_blah_config, "bupdater_use_bhist_for_killed")
print "bupdater_use_bhist_for_killed = " + ret_val + " Should be INITIALIZED"
print "Creating proxy ..."
cream_testing.create_proxy("sarabINFN","dteam")
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl("dteam","300", "/tmp")
print "Submitting job " + jdl_fname
cream_job_id = cream_testing.submit_job(jdl_fname, "cream-48.pd.infn.it:8443/cream-lsf-cert" )
print cream_job_id
time.sleep(60)
print "Verifying status of job " + cream_job_id
cream_regression.job_status_should_be_in(cream_job_id, ['PENDING', 'RUNNING', 'REALLY-RUNNING'])
print "Cancelling job " + cream_job_id
cream_testing.cancel_job(cream_job_id)
print "Sleeping 200sec"
time.sleep(200)
job_num = blah_testing.get_job_num_from_jid(cream_job_id)
import paramiko, os, datetime
import testsuite_utils, cream_regression
ce_host = "cream-48.pd.infn.it"
admin_name = 'root'
tester_home = os.environ['HOME']
my_utils = testsuite_utils.Utils()
cmd_mng = testsuite_utils.CommandMng()
print "++++++++ Get local copy of file and save it"
now = datetime.datetime.now()
suffix = now.strftime("%Y%m%d_%M%S")
os.mkdir("/tmp/tmp_" + suffix)
local_copy_of_ComputingShare_ldif = cream_regression.get_file_from_ce(
"/var/lib/bdii/gip/ldif/ComputingShare.ldif", "/tmp/tmp_" + suffix)
print "++++++++ Saving services file on a local copy _save"
cream_regression.exec_local_command("cp " + local_copy_of_ComputingShare_ldif +
" " + local_copy_of_ComputingShare_ldif +
"_save")
print "++++++++ Modify the file"
with open(local_copy_of_ComputingShare_ldif, "a") as myfile:
myfile.write("GLUE2PolicyRule:\n")
myfile.close()
print "++++++++ Put modified file on ce"
my_utils.put_file_on_ce(local_copy_of_ComputingShare_ldif,
"/var/lib/bdii/gip/ldif/ComputingShare.ldif")
