def __init__(self):
'''
| Description: | Constructor. |
| Arguments: | None. |
| Returns: | |
|Exceptions: | |
'''
self.my_log = logging.getLogger('CreamConfigLayoutMng')
self.my_log.info("Creating CreamConfigLayoutMng object")
self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
self.ce_host = self.my_conf.getParam('submission_info', 'ce_host')
self.admin_name = self.my_conf.getParam('ce_specific',
'cream_root_usr')
self.admin_pass = self.my_conf.getParam('ce_specific',
'cream_root_pass')
#self.output_dir = self.my_conf.getParam('testsuite_behaviour','tmp_dir')
self.output_dir = regression_vars.tmp_dir # The output dir must be always the same in all the testsuite,
# to allow cleaning procedure
self.my_utils = testsuite_utils.Utils()
if len(self.ce_host) == 0:
raise cream_testsuite_exception.TestsuiteError(
"Mandatory parameter ce_host is empty. Check testsuite configuration"
)
if len(self.admin_name) == 0:
raise cream_testsuite_exception.TestsuiteError(
"Mandatory parameter cream_root_usr is empty. Check testsuite configuration"
)
if len(self.admin_pass) == 0:
raise cream_testsuite_exception.TestsuiteError(
"Mandatory parameter cream_root_pass is empty. Check testsuite configuration"
)
def submit_n_jobs(jobs_num, jdl_fname):
'''
| Description: | Send jobs_num jobs |
| Arguments: | jobs_num | number of jobs to send |
| | jdl_fname | jdl file to submit |
| Returns: | the job_ids list pf submitted jobs |
| Exceptions: | |
'''
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
ce_endpoint = my_conf.getParam('submission_info', 'ce_endpoint')
cream_queue = my_conf.getParam('submission_info', 'cream_queue')
if len(ce_endpoint) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter ce_endpoint is empty. Check testsuite configuration"
)
if len(cream_queue) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter cream_queue is empty. Check testsuite configuration"
)
ce = ce_endpoint + "/" + cream_queue
print "send " + str(jobs_num) + " jobs"
cream_job_ids = list()
for i in range(int(jobs_num)):
cream_job_id = cream_testing.submit_job(jdl_fname, ce)
cream_job_ids.append(cream_job_id)
return cream_job_ids
def __init__(self):
self.my_log = logging.getLogger('SLURMBatchSys')
self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
self.slurm_host = self.my_conf.getParam('batch_system',
'batch_master_host')
print "slurm_host " + self.slurm_host
def __init__(self):
self.my_log = logging.getLogger('Utils')
Utils.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
Utils.my_ce_host = Utils.my_conf.getParam('submission_info','ce_host')
Utils.my_admin_name = Utils.my_conf.getParam('ce_specific','cream_root_usr')
if len(Utils.my_ce_host) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter ce_host is empty. Check testsuite configuration")
if len(Utils.my_admin_name) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter cream_root_usr is empty. Check testsuite configuration")
def __init__(self):
self.my_log = logging.getLogger('CommandMng')
CommandMng.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
CommandMng.my_ce_host = CommandMng.my_conf.getParam('submission_info','ce_host')
CommandMng.my_admin_name = CommandMng.my_conf.getParam('ce_specific','cream_root_usr')
CommandMng.my_my_tmpDir = CommandMng.my_conf.getParam('testsuite_behaviour','tmp_dir')
if len(CommandMng.my_ce_host) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter ce_host is empty. Check testsuite configuration")
if len(CommandMng.my_admin_name) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter cream_root_usr is empty. Check testsuite configuration")
def __init__(self, msg_queue, job_num, jdl_fname):
threading.Thread.__init__(self)
self._stop = threading.Event()
self.working = None
self.msg_queue = msg_queue
self.cream_job_ids = list()
self.job_num = job_num
self.jdl_fname = jdl_fname
try:
self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
except Exception:
self.msg_queue.put(sys.exc_info())
def saturate_batch_system(jdl_file_name='empty'):
'''
| Description: | Reads from test suite configuration file the value of total CPU number |
| | present in the batch cluster and submits a number of jobs equal to |
| | the total CPU number, to saturete the batch system reading submission |
| | parameters from configuration file. |
| Arguments: | None |
| Returns: | The list of cream job ids of submitted jobs |
| Exceprtions: | TestsuiteError | if an error is present in parameters read from config |
'''
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
tot_cpu_in_batch_cluster = my_conf.getParam('batch_system', 'tot_cpu_num')
vo = my_conf.getParam('submission_info', 'vo')
proxy_pass = my_conf.getParam('submission_info', 'proxy_pass')
ce_endpoint = my_conf.getParam('submission_info', 'ce_endpoint')
cream_queue = my_conf.getParam('submission_info', 'cream_queue')
#output_dir = my_conf.getParam('testsuite_behaviour','tmp_dir')
output_dir = regression_vars.tmp_dir
ce = ce_endpoint + "/" + cream_queue
if len(tot_cpu_in_batch_cluster) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter tot_cpu_num is empty. Check testsuite configuration"
)
if len(vo) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter vo is empty. Check testsuite configuration")
if len(output_dir) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter tmp_dir is empty. Check testsuite configuration"
)
print "Creating proxy ..."
cream_testing.create_proxy(proxy_pass, vo)
jdl_fname = ""
if jdl_file_name == 'empty':
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl(vo, "300", output_dir)
else:
jdl_fname = jdl_file_name
print "Submitting " + tot_cpu_in_batch_cluster + " jobs ..."
cream_job_ids = list()
cream_job_ids = submit_n_jobs(tot_cpu_in_batch_cluster, jdl_fname)
print cream_job_ids
return cream_job_ids
def check_for_bug_94414(jobs_list):
'''
| Description: | Given a dictionary of couples cream_job_id:job_status |
| | check if status is DONE-OK and if in blparser log file |
| | notifications for suspended and resumed jobs are present |
| Arguments: | jobs_statuses | dictionary of couples |
| | cream_job_id:job_status |
| Returns: | SUCCESS/FAILED |
| Exceptions: | |
'''
ret_val = ['CHECK SUCCESSFUL', 'CHECK FAILED']
final_ret_val = ret_val[0]
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
output_dir = my_conf.getParam('testsuite_behaviour', 'tmp_dir')
print "Getting final jobs status ... "
jobs_final_states, failed_jobs = cream_regression.get_n_job_status(
jobs_list, "DONE-OK", 200)
keys = jobs_final_states.keys()
for index in keys:
print "Job " + str(index) + " --- " + jobs_final_states[index]
if len(failed_jobs) == 0:
print "Statuses check successful"
final_ret_val = ret_val[0]
else:
print "Statuses check failed"
final_ret_val = ret_val[1]
# Check also blparser notifications because cream has a mechanism to detect job done independently by blparser
blparser_log = blah_testing.get_blah_parser_log_file_name()
local_blah_parser_log_file = cream_regression.get_file_from_ce(
blparser_log, output_dir)
for cream_job_id in keys:
batch_job_id = blah_testing.get_job_num_from_jid(cream_job_id)
notifications_list = blah_testing.get_notifications_in_blah_parser_log(
local_blah_parser_log_file, batch_job_id)
result = blah_testing.check_notifications_for_normally_finished(
notifications_list)
if result == 'NOTIFICATIONS OK':
print "Notifications check successful for job " + cream_job_id
else:
print "Notifications failure for job " + cream_job_id
final_ret_val = ret_val[1]
return final_ret_val
def __init__(self):
self.my_log = logging.getLogger('LSFBatchSys')
self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
lsf_host = self.my_conf.getParam('batch_system', 'batch_master_host')
lsf_admin = self.my_conf.getParam('batch_system', 'batch_master_admin')
lsf_pass = self.my_conf.getParam('batch_system',
'batch_master_admin_password')
print "Get ssh connection with cream ce to manage batch system commands"
self.ssh_connection.set_missing_host_key_policy(
paramiko.AutoAddPolicy()
) #don't ask for acceptance of foreign host key (auto accept)
self.ssh_connection.connect(lsf_host,
username=lsf_admin,
password=lsf_pass)
def get_blah_parser_log_file_name():
'''
| Description: | Get blah log parser file name from blah testsuite configuration |
| Arguments: | None |
| Returns: | blah_parser_log_file_name (complete path) |
| Exceptions: | |
'''
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
blparser_log = my_conf.getParam('blah_specific', 'parser_log_file')
if len(blparser_log) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter parser_log_file is empty. Check testsuite configuration"
)
print "Blah parser log file name " + blparser_log
return blparser_log
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import cream_testsuite_conf
import cream_testing
import os
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
# MIDDLEWARE
# gLite middleware version to test (can be EMI1 or EMI2)
middleware_version = my_conf.getParam('middleware', 'middleware_version')
my_conf.checkIfParamIsNull('middleware_version', middleware_version)
if (middleware_version == 'EMI1') or (middleware_version == 'EMI2'):
pass
else:
raise _error(
'Middleware version NOT correctly set. Admitted values = EMI1 or EMI2')
# BATCH_SYSTEM
#The underlying batch system of the CREAM endpoint.Either pbs or lsf.
batch_system = my_conf.getParam('batch_system', 'batch_sys')
print " batch_system = " + batch_system