def __init__(self): ''' | Description: | Constructor. | | Arguments: | None. | | Returns: | | |Exceptions: | | ''' self.my_log = logging.getLogger('CreamConfigLayoutMng') self.my_log.info("Creating CreamConfigLayoutMng object") self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() self.ce_host = self.my_conf.getParam('submission_info', 'ce_host') self.admin_name = self.my_conf.getParam('ce_specific', 'cream_root_usr') self.admin_pass = self.my_conf.getParam('ce_specific', 'cream_root_pass') #self.output_dir = self.my_conf.getParam('testsuite_behaviour','tmp_dir') self.output_dir = regression_vars.tmp_dir # The output dir must be always the same in all the testsuite, # to allow cleaning procedure self.my_utils = testsuite_utils.Utils() if len(self.ce_host) == 0: raise cream_testsuite_exception.TestsuiteError( "Mandatory parameter ce_host is empty. Check testsuite configuration" ) if len(self.admin_name) == 0: raise cream_testsuite_exception.TestsuiteError( "Mandatory parameter cream_root_usr is empty. Check testsuite configuration" ) if len(self.admin_pass) == 0: raise cream_testsuite_exception.TestsuiteError( "Mandatory parameter cream_root_pass is empty. Check testsuite configuration" )
def submit_n_jobs(jobs_num, jdl_fname): ''' | Description: | Send jobs_num jobs | | Arguments: | jobs_num | number of jobs to send | | | jdl_fname | jdl file to submit | | Returns: | the job_ids list pf submitted jobs | | Exceptions: | | ''' my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() ce_endpoint = my_conf.getParam('submission_info', 'ce_endpoint') cream_queue = my_conf.getParam('submission_info', 'cream_queue') if len(ce_endpoint) == 0: raise testsuite_exception.TestsuiteError( "Mandatory parameter ce_endpoint is empty. Check testsuite configuration" ) if len(cream_queue) == 0: raise testsuite_exception.TestsuiteError( "Mandatory parameter cream_queue is empty. Check testsuite configuration" ) ce = ce_endpoint + "/" + cream_queue print "send " + str(jobs_num) + " jobs" cream_job_ids = list() for i in range(int(jobs_num)): cream_job_id = cream_testing.submit_job(jdl_fname, ce) cream_job_ids.append(cream_job_id) return cream_job_ids
def __init__(self): self.my_log = logging.getLogger('SLURMBatchSys') self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() self.slurm_host = self.my_conf.getParam('batch_system', 'batch_master_host') print "slurm_host " + self.slurm_host
def __init__(self): self.my_log = logging.getLogger('Utils') Utils.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() Utils.my_ce_host = Utils.my_conf.getParam('submission_info','ce_host') Utils.my_admin_name = Utils.my_conf.getParam('ce_specific','cream_root_usr') if len(Utils.my_ce_host) == 0: raise testsuite_exception.TestsuiteError("Mandatory parameter ce_host is empty. Check testsuite configuration") if len(Utils.my_admin_name) == 0: raise testsuite_exception.TestsuiteError("Mandatory parameter cream_root_usr is empty. Check testsuite configuration")
def __init__(self): self.my_log = logging.getLogger('CommandMng') CommandMng.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() CommandMng.my_ce_host = CommandMng.my_conf.getParam('submission_info','ce_host') CommandMng.my_admin_name = CommandMng.my_conf.getParam('ce_specific','cream_root_usr') CommandMng.my_my_tmpDir = CommandMng.my_conf.getParam('testsuite_behaviour','tmp_dir') if len(CommandMng.my_ce_host) == 0: raise testsuite_exception.TestsuiteError("Mandatory parameter ce_host is empty. Check testsuite configuration") if len(CommandMng.my_admin_name) == 0: raise testsuite_exception.TestsuiteError("Mandatory parameter cream_root_usr is empty. Check testsuite configuration")
def __init__(self, msg_queue, job_num, jdl_fname): threading.Thread.__init__(self) self._stop = threading.Event() self.working = None self.msg_queue = msg_queue self.cream_job_ids = list() self.job_num = job_num self.jdl_fname = jdl_fname try: self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() except Exception: self.msg_queue.put(sys.exc_info())
def saturate_batch_system(jdl_file_name='empty'): ''' | Description: | Reads from test suite configuration file the value of total CPU number | | | present in the batch cluster and submits a number of jobs equal to | | | the total CPU number, to saturete the batch system reading submission | | | parameters from configuration file. | | Arguments: | None | | Returns: | The list of cream job ids of submitted jobs | | Exceprtions: | TestsuiteError | if an error is present in parameters read from config | ''' my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() tot_cpu_in_batch_cluster = my_conf.getParam('batch_system', 'tot_cpu_num') vo = my_conf.getParam('submission_info', 'vo') proxy_pass = my_conf.getParam('submission_info', 'proxy_pass') ce_endpoint = my_conf.getParam('submission_info', 'ce_endpoint') cream_queue = my_conf.getParam('submission_info', 'cream_queue') #output_dir = my_conf.getParam('testsuite_behaviour','tmp_dir') output_dir = regression_vars.tmp_dir ce = ce_endpoint + "/" + cream_queue if len(tot_cpu_in_batch_cluster) == 0: raise testsuite_exception.TestsuiteError( "Mandatory parameter tot_cpu_num is empty. Check testsuite configuration" ) if len(vo) == 0: raise testsuite_exception.TestsuiteError( "Mandatory parameter vo is empty. Check testsuite configuration") if len(output_dir) == 0: raise testsuite_exception.TestsuiteError( "Mandatory parameter tmp_dir is empty. Check testsuite configuration" ) print "Creating proxy ..." cream_testing.create_proxy(proxy_pass, vo) jdl_fname = "" if jdl_file_name == 'empty': print "Creating jdl" jdl_fname = cream_testing.sleep_jdl(vo, "300", output_dir) else: jdl_fname = jdl_file_name print "Submitting " + tot_cpu_in_batch_cluster + " jobs ..." cream_job_ids = list() cream_job_ids = submit_n_jobs(tot_cpu_in_batch_cluster, jdl_fname) print cream_job_ids return cream_job_ids
def check_for_bug_94414(jobs_list): ''' | Description: | Given a dictionary of couples cream_job_id:job_status | | | check if status is DONE-OK and if in blparser log file | | | notifications for suspended and resumed jobs are present | | Arguments: | jobs_statuses | dictionary of couples | | | cream_job_id:job_status | | Returns: | SUCCESS/FAILED | | Exceptions: | | ''' ret_val = ['CHECK SUCCESSFUL', 'CHECK FAILED'] final_ret_val = ret_val[0] my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() output_dir = my_conf.getParam('testsuite_behaviour', 'tmp_dir') print "Getting final jobs status ... " jobs_final_states, failed_jobs = cream_regression.get_n_job_status( jobs_list, "DONE-OK", 200) keys = jobs_final_states.keys() for index in keys: print "Job " + str(index) + " --- " + jobs_final_states[index] if len(failed_jobs) == 0: print "Statuses check successful" final_ret_val = ret_val[0] else: print "Statuses check failed" final_ret_val = ret_val[1] # Check also blparser notifications because cream has a mechanism to detect job done independently by blparser blparser_log = blah_testing.get_blah_parser_log_file_name() local_blah_parser_log_file = cream_regression.get_file_from_ce( blparser_log, output_dir) for cream_job_id in keys: batch_job_id = blah_testing.get_job_num_from_jid(cream_job_id) notifications_list = blah_testing.get_notifications_in_blah_parser_log( local_blah_parser_log_file, batch_job_id) result = blah_testing.check_notifications_for_normally_finished( notifications_list) if result == 'NOTIFICATIONS OK': print "Notifications check successful for job " + cream_job_id else: print "Notifications failure for job " + cream_job_id final_ret_val = ret_val[1] return final_ret_val
def __init__(self): self.my_log = logging.getLogger('LSFBatchSys') self.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() lsf_host = self.my_conf.getParam('batch_system', 'batch_master_host') lsf_admin = self.my_conf.getParam('batch_system', 'batch_master_admin') lsf_pass = self.my_conf.getParam('batch_system', 'batch_master_admin_password') print "Get ssh connection with cream ce to manage batch system commands" self.ssh_connection.set_missing_host_key_policy( paramiko.AutoAddPolicy() ) #don't ask for acceptance of foreign host key (auto accept) self.ssh_connection.connect(lsf_host, username=lsf_admin, password=lsf_pass)
def get_blah_parser_log_file_name(): ''' | Description: | Get blah log parser file name from blah testsuite configuration | | Arguments: | None | | Returns: | blah_parser_log_file_name (complete path) | | Exceptions: | | ''' my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() blparser_log = my_conf.getParam('blah_specific', 'parser_log_file') if len(blparser_log) == 0: raise testsuite_exception.TestsuiteError( "Mandatory parameter parser_log_file is empty. Check testsuite configuration" ) print "Blah parser log file name " + blparser_log return blparser_log
# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import cream_testsuite_conf import cream_testing import os my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton() # MIDDLEWARE # gLite middleware version to test (can be EMI1 or EMI2) middleware_version = my_conf.getParam('middleware', 'middleware_version') my_conf.checkIfParamIsNull('middleware_version', middleware_version) if (middleware_version == 'EMI1') or (middleware_version == 'EMI2'): pass else: raise _error( 'Middleware version NOT correctly set. Admitted values = EMI1 or EMI2') # BATCH_SYSTEM #The underlying batch system of the CREAM endpoint.Either pbs or lsf. batch_system = my_conf.getParam('batch_system', 'batch_sys') print " batch_system = " + batch_system