def open_local_copy_of_ce_file(self, remote_file_name, out_dir):
'''
| Description: | Get a local copy of remote_file_name, |
| | returns reference to the local file opened |
| Arguments: | remote_file_name | full path name |
| Returns: | reference to local file opened |
| Exceptions: | |
'''
print "Remote file to be opened : " + remote_file_name
local_file = self.get_file_from_ce(remote_file_name, out_dir)
print "Local file to be opened : " + local_file
if len(local_file) == 0:
print "File " + local_file + " NOT FOUND on " + self.ce_host
raise testsuite_exception.TestsuiteError("File " + local_file + " NOT FOUND on " + self.ce_host)
print "File name = " + local_file
self.my_log.debug("File name = " + local_file)
try:
in_file = open(local_file,"r")
except Exception as exc:
self.my_log.error("Error opening file " + local_file)
print "Error opening file " + local_file
self.my_log.error(exc)
raise testsuite_exception.TestsuiteError("Error opening file " + local_file)
return in_file
def submit_n_jobs(jobs_num, jdl_fname):
'''
| Description: | Send jobs_num jobs |
| Arguments: | jobs_num | number of jobs to send |
| | jdl_fname | jdl file to submit |
| Returns: | the job_ids list pf submitted jobs |
| Exceptions: | |
'''
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
ce_endpoint = my_conf.getParam('submission_info', 'ce_endpoint')
cream_queue = my_conf.getParam('submission_info', 'cream_queue')
if len(ce_endpoint) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter ce_endpoint is empty. Check testsuite configuration"
)
if len(cream_queue) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter cream_queue is empty. Check testsuite configuration"
)
ce = ce_endpoint + "/" + cream_queue
print "send " + str(jobs_num) + " jobs"
cream_job_ids = list()
for i in range(int(jobs_num)):
cream_job_id = cream_testing.submit_job(jdl_fname, ce)
cream_job_ids.append(cream_job_id)
return cream_job_ids
def __init__(self):
self.my_log = logging.getLogger('Utils')
Utils.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
Utils.my_ce_host = Utils.my_conf.getParam('submission_info','ce_host')
Utils.my_admin_name = Utils.my_conf.getParam('ce_specific','cream_root_usr')
if len(Utils.my_ce_host) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter ce_host is empty. Check testsuite configuration")
if len(Utils.my_admin_name) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter cream_root_usr is empty. Check testsuite configuration")
def __init__(self):
self.my_log = logging.getLogger('CommandMng')
CommandMng.my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
CommandMng.my_ce_host = CommandMng.my_conf.getParam('submission_info','ce_host')
CommandMng.my_admin_name = CommandMng.my_conf.getParam('ce_specific','cream_root_usr')
CommandMng.my_my_tmpDir = CommandMng.my_conf.getParam('testsuite_behaviour','tmp_dir')
if len(CommandMng.my_ce_host) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter ce_host is empty. Check testsuite configuration")
if len(CommandMng.my_admin_name) == 0:
raise testsuite_exception.TestsuiteError("Mandatory parameter cream_root_usr is empty. Check testsuite configuration")
def saturate_batch_system(jdl_file_name='empty'):
'''
| Description: | Reads from test suite configuration file the value of total CPU number |
| | present in the batch cluster and submits a number of jobs equal to |
| | the total CPU number, to saturete the batch system reading submission |
| | parameters from configuration file. |
| Arguments: | None |
| Returns: | The list of cream job ids of submitted jobs |
| Exceprtions: | TestsuiteError | if an error is present in parameters read from config |
'''
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
tot_cpu_in_batch_cluster = my_conf.getParam('batch_system', 'tot_cpu_num')
vo = my_conf.getParam('submission_info', 'vo')
proxy_pass = my_conf.getParam('submission_info', 'proxy_pass')
ce_endpoint = my_conf.getParam('submission_info', 'ce_endpoint')
cream_queue = my_conf.getParam('submission_info', 'cream_queue')
#output_dir = my_conf.getParam('testsuite_behaviour','tmp_dir')
output_dir = regression_vars.tmp_dir
ce = ce_endpoint + "/" + cream_queue
if len(tot_cpu_in_batch_cluster) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter tot_cpu_num is empty. Check testsuite configuration"
)
if len(vo) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter vo is empty. Check testsuite configuration")
if len(output_dir) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter tmp_dir is empty. Check testsuite configuration"
)
print "Creating proxy ..."
cream_testing.create_proxy(proxy_pass, vo)
jdl_fname = ""
if jdl_file_name == 'empty':
print "Creating jdl"
jdl_fname = cream_testing.sleep_jdl(vo, "300", output_dir)
else:
jdl_fname = jdl_file_name
print "Submitting " + tot_cpu_in_batch_cluster + " jobs ..."
cream_job_ids = list()
cream_job_ids = submit_n_jobs(tot_cpu_in_batch_cluster, jdl_fname)
print cream_job_ids
return cream_job_ids
def exec_remote_command(self, command):
'''
| Description: | Executes a generic unix command on the cream ce under test |
| Arguments: | command to execute |
| Returns: | command output and error as strings |
'''
ce_host = CommandMng.my_ce_host
admin_name = CommandMng.my_admin_name
print "Check point A"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#In case the server's key is unknown,
#we will be adding it automatically to the list of known hosts
client.load_host_keys(os.path.expanduser(os.path.join(CommandMng.tester_home, ".ssh", "known_hosts")))
print "Check point B"
client.connect(ce_host, username=admin_name, key_filename=CommandMng.tester_home+"/.ssh/id_rsa")
print "Check point C"
print "running '%s'" % command
ssh_stdin, ssh_stdout, ssh_stderr = client.exec_command(command)
exit_status = ssh_stdout.channel.recv_exit_status()
print "Command %s on ce %s exit status: %s" % (command, ce_host, exit_status)
if exit_status != 0 :
client.close()
raise testsuite_exception.TestsuiteError("Error on command %s execution on %s" % (command, ce_host))
out = ""
err = ""
if ssh_stdout is not None:
out = []
for i in ssh_stdout.readlines():
out.append(i)
else:
out = ['']
print "output of " + command + " is empty"
if ssh_stderr is not None:
err = []
for i in ssh_stderr:
err.append(i)
else:
err =['']
print "error of " + command + " is empty"
print "output of " + command
print out
print "error of " + command
print err
return "".join(out), "".join(err)
def get_blah_parser_log_file_name():
'''
| Description: | Get blah log parser file name from blah testsuite configuration |
| Arguments: | None |
| Returns: | blah_parser_log_file_name (complete path) |
| Exceptions: | |
'''
my_conf = cream_testsuite_conf.CreamTestsuiteConfSingleton()
blparser_log = my_conf.getParam('blah_specific', 'parser_log_file')
if len(blparser_log) == 0:
raise testsuite_exception.TestsuiteError(
"Mandatory parameter parser_log_file is empty. Check testsuite configuration"
)
print "Blah parser log file name " + blparser_log
return blparser_log
def exec_command(self, command):
'''
| Description: | Executes a generic shell command opening a subprocess. |
| Arguments: | command, a generic shell command. |
| Returns: | command output and error as strings. |
| Exceptions: | throws an exception if something goes bad. |
'''
self.my_log.info("Executing command : " + command)
print "Executing command : " + command
args = shlex.split(command.encode('ascii'))
p = subprocess.Popen( args , shell=False , stderr=subprocess.PIPE , stdout=subprocess.PIPE )
retVal = p.wait()
if p.stdout is not None:
out = p.stdout
#my_output = out.readlines()
my_output = out.read()
self.my_log.debug("".join(my_output))
else:
my_output = ""
if p.stderr is not None:
err = p.stderr
my_error = err.read()
self.my_log.debug("".join(my_error))
else:
my_error = ""
print "exec local command output: \n" + my_output
print "exec local command error \n" + my_error
if retVal != 0 and len(my_output) != 0:
if "error" in ','.join(my_output).lower() or "fatal" in ','.join(my_output).lower() or "fault" in ','.join(my_output).lower() or retVal != 0 :
raise testsuite_exception.TestsuiteError("Command %s execution failed with return value: %s\nCommand reported: %s" % (command, str(p.returncode), ','.join(my_output)))
return my_output, my_error
def check_param_in_conf_file(self, conf_f, param_to_check):
''' | Description: | This function searches if the parameter param_to_check is present |
| | in the provided configuration file conf_f based on the ipothesys |
| | that the configuration file is written as set of pairs |
| | param_name = param_value. It jumps rows containing '#' character |
| | before param_name, supposing they are comments. |
| Arguments: | conf_f: file where search the parameter. |
| | param_to_check: name of the parameter to search |
| Returns: | ret_val: with possible values INITIALIZED, NOT_PRESENT, |
| | or NOT_INITIALIZED |
| | parameter_value may contain the param_to_check value or an empty |
| | string |
| Exceptions: | |
'''
ret_val = ['INITIALIZED', 'NOT_PRESENT', 'NOT_INITIALIZED']
param_row = ""
param_value = ""
self.my_log.debug("Configuration file name = " + conf_f)
try:
in_file = open(conf_f,"r")
except Exception as exc:
print "Error opening file " + conf_f
self.my_log.error("Error opening file " + conf_f)
self.my_log.error(exc)
raise testsuite_exception.TestsuiteError("Error opening file " + conf_f)
in_line = in_file.readline()
while in_line:
in_line = in_line.strip()
if re.search('^#', in_line):
in_line = in_file.readline()
continue
else:
pass
if re.search(param_to_check, in_line):
param_row = in_line
first_part, separator, second_part = param_row.partition("#")
self.my_log.debug("first part = " + first_part)
self.my_log.debug("second part = " + second_part)
self.my_log.debug("separator = " + separator)
if len(first_part) == 0:
continue
else:
param_row = first_part
break
in_line = in_file.readline()
in_file.close()
str_to_match = '(?<=' + param_to_check + '=).*'
m = re.search(str_to_match, param_row)
if param_row != "":
if m == None:
print "NOT INITIALIZED 1"
return ret_val[2], param_value.strip() # NOT INITIALIZED
else:
print "FOUND"
m = m.group(0)
self.my_log.debug("Parameter value = __" + m.strip() + "__")
print "Parameter value = __" + m.strip() + "__"
if m == "" or m == "\n" :
print "NOT INITIALIZED 2"
return ret_val[2], param_value.strip() # NOT INITIALIZED
else:
param_value = m
print "INITIALIZED return " + ret_val[0] + " " + param_value.strip()
return ret_val[0], param_value.strip() # INITIALIZED
else:
print "NOT PRESENT"
return ret_val[1], param_value.strip() # NOT PRESENT
def checkIfParamIsNull(cls, paramName, paramValue):
if len(paramValue) == 0:
raise testsuite_exception.TestsuiteError(
"Parameter %s is empty. Check configuration" % paramName)