def get_target_ip(target): try: client_ip = settings.get_value('CLIENT', 'ip', type=str) except Exception: raise else: return client_ip
def get_actual_commands(commands, target): if commands is None or commands == '': return None post_commands = commands if re.findall('\$SERVER_IP', commands): try: server_ip = settings.get_value('SERVER', 'ip', type=str) except Exception, e: server_ips = server_utils.get_local_ip() server_ip = "" for each_ip in server_ips: items = each_ip.split(".")[0:2] pre = '.'.join(items) if target.ip.startswith(pre): server_ip = each_ip break if not server_ip: if len(server_ips) > 1: try: server_ips.remove("127.0.0.1") except Exception: raise e server_ip = server_ips[0] strinfo = re.compile('\$SERVER_IP') post_commands = strinfo.sub(server_ip, commands)
def get_target_ip(target): try: client_ip = settings.get_value('TARGET', 'ip', type=str) except Exception: raise else: return client_ip
def get_BMC_host(self): try: bmc_ip = settings.get_value('BMCER', 'host', type=str) port = settings.get_value('BMCER', 'port', type=int) user = settings.get_value('BMCER', 'user', type=str) passwd = settings.get_value('BMCER', 'password', type=str) command = settings.get_value('BMCER', 'command', type=str) except Exception as e: raise error.ServRunError(e.args[0], e.args[1]) else: self.command = command try: bmc_host = host_factory.create_host(bmc_ip, user, passwd, port) except Exception as e: raise error.ServRunError(e.args[0], e.args[1]) else: self.host = bmc_host
def get_pid_path(program_name, pid_files_dir=None): if pid_files_dir is None: pid_files_dir = settings.get_value("SERVER", "pid_files_dir", default="") if not pid_files_dir: base_dir = os.path.dirname(__file__) pid_path = os.path.abspath(os.path.join(base_dir, "...", "..", "%s.pid" % program_name)) else: pid_path = os.path.join(pid_files_dir, "%s.pid" % program_name) return pid_path
def get_pid_path(program_name, pid_files_dir=None): if pid_files_dir is None: pid_files_dir = settings.get_value("SERVER", "pid_files_dir", default="") if not pid_files_dir: base_dir = os.path.dirname(__file__) pid_path = os.path.abspath( os.path.join(base_dir, "...", "..", "%s.pid" % program_name)) else: pid_path = os.path.join(pid_files_dir, "%s.pid" % program_name) return pid_path
def get_user_keys(hostname): """Returns a mapping of path -> paramiko.PKey entries available for this user. Keys are found in the default locations (~/.ssh/id_[d|r]sa) as well as any IdentityFile entries in the standard ssh config files. """ raw_identity_files = ["~/.ssh/id_dsa", "~/.ssh/id_rsa"] for config_path in ("/etc/ssh/ssh_config", "~/.ssh/config"): config_path = os.path.expanduser(config_path) if not os.path.exists(config_path): continue host_pattern = "*" config_lines = open(config_path).readlines() for line in config_lines: key, value = ParamikoHost._parse_config_line(line) if key == "Host": host_pattern = value elif (key == "IdentityFile" and fnmatch.fnmatch(hostname, host_pattern)): raw_identity_files.append(value) # drop any files that use percent-escapes; we don't support them identity_files = [] UNSUPPORTED_ESCAPES = ["%d", "%u", "%l", "%h", "%r"] for path in raw_identity_files: # skip this path if it uses % escapes if sum((escape in path) for escape in UNSUPPORTED_ESCAPES): continue path = os.path.expanduser(path) if os.path.exists(path): identity_files.append(path) # load up all the keys that we can and return them user_keys = {} for path in identity_files: key = ParamikoHost._load_key(path) if key: user_keys[path] = key # load up all the ssh agent keys use_sshagent = settings.get_value('AUTOSERV', 'use_sshagent_with_paramiko', type=bool) if use_sshagent: ssh_agent = paramiko.Agent() for i, key in enumerate(ssh_agent.get_keys()): user_keys['agent-key-%d' % i] = key return user_keys
def judge_target_crash(): result = -1 try: client = settings.get_value('CLIENT', 'ip', type=str) commands = 'ping -c 10 %s' % client result = utils.run(commands) except error.CmdError: return True else: if result.exit_status: return True else: output = result.stdout if re.search("100%\spacket\sloss", output): return True else: return False
def run_all_cases(target_exec_dir, target, kind_bench, bench_name, run_file, parser_file): """ function: run one benchmark which was selected in the configuration files """ try: # get the abspath, which is filename of run config for the benchmark bench_conf_file = os.path.join(caliper_path.config_files.tests_cfg_dir, kind_bench, run_file) # get the config sections for the benchmrk configRun, sections_run = server_utils.read_config_file( bench_conf_file) except AttributeError as e: raise AttributeError except Exception: raise logging.debug("the sections to run are: %s" % sections_run) if not os.path.exists(Folder.exec_dir): os.mkdir(Folder.exec_dir) log_bench = os.path.join(Folder.exec_dir, bench_name) logfile = log_bench + "_output.log" tmp_log_file = log_bench + "_output_tmp.log" parser_result_file = log_bench + "_parser.log" tmp_parser_file = log_bench + "_parser_tmp.log" if os.path.exists(parser_result_file): os.remove(parser_result_file) if os.path.exists(logfile): os.remove(logfile) starttime = datetime.datetime.now() result = subprocess.call( "echo '$$ %s EXECUTION START: %s' >> %s" % (bench_name, str(starttime)[:19], Folder.caliper_log_file), shell=True) bench_test = "ltp" if bench_name == bench_test: tar_ip = settings.get_value('CLIENT', 'ip', type=str) target.run("if [[ ! -e /mnt/ltp ]]; then mkdir -p /mnt/ltp; fi") # fix me , now that we create the folder, why not we mount it directly here s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: # fix me , getting host ip to be optimised s.connect(("8.8.8.8", 80)) except Exception: logging.debug( "Socket connection failed during ltp pre-requisite check") host_ip = s.getsockname()[0] try: xyz = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/ltp" % (host_ip)) except Exception: try: xyz = target.run("umount /mnt/ltp/") xyz = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/ltp" % (host_ip)) except Exception: logging.debug("Unable to mount") return result readme_file = log_bench + "_README" resultltp = subprocess.call("touch %s" % (readme_file), shell=True) resultltp = subprocess.call( "echo 'The categorization of ltp in caliper is\nCATEGORY\t\t\t\t\t\tSCENARIOS OF LTP\n\n[command]\t\t\t\t\t\tcommands\n[cpu]\t\t\t\t\t\tsched,cpuhotplug\n[memory]\t\t\t\t\t\tmm.numa,hugetlb\n[dio]\t\t\t\t\t\tdio,io,dma_thread_diotest,timers\n[filesystem]\t\t\t\t\t\tfilecaps,fs,fs_bind,fs_ext4,fs_perms_simple,fs_readonly\n[kernel/\t\t\t\t\t\tsyscalls,controllers,pty,containers,admin_tools,modules,can\n[proc]\t\t\t\t\t\tipc,hyperthreading,nptl,cap_bounds,connectors,pipes\n\n\nltp_output.log contains the screenshot of complete ltp execution and ltp_parser.log contains the information regarding the number of tests executed and among them which all have passed failed or skipped.\n\nFor more information regarding a particular category please see ltp_<category>_output.log which contains the output screen and parser log for that particular category' >> %s" % (readme_file), shell=True) # for each command in run config file, read the config for the benchmark for i in range(0, len(sections_run)): flag = 0 try: category = configRun.get(sections_run[i], 'category') scores_way = configRun.get(sections_run[i], 'scores_way') parser = configRun.get(sections_run[i], 'parser') command = configRun.get(sections_run[i], 'command') except Exception: logging.debug("no value for the %s" % sections_run[i]) continue if bench_name == bench_test: subsection = sections_run[i].split(" ")[1] subsection_file = log_bench + "_" + subsection + "_output.log" if os.path.exists(tmp_parser_file): os.remove(tmp_parser_file) if os.path.exists(tmp_log_file): os.remove(tmp_log_file) server_run_command = get_server_command(kind_bench, sections_run[i]) logging.debug("Get the server command is: %s" % server_run_command) # run the command of the benchmarks try: flag = run_kinds_commands(sections_run[i], server_run_command, tmp_log_file, kind_bench, target, command) except Exception, e: logging.info(e) crash_handle.main() if bench_name == bench_test: xyz = subprocess.call("mv /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir), shell=True) server_utils.file_copy(logfile, tmp_log_file, 'a+') if os.path.exists(tmp_log_file): os.remove(tmp_log_file) run_flag = server_utils.get_fault_tolerance_config( 'fault_tolerance', 'run_error_continue') if run_flag == 1: continue else: return result else: if bench_name == bench_test: xyz = subprocess.call("mv /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir), shell=True) if os.path.exists(subsection_file): server_utils.file_copy(tmp_log_file, subsection_file, 'a+') server_utils.file_copy(logfile, tmp_log_file, 'a+') if flag != 1: logging.info("There is wrong when running the command \"%s\"" % command) if os.path.exists(tmp_log_file): os.remove(tmp_log_file) crash_handle.main() run_flag = server_utils.get_fault_tolerance_config( 'fault_tolerance', 'run_error_continue') if run_flag == 1: if bench_name != bench_test: continue else: return result # parser the result in the tmp_log_file, the result is the output of # running the command try: logging.debug("Parsering the result of command: %s" % command) if bench_name == bench_test: outfp = open(tmp_parser_file, "w") outfp.write("%s" % (subsection)) outfp.close() parser_result = parser_case(kind_bench, bench_name, parser_file, parser, subsection_file, tmp_parser_file) else: parser_result = parser_case(kind_bench, bench_name, parser_file, parser, tmp_log_file, tmp_parser_file) except Exception, e: logging.info( "There's wrong when parsering the result of \" %s \"" % sections_run[i]) logging.info(e) if os.path.exists(tmp_parser_file): os.remove(tmp_parser_file) if os.path.exists(tmp_log_file): os.remove(tmp_log_file)
msg = "Caliper does not support this kind of arch machine" raise error.ServUnsupportedArchError(msg) def get_host_name(host): try: arch_result = host.run("/bin/uname -a") except error.CmdError, e: raise error.ServRunError(e.args[0], e.args[1]) else: returncode = arch_result.exit_status if returncode == 0: output = arch_result.stdout try: machine_name = settings.get_value('TARGET', 'Platform_name', type=str) except: machine_name = output.split(" ")[1] return machine_name else: msg = "Caliper does not support this kind of arch machine" raise error.ServUnsupportedArchError(msg) def get_host_hardware_info(host): hardware_info = {} try: cpu_type = host.run("grep 'model name' /proc/cpuinfo |uniq \ |awk -F : '{print $2}' |sed 's/^[ \t]*//g'\ |sed 's/ \+/ /g'")
lscpu_list = [ 'Architecture', 'Socket\(s\)', 'Cpu_Type', 'Core\(s\) per socket', 'Thread\(s\) per core', 'Model name', 'CPU\(s\)', 'NUMA node\(s\)', 'BogoMIPS', 'Byte Order' ] lsb_release_list = ['Distributor ID', 'Description', 'Release', 'Codename'] lspci_list = ['Ethernet controller [0200]'] def get_remote_host(): try: client_ip = settings.get_value('TARGET', 'ip', type=str) except Exception, e: client_ip = '127.0.0.1' try: port = settings.get_value('TARGET', 'port', type=int) except Exception, e: port = 22 try: user = settings.get_value('TARGET', 'user', type=str) except Exception, e: user = os.getlogin() try: password = settings.get_value('TARGET', 'password', type=str) except Exception, e: raise error.ServRunError(e.args[0], e.args[1]) remote_host = host_factory.create_host(client_ip, user, password, port) return remote_host
def get_remote_host(): try: client_ip = settings.get_value('TARGET', 'ip', type=str) except Exception, e: client_ip = '127.0.0.1'
else: msg = "Caliper does not support this kind of arch machine" raise error.ServUnsupportedArchError(msg) def get_host_name(host): try: arch_result = host.run("/bin/uname -a") except error.CmdError, e: raise error.ServRunError(e.args[0], e.args[1]) else: returncode = arch_result.exit_status if returncode == 0: output = arch_result.stdout try: machine_name = settings.get_value('CLIENT', 'Platform_name', type=str) except: machine_name = output.split(" ")[1] return machine_name else: msg = "Caliper does not support this kind of arch machine" raise error.ServUnsupportedArchError(msg) def get_host_hardware_info(host): hardware_info = {} try: cpu_type = host.run("grep 'model name' /proc/cpuinfo |uniq \ |awk -F : '{print $2}' |sed 's/^[ \t]*//g'\ |sed 's/ \+/ /g'") logic_cpu = host.run("grep 'processor' /proc/cpuinfo |sort |uniq \
def caliper_run(target_exec_dir, server, target): # get the test cases defined files config_files = server_utils.get_cases_def_files(target_exec_dir) logging.debug("the selected configuration are %s" % config_files) for i in range(0, len(config_files)): # run benchmarks selected in each configuration file # config_file = os.path.join(caliper_path.CALIPER_PRE, config_files[i]) config_file = os.path.join(config_files[i]) config, sections = server_utils.read_config_file(config_file) logging.debug(sections) # get if it is the 'common' or 'arm' or 'android' classify = config_files[i].split("/")[-1].strip().split("_")[0] logging.debug(classify) if classify == "server" and server: try: server_ip = settings.get_value("SERVER", "ip", type=str) server_port = settings.get_value("SERVER", "port", type=int) server_user = settings.get_value("SERVER", "user", type=str) logging.info( "Please wait while caliper triggers the server.py script in the server" ) server_pwd = server.run("pwd").stdout server_pwd = server_pwd.split("\n")[0] server_caliper_dir = os.path.join(server_pwd, "caliper_server") server_caliper_dir = os.path.join(server_caliper_dir, "server.py") server_user = server_user + '@' + server_ip script = server_caliper_dir + ' ' + str(server_port) subprocess.Popen( ['ssh', '%s' % server_user, 'python %s' % script]) except Exception as e: logging.info(e) raise AttributeError( "Error in establising connection with server") for i in range(0, len(sections)): # run for each benchmark target_arch = server_utils.get_host_arch(target) build_name = sections[i] + '_' + target_arch + '.suc' build_suc = os.path.join(Folder.build_dir, build_name) if not os.path.exists(build_suc): continue build_host_name = sections[i] + '_' + \ server_utils.get_local_machine_arch() + '.fail' if os.path.exists(build_host_name): continue # try to resolve the configuration of the configuration file try: run_file = config.get(sections[i], 'run') parser = config.get(sections[i], 'parser') except Exception: raise AttributeError("The is no option value of parser") print_format() logging.info("Running %s" % sections[i]) bench = os.path.join(classify, sections[i]) try: system_initialise(target) if classify == "server": logging.info("Waiting for server to grant access") sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((server_ip, server_port)) logging.info("%s" % str(sock.recv(1024))) result = run_all_cases(target_exec_dir, target, bench, sections[i], run_file) if classify == "server": sock.send("1") sock.close() except Exception: logging.info("Running %s Exception" % sections[i]) crash_handle.main() print_format() if sections[i] == "ltp": try: unmount = target.run( "if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/; fi" ) except Exception: unmount = target.run( "if df -h |grep caliper_nfs ; then fuser -km /mnt/caliper_nfs ;fi" ) unmount = target.run( "if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/ ;fi" ) run_flag = server_utils.get_fault_tolerance_config( 'fault_tolerance', 'run_error_continue') if run_flag == 1: continue else: return result else: logging.info("Running %s Finished" % sections[i]) if sections[i] == "ltp": try: unmount = target.run( "if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/ ;fi" ) except Exception: unmount = target.run( "if df -h |grep caliper_nfs ; then fuser -km /mnt/caliper_nfs/ ;fi" ) unmount = target.run( "if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/ ;fi" ) print_format() return 0
# import os import logging import tempfile import socket import glob import shutil from caliper.client.shared import error, autotemp from caliper.server import utils from caliper.server.hosts import remote_host from caliper.client.shared.settings import settings enable_master_ssh = settings.get_value('SERVER', 'enable_master_ssh', type=bool, default=False) def _make_ssh_cmd_default(user="******", port=22, opts='', hosts_file='/dev/null', connect_timeout=30, alive_interval=300): base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=%s -o BatchMode=yes " "-o ConnectTimeout=%d -o ServerAliveInterval=%d " "-l %s -p %d") assert isinstance(connect_timeout, (int, long)) assert connect_timeout > 0
def get_client_autodir_paths(cls, host): return settings.get_value('auto', 'client_autodir_paths', type=list)
def run_all_cases(target_exec_dir, target, kind_bench, bench_name, run_file): """ function: run one benchmark which was selected in the configuration files """ try: # get the abspath, which is filename of run config for the benchmark bench_conf_file = os.path.join( caliper_path.config_files.tests_cfg_dir, kind_bench, run_file) # get the config sections for the benchmrk configRun, sections_run = server_utils.read_config_file( bench_conf_file) except AttributeError as e: raise AttributeError except Exception: raise logging.debug("the sections to run are: %s" % sections_run) if not os.path.exists(Folder.exec_dir): os.mkdir(Folder.exec_dir) log_bench = os.path.join(Folder.exec_dir, bench_name) logfile = log_bench + "_output.log" tmp_log_file = log_bench + "_output_tmp.log" if os.path.exists(logfile): os.remove(logfile) starttime = datetime.datetime.now() if os.path.exists(Folder.caliper_log_file): sections = bench_name + " EXECUTION" fp = open(Folder.caliper_log_file,"r") f = fp.readlines() fp.close() op = open(Folder.caliper_log_file,"w") for line in f: if not(sections in line): op.write(line) op.close() result = subprocess.call("echo '$$ %s EXECUTION START: %s' >> %s" % (bench_name, str(starttime)[:19], Folder.caliper_log_file), shell=True) bench_test = "ltp" if bench_name == bench_test: tar_ip = settings.get_value('CLIENT', 'ip', type=str) target.run("if [[ ! -e /mnt/caliper_nfs ]]; then mkdir -p /mnt/caliper_nfs; fi") # fix me , now that we create the folder, why not we mount it directly here try: tar_mask = ".".join(tar_ip.split(".")[0:3]) p1 = subprocess.Popen(["ifconfig"], stdout=subprocess.PIPE) p2 = subprocess.Popen(["grep", tar_mask], stdin=p1.stdout, stdout=subprocess.PIPE) p1.stdout.close() output,err = p2.communicate() output = output.strip() host_ip = output.split("inet addr:")[1].split(" ")[0] except Exception: logging.debug("Unable to get the host_ip" ) try: mount_cmd = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" % (host_ip) ) except Exception: try: umount_cmd = target.run("umount /mnt/caliper_nfs/") mount_cmd = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" % (host_ip) ) except Exception: logging.debug("Unable to mount") return result readme_file=log_bench+"_README" resultltp = subprocess.call("touch %s" %(readme_file),shell=True) resultltp = subprocess.call("echo 'The categorization of ltp in caliper is\nCATEGORY\t\t\t\t\t\tSCENARIOS OF LTP\n\n[command]\t\t\t\t\t\tcommands\n[cpu]\t\t\t\t\t\tsched,cpuhotplug\n[memory]\t\t\t\t\t\tmm.numa,hugetlb\n[dio]\t\t\t\t\t\tdio,io,dma_thread_diotest,timers\n[filesystem]\t\t\t\t\t\tfilecaps,fs,fs_bind,fs_ext4,fs_perms_simple,fs_readonly\n[kernel/\t\t\t\t\t\tsyscalls,controllers,pty,containers,admin_tools,modules,can\n[proc]\t\t\t\t\t\tipc,hyperthreading,nptl,cap_bounds,connectors,pipes\n\n\nltp_output.log contains the screenshot of complete ltp execution and ltp_parser.log contains the information regarding the number of tests executed and among them which all have passed failed or skipped.\n\nFor more information regarding a particular category please see ltp_<category>_output.log which contains the output screen and parser log for that particular category' >> %s" %(readme_file),shell=True) # for each command in run config file, read the config for the benchmark for i in range(0, len(sections_run)): flag = 0 try: category = configRun.get(sections_run[i], 'category') command = configRun.get(sections_run[i], 'command') except Exception: logging.debug("no value for the %s" % sections_run[i]) continue if bench_name == bench_test: subsection = sections_run[i].split(" ")[1] subsection_file = log_bench + "_" + subsection + "_output.log" if os.path.exists(tmp_log_file): os.remove(tmp_log_file) server_run_command = get_server_command(kind_bench, sections_run[i]) logging.debug("Get the server command is: %s" % server_run_command) # run the command of the benchmarks try: flag = run_kinds_commands(sections_run[i], server_run_command, tmp_log_file, kind_bench, target, command) except Exception, e: logging.info(e) crash_handle.main() if bench_name == bench_test: move_logs = subprocess.call("cp /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir), shell=True) server_utils.file_copy(logfile, tmp_log_file, 'a+') if os.path.exists(tmp_log_file): os.remove(tmp_log_file) run_flag = server_utils.get_fault_tolerance_config( 'fault_tolerance', 'run_error_continue') if run_flag == 1: continue else: return result else: if bench_name == bench_test: move_logs = subprocess.call("cp /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir), shell=True) if os.path.exists(subsection_file): server_utils.file_copy(tmp_log_file,subsection_file, 'a+') server_utils.file_copy(logfile, tmp_log_file, 'a+') if flag != 1: logging.info("There is wrong when running the command \"%s\"" % command) if os.path.exists(tmp_log_file): os.remove(tmp_log_file) crash_handle.main() run_flag = server_utils.get_fault_tolerance_config( 'fault_tolerance', 'run_error_continue') if run_flag == 1: if bench_name != bench_test: continue else: return result if os.path.exists(tmp_log_file): os.remove(tmp_log_file)
msg = "Caliper does not support this kind of arch machine" raise error.ServUnsupportedArchError(msg) def get_host_name(host): try: arch_result = host.run("/bin/uname -a") except error.CmdError, e: raise error.ServRunError(e.args[0], e.args[1]) else: returncode = arch_result.exit_status if returncode == 0: output = arch_result.stdout try: machine_name = settings.get_value('CLIENT', 'Platform_name', type=str) except: machine_name = output.split(" ")[1] return machine_name else: msg = "Caliper does not support this kind of arch machine" raise error.ServUnsupportedArchError(msg) def get_host_hardware_info(host): hardware_info = {} try: cpu_type = host.run("grep 'model name' /proc/cpuinfo |uniq \ |awk -F : '{print $2}' |sed 's/^[ \t]*//g'\ |sed 's/ \+/ /g'")
from caliper.server.hosts import host_factory from caliper.client.shared import error from caliper.client.shared.settings import settings from caliper.client.shared.caliper_path import folder_ope as Folder path = "./HardwareInfo" out_path = "./hardware_yaml" lscpu_list = ['Architecture','Socket\(s\)','Cpu_Type','Core\(s\) per socket','Thread\(s\) per core','Model name','CPU\(s\)','NUMA node\(s\)','BogoMIPS','Byte Order'] lsb_release_list = ['Distributor ID','Description','Release','Codename'] lspci_list = ['Ethernet controller [0200]'] def get_remote_host(): try: client_ip = settings.get_value('CLIENT', 'ip', type=str) except Exception, e: client_ip = '127.0.0.1' try: port = settings.get_value('CLIENT', 'port', type=int) except Exception, e: port = 22 try: user = settings.get_value('CLIENT', 'user', type=str) except Exception, e: user = os.getlogin() try: password = settings.get_value('CLIENT', 'password', type=str) except Exception, e: raise error.ServRunError(e.args[0], e.args[1]) remote_host = host_factory.create_host(client_ip, user, password, port) return remote_host def network_populate(dic,contents):
def get_remote_host(): try: client_ip = settings.get_value('CLIENT', 'ip', type=str) except Exception, e: client_ip = '127.0.0.1'
# import os import logging import tempfile import socket import glob import shutil from caliper.client.shared import error, autotemp from caliper.server import utils from caliper.server.hosts import remote_host from caliper.client.shared.settings import settings enable_master_ssh = settings.get_value('SERVER', 'enable_master_ssh', type=bool, default=False) def _make_ssh_cmd_default(user="******", port=22, opts='', hosts_file='/dev/null', connect_timeout=30, alive_interval=300): base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=%s -o BatchMode=yes " "-o ConnectTimeout=%d -o ServerAliveInterval=%d " "-l %s -p %d") assert isinstance(connect_timeout, (int, long)) assert connect_timeout > 0 if isinstance(port, str): port = int(port) return base_command % (opts, hosts_file, connect_timeout, alive_interval, user, port)
def run_all_cases(target_exec_dir, target, kind_bench, bench_name, run_file): """ function: run one benchmark which was selected in the configuration files """ try: # get the abspath, which is filename of run config for the benchmark bench_conf_file = os.path.join(caliper_path.config_files.tests_cfg_dir, kind_bench, run_file) # get the config sections for the benchmrk configRun, sections_run = server_utils.read_config_file( bench_conf_file) except AttributeError as e: raise AttributeError except Exception: raise logging.debug("the sections to run are: %s" % sections_run) if not os.path.exists(Folder.exec_dir): os.mkdir(Folder.exec_dir) log_bench = os.path.join(Folder.exec_dir, bench_name) logfile = log_bench + "_output.log" tmp_log_file = log_bench + "_output_tmp.log" if os.path.exists(logfile): os.remove(logfile) starttime = datetime.datetime.now() if os.path.exists(Folder.caliper_log_file): sections = bench_name + " EXECUTION" fp = open(Folder.caliper_log_file, "r") f = fp.readlines() fp.close() op = open(Folder.caliper_log_file, "w") for line in f: if not (sections in line): op.write(line) op.close() result = subprocess.call( "echo '$$ %s EXECUTION START: %s' >> %s" % (bench_name, str(starttime)[:19], Folder.caliper_log_file), shell=True) bench_test = "ltp" if bench_name == bench_test: tar_ip = settings.get_value('CLIENT', 'ip', type=str) target.run( "if [[ ! -e /mnt/caliper_nfs ]]; then mkdir -p /mnt/caliper_nfs; fi" ) # fix me , now that we create the folder, why not we mount it directly here try: tar_mask = ".".join(tar_ip.split(".")[0:3]) p1 = subprocess.Popen(["ifconfig"], stdout=subprocess.PIPE) p2 = subprocess.Popen(["grep", tar_mask], stdin=p1.stdout, stdout=subprocess.PIPE) p1.stdout.close() output, err = p2.communicate() output = output.strip() host_ip = output.split("inet addr:")[1].split(" ")[0] except Exception: logging.debug("Unable to get the host_ip") try: mount_cmd = target.run( "mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" % (host_ip)) except Exception: try: umount_cmd = target.run("umount /mnt/caliper_nfs/") mount_cmd = target.run( "mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" % (host_ip)) except Exception: logging.debug("Unable to mount") return result readme_file = log_bench + "_README" resultltp = subprocess.call("touch %s" % (readme_file), shell=True) resultltp = subprocess.call( "echo 'The categorization of ltp in caliper is\nCATEGORY\t\t\t\t\t\tSCENARIOS OF LTP\n\n[command]\t\t\t\t\t\tcommands\n[cpu]\t\t\t\t\t\tsched,cpuhotplug\n[memory]\t\t\t\t\t\tmm.numa,hugetlb\n[dio]\t\t\t\t\t\tdio,io,dma_thread_diotest,timers\n[filesystem]\t\t\t\t\t\tfilecaps,fs,fs_bind,fs_ext4,fs_perms_simple,fs_readonly\n[kernel/\t\t\t\t\t\tsyscalls,controllers,pty,containers,admin_tools,modules,can\n[proc]\t\t\t\t\t\tipc,hyperthreading,nptl,cap_bounds,connectors,pipes\n\n\nltp_output.log contains the screenshot of complete ltp execution and ltp_parser.log contains the information regarding the number of tests executed and among them which all have passed failed or skipped.\n\nFor more information regarding a particular category please see ltp_<category>_output.log which contains the output screen and parser log for that particular category' >> %s" % (readme_file), shell=True) # for each command in run config file, read the config for the benchmark for i in range(0, len(sections_run)): flag = 0 try: category = configRun.get(sections_run[i], 'category') command = configRun.get(sections_run[i], 'command') except Exception: logging.debug("no value for the %s" % sections_run[i]) continue if bench_name == bench_test: subsection = sections_run[i].split(" ")[1] subsection_file = log_bench + "_" + subsection + "_output.log" if os.path.exists(tmp_log_file): os.remove(tmp_log_file) server_run_command = get_server_command(kind_bench, sections_run[i]) logging.debug("Get the server command is: %s" % server_run_command) # run the command of the benchmarks try: flag = run_kinds_commands(sections_run[i], server_run_command, tmp_log_file, kind_bench, target, command) except Exception, e: logging.info(e) crash_handle.main() if bench_name == bench_test: move_logs = subprocess.call( "cp /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir), shell=True) server_utils.file_copy(logfile, tmp_log_file, 'a+') if os.path.exists(tmp_log_file): os.remove(tmp_log_file) run_flag = server_utils.get_fault_tolerance_config( 'fault_tolerance', 'run_error_continue') if run_flag == 1: continue else: return result else: if bench_name == bench_test: move_logs = subprocess.call( "cp /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir), shell=True) if os.path.exists(subsection_file): server_utils.file_copy(tmp_log_file, subsection_file, 'a+') server_utils.file_copy(logfile, tmp_log_file, 'a+') if flag != 1: logging.info("There is wrong when running the command \"%s\"" % command) if os.path.exists(tmp_log_file): os.remove(tmp_log_file) crash_handle.main() run_flag = server_utils.get_fault_tolerance_config( 'fault_tolerance', 'run_error_continue') if run_flag == 1: if bench_name != bench_test: continue else: return result if os.path.exists(tmp_log_file): os.remove(tmp_log_file)
break if not server_ip: if len(server_ips) > 1: try: server_ips.remove("127.0.0.1") except Exception: raise e server_ip = server_ips[0] strinfo = re.compile('\$SERVER_IP') post_commands = strinfo.sub(server_ip, commands) commands = post_commands if re.findall('\$CLIENT_IP', commands): try: client_ip = settings.get_value('CLIENT', 'ip', type=str) except Exception, e: client_ip = '127.0.0.1' strinfo = re.compile('\$CLIENT_IP') post_commands = strinfo.sub(client_ip, commands) commands = post_commands if commands[0] == '\'' and commands[-1] == '\'': actual_commands = commands[1:-1] elif commands[0] == '\"' and commands[-1] == '\"': actual_commands = commands[1:-1] else: actual_commands = commands if actual_commands == '': return '' return actual_commands
# -*- coding:utf-8 -*- import os import logging import tempfile import socket import glob import shutil from caliper.client.shared import error, autotemp from caliper.server import utils from caliper.server.hosts import remote_host from caliper.client.shared.settings import settings enable_master_ssh = settings.get_value('TestNode', 'enable_master_ssh', type=bool, default=False) def _make_ssh_cmd_default(user="******", port=22, opts='', hosts_file='/dev/null', connect_timeout=30, alive_interval=300): base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no " "-o UserKnownHostsFile=%s -o BatchMode=yes " "-o ConnectTimeout=%d -o ServerAliveInterval=%d " "-l %s -p %d") assert isinstance(connect_timeout, (int, long)) assert connect_timeout > 0