def get_target_ip(target):
try:
client_ip = settings.get_value('CLIENT', 'ip', type=str)
except Exception:
raise
else:
return client_ip
def get_actual_commands(commands, target):
if commands is None or commands == '':
return None
post_commands = commands
if re.findall('\$SERVER_IP', commands):
try:
server_ip = settings.get_value('SERVER', 'ip', type=str)
except Exception, e:
server_ips = server_utils.get_local_ip()
server_ip = ""
for each_ip in server_ips:
items = each_ip.split(".")[0:2]
pre = '.'.join(items)
if target.ip.startswith(pre):
server_ip = each_ip
break
if not server_ip:
if len(server_ips) > 1:
try:
server_ips.remove("127.0.0.1")
except Exception:
raise e
server_ip = server_ips[0]
strinfo = re.compile('\$SERVER_IP')
post_commands = strinfo.sub(server_ip, commands)
def get_actual_commands(commands, target):
if commands is None or commands == '':
return None
post_commands = commands
if re.findall('\$SERVER_IP', commands):
try:
server_ip = settings.get_value('SERVER', 'ip', type=str)
except Exception, e:
server_ips = server_utils.get_local_ip()
server_ip = ""
for each_ip in server_ips:
items = each_ip.split(".")[0:2]
pre = '.'.join(items)
if target.ip.startswith(pre):
server_ip = each_ip
break
if not server_ip:
if len(server_ips) > 1:
try:
server_ips.remove("127.0.0.1")
except Exception:
raise e
server_ip = server_ips[0]
strinfo = re.compile('\$SERVER_IP')
post_commands = strinfo.sub(server_ip, commands)
def get_target_ip(target):
try:
client_ip = settings.get_value('TARGET', 'ip', type=str)
except Exception:
raise
else:
return client_ip
def get_BMC_host(self):
try:
bmc_ip = settings.get_value('BMCER', 'host', type=str)
port = settings.get_value('BMCER', 'port', type=int)
user = settings.get_value('BMCER', 'user', type=str)
passwd = settings.get_value('BMCER', 'password', type=str)
command = settings.get_value('BMCER', 'command', type=str)
except Exception as e:
raise error.ServRunError(e.args[0], e.args[1])
else:
self.command = command
try:
bmc_host = host_factory.create_host(bmc_ip, user, passwd, port)
except Exception as e:
raise error.ServRunError(e.args[0], e.args[1])
else:
self.host = bmc_host
def get_pid_path(program_name, pid_files_dir=None):
if pid_files_dir is None:
pid_files_dir = settings.get_value("SERVER", "pid_files_dir",
default="")
if not pid_files_dir:
base_dir = os.path.dirname(__file__)
pid_path = os.path.abspath(os.path.join(base_dir, "...", "..",
"%s.pid" % program_name))
else:
pid_path = os.path.join(pid_files_dir, "%s.pid" % program_name)
return pid_path
def get_pid_path(program_name, pid_files_dir=None):
if pid_files_dir is None:
pid_files_dir = settings.get_value("SERVER",
"pid_files_dir",
default="")
if not pid_files_dir:
base_dir = os.path.dirname(__file__)
pid_path = os.path.abspath(
os.path.join(base_dir, "...", "..", "%s.pid" % program_name))
else:
pid_path = os.path.join(pid_files_dir, "%s.pid" % program_name)
return pid_path
def get_user_keys(hostname):
"""Returns a mapping of path -> paramiko.PKey entries available for
this user. Keys are found in the default locations (~/.ssh/id_[d|r]sa)
as well as any IdentityFile entries in the standard ssh config files.
"""
raw_identity_files = ["~/.ssh/id_dsa", "~/.ssh/id_rsa"]
for config_path in ("/etc/ssh/ssh_config", "~/.ssh/config"):
config_path = os.path.expanduser(config_path)
if not os.path.exists(config_path):
continue
host_pattern = "*"
config_lines = open(config_path).readlines()
for line in config_lines:
key, value = ParamikoHost._parse_config_line(line)
if key == "Host":
host_pattern = value
elif (key == "IdentityFile"
and fnmatch.fnmatch(hostname, host_pattern)):
raw_identity_files.append(value)
# drop any files that use percent-escapes; we don't support them
identity_files = []
UNSUPPORTED_ESCAPES = ["%d", "%u", "%l", "%h", "%r"]
for path in raw_identity_files:
# skip this path if it uses % escapes
if sum((escape in path) for escape in UNSUPPORTED_ESCAPES):
continue
path = os.path.expanduser(path)
if os.path.exists(path):
identity_files.append(path)
# load up all the keys that we can and return them
user_keys = {}
for path in identity_files:
key = ParamikoHost._load_key(path)
if key:
user_keys[path] = key
# load up all the ssh agent keys
use_sshagent = settings.get_value('AUTOSERV',
'use_sshagent_with_paramiko',
type=bool)
if use_sshagent:
ssh_agent = paramiko.Agent()
for i, key in enumerate(ssh_agent.get_keys()):
user_keys['agent-key-%d' % i] = key
return user_keys
def judge_target_crash():
result = -1
try:
client = settings.get_value('CLIENT', 'ip', type=str)
commands = 'ping -c 10 %s' % client
result = utils.run(commands)
except error.CmdError:
return True
else:
if result.exit_status:
return True
else:
output = result.stdout
if re.search("100%\spacket\sloss", output):
return True
else:
return False
def run_all_cases(target_exec_dir, target, kind_bench, bench_name, run_file,
parser_file):
"""
function: run one benchmark which was selected in the configuration files
"""
try:
# get the abspath, which is filename of run config for the benchmark
bench_conf_file = os.path.join(caliper_path.config_files.tests_cfg_dir,
kind_bench, run_file)
# get the config sections for the benchmrk
configRun, sections_run = server_utils.read_config_file(
bench_conf_file)
except AttributeError as e:
raise AttributeError
except Exception:
raise
logging.debug("the sections to run are: %s" % sections_run)
if not os.path.exists(Folder.exec_dir):
os.mkdir(Folder.exec_dir)
log_bench = os.path.join(Folder.exec_dir, bench_name)
logfile = log_bench + "_output.log"
tmp_log_file = log_bench + "_output_tmp.log"
parser_result_file = log_bench + "_parser.log"
tmp_parser_file = log_bench + "_parser_tmp.log"
if os.path.exists(parser_result_file):
os.remove(parser_result_file)
if os.path.exists(logfile):
os.remove(logfile)
starttime = datetime.datetime.now()
result = subprocess.call(
"echo '$$ %s EXECUTION START: %s' >> %s" %
(bench_name, str(starttime)[:19], Folder.caliper_log_file),
shell=True)
bench_test = "ltp"
if bench_name == bench_test:
tar_ip = settings.get_value('CLIENT', 'ip', type=str)
target.run("if [[ ! -e /mnt/ltp ]]; then mkdir -p /mnt/ltp; fi")
# fix me , now that we create the folder, why not we mount it directly here
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# fix me , getting host ip to be optimised
s.connect(("8.8.8.8", 80))
except Exception:
logging.debug(
"Socket connection failed during ltp pre-requisite check")
host_ip = s.getsockname()[0]
try:
xyz = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/ltp" %
(host_ip))
except Exception:
try:
xyz = target.run("umount /mnt/ltp/")
xyz = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/ltp" %
(host_ip))
except Exception:
logging.debug("Unable to mount")
return result
readme_file = log_bench + "_README"
resultltp = subprocess.call("touch %s" % (readme_file), shell=True)
resultltp = subprocess.call(
"echo 'The categorization of ltp in caliper is\nCATEGORY\t\t\t\t\t\tSCENARIOS OF LTP\n\n[command]\t\t\t\t\t\tcommands\n[cpu]\t\t\t\t\t\tsched,cpuhotplug\n[memory]\t\t\t\t\t\tmm.numa,hugetlb\n[dio]\t\t\t\t\t\tdio,io,dma_thread_diotest,timers\n[filesystem]\t\t\t\t\t\tfilecaps,fs,fs_bind,fs_ext4,fs_perms_simple,fs_readonly\n[kernel/\t\t\t\t\t\tsyscalls,controllers,pty,containers,admin_tools,modules,can\n[proc]\t\t\t\t\t\tipc,hyperthreading,nptl,cap_bounds,connectors,pipes\n\n\nltp_output.log contains the screenshot of complete ltp execution and ltp_parser.log contains the information regarding the number of tests executed and among them which all have passed failed or skipped.\n\nFor more information regarding a particular category please see ltp_<category>_output.log which contains the output screen and parser log for that particular category' >> %s"
% (readme_file),
shell=True)
# for each command in run config file, read the config for the benchmark
for i in range(0, len(sections_run)):
flag = 0
try:
category = configRun.get(sections_run[i], 'category')
scores_way = configRun.get(sections_run[i], 'scores_way')
parser = configRun.get(sections_run[i], 'parser')
command = configRun.get(sections_run[i], 'command')
except Exception:
logging.debug("no value for the %s" % sections_run[i])
continue
if bench_name == bench_test:
subsection = sections_run[i].split(" ")[1]
subsection_file = log_bench + "_" + subsection + "_output.log"
if os.path.exists(tmp_parser_file):
os.remove(tmp_parser_file)
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
server_run_command = get_server_command(kind_bench, sections_run[i])
logging.debug("Get the server command is: %s" % server_run_command)
# run the command of the benchmarks
try:
flag = run_kinds_commands(sections_run[i], server_run_command,
tmp_log_file, kind_bench, target,
command)
except Exception, e:
logging.info(e)
crash_handle.main()
if bench_name == bench_test:
xyz = subprocess.call("mv /opt/caliper_nfs/ltp_log/* %s " %
(Folder.exec_dir),
shell=True)
server_utils.file_copy(logfile, tmp_log_file, 'a+')
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
run_flag = server_utils.get_fault_tolerance_config(
'fault_tolerance', 'run_error_continue')
if run_flag == 1:
continue
else:
return result
else:
if bench_name == bench_test:
xyz = subprocess.call("mv /opt/caliper_nfs/ltp_log/* %s " %
(Folder.exec_dir),
shell=True)
if os.path.exists(subsection_file):
server_utils.file_copy(tmp_log_file, subsection_file, 'a+')
server_utils.file_copy(logfile, tmp_log_file, 'a+')
if flag != 1:
logging.info("There is wrong when running the command \"%s\"" %
command)
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
crash_handle.main()
run_flag = server_utils.get_fault_tolerance_config(
'fault_tolerance', 'run_error_continue')
if run_flag == 1:
if bench_name != bench_test:
continue
else:
return result
# parser the result in the tmp_log_file, the result is the output of
# running the command
try:
logging.debug("Parsering the result of command: %s" % command)
if bench_name == bench_test:
outfp = open(tmp_parser_file, "w")
outfp.write("%s" % (subsection))
outfp.close()
parser_result = parser_case(kind_bench, bench_name,
parser_file, parser,
subsection_file, tmp_parser_file)
else:
parser_result = parser_case(kind_bench, bench_name,
parser_file, parser, tmp_log_file,
tmp_parser_file)
except Exception, e:
logging.info(
"There's wrong when parsering the result of \" %s \"" %
sections_run[i])
logging.info(e)
if os.path.exists(tmp_parser_file):
os.remove(tmp_parser_file)
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
msg = "Caliper does not support this kind of arch machine"
raise error.ServUnsupportedArchError(msg)
def get_host_name(host):
try:
arch_result = host.run("/bin/uname -a")
except error.CmdError, e:
raise error.ServRunError(e.args[0], e.args[1])
else:
returncode = arch_result.exit_status
if returncode == 0:
output = arch_result.stdout
try:
machine_name = settings.get_value('TARGET',
'Platform_name',
type=str)
except:
machine_name = output.split(" ")[1]
return machine_name
else:
msg = "Caliper does not support this kind of arch machine"
raise error.ServUnsupportedArchError(msg)
def get_host_hardware_info(host):
hardware_info = {}
try:
cpu_type = host.run("grep 'model name' /proc/cpuinfo |uniq \
|awk -F : '{print $2}' |sed 's/^[ \t]*//g'\
|sed 's/ \+/ /g'")
lscpu_list = [
'Architecture', 'Socket\(s\)', 'Cpu_Type', 'Core\(s\) per socket',
'Thread\(s\) per core', 'Model name', 'CPU\(s\)', 'NUMA node\(s\)',
'BogoMIPS', 'Byte Order'
]
lsb_release_list = ['Distributor ID', 'Description', 'Release', 'Codename']
lspci_list = ['Ethernet controller [0200]']
def get_remote_host():
try:
client_ip = settings.get_value('TARGET', 'ip', type=str)
except Exception, e:
client_ip = '127.0.0.1'
try:
port = settings.get_value('TARGET', 'port', type=int)
except Exception, e:
port = 22
try:
user = settings.get_value('TARGET', 'user', type=str)
except Exception, e:
user = os.getlogin()
try:
password = settings.get_value('TARGET', 'password', type=str)
except Exception, e:
raise error.ServRunError(e.args[0], e.args[1])
remote_host = host_factory.create_host(client_ip, user, password, port)
return remote_host
def get_remote_host():
try:
client_ip = settings.get_value('TARGET', 'ip', type=str)
except Exception, e:
client_ip = '127.0.0.1'
else:
msg = "Caliper does not support this kind of arch machine"
raise error.ServUnsupportedArchError(msg)
def get_host_name(host):
try:
arch_result = host.run("/bin/uname -a")
except error.CmdError, e:
raise error.ServRunError(e.args[0], e.args[1])
else:
returncode = arch_result.exit_status
if returncode == 0:
output = arch_result.stdout
try:
machine_name = settings.get_value('CLIENT', 'Platform_name', type=str)
except:
machine_name = output.split(" ")[1]
return machine_name
else:
msg = "Caliper does not support this kind of arch machine"
raise error.ServUnsupportedArchError(msg)
def get_host_hardware_info(host):
hardware_info = {}
try:
cpu_type = host.run("grep 'model name' /proc/cpuinfo |uniq \
|awk -F : '{print $2}' |sed 's/^[ \t]*//g'\
|sed 's/ \+/ /g'")
logic_cpu = host.run("grep 'processor' /proc/cpuinfo |sort |uniq \
def caliper_run(target_exec_dir, server, target):
# get the test cases defined files
config_files = server_utils.get_cases_def_files(target_exec_dir)
logging.debug("the selected configuration are %s" % config_files)
for i in range(0, len(config_files)):
# run benchmarks selected in each configuration file
# config_file = os.path.join(caliper_path.CALIPER_PRE, config_files[i])
config_file = os.path.join(config_files[i])
config, sections = server_utils.read_config_file(config_file)
logging.debug(sections)
# get if it is the 'common' or 'arm' or 'android'
classify = config_files[i].split("/")[-1].strip().split("_")[0]
logging.debug(classify)
if classify == "server" and server:
try:
server_ip = settings.get_value("SERVER", "ip", type=str)
server_port = settings.get_value("SERVER", "port", type=int)
server_user = settings.get_value("SERVER", "user", type=str)
logging.info(
"Please wait while caliper triggers the server.py script in the server"
)
server_pwd = server.run("pwd").stdout
server_pwd = server_pwd.split("\n")[0]
server_caliper_dir = os.path.join(server_pwd, "caliper_server")
server_caliper_dir = os.path.join(server_caliper_dir,
"server.py")
server_user = server_user + '@' + server_ip
script = server_caliper_dir + ' ' + str(server_port)
subprocess.Popen(
['ssh', '%s' % server_user,
'python %s' % script])
except Exception as e:
logging.info(e)
raise AttributeError(
"Error in establising connection with server")
for i in range(0, len(sections)):
# run for each benchmark
target_arch = server_utils.get_host_arch(target)
build_name = sections[i] + '_' + target_arch + '.suc'
build_suc = os.path.join(Folder.build_dir, build_name)
if not os.path.exists(build_suc):
continue
build_host_name = sections[i] + '_' + \
server_utils.get_local_machine_arch() + '.fail'
if os.path.exists(build_host_name):
continue
# try to resolve the configuration of the configuration file
try:
run_file = config.get(sections[i], 'run')
parser = config.get(sections[i], 'parser')
except Exception:
raise AttributeError("The is no option value of parser")
print_format()
logging.info("Running %s" % sections[i])
bench = os.path.join(classify, sections[i])
try:
system_initialise(target)
if classify == "server":
logging.info("Waiting for server to grant access")
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((server_ip, server_port))
logging.info("%s" % str(sock.recv(1024)))
result = run_all_cases(target_exec_dir, target, bench,
sections[i], run_file)
if classify == "server":
sock.send("1")
sock.close()
except Exception:
logging.info("Running %s Exception" % sections[i])
crash_handle.main()
print_format()
if sections[i] == "ltp":
try:
unmount = target.run(
"if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/; fi"
)
except Exception:
unmount = target.run(
"if df -h |grep caliper_nfs ; then fuser -km /mnt/caliper_nfs ;fi"
)
unmount = target.run(
"if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/ ;fi"
)
run_flag = server_utils.get_fault_tolerance_config(
'fault_tolerance', 'run_error_continue')
if run_flag == 1:
continue
else:
return result
else:
logging.info("Running %s Finished" % sections[i])
if sections[i] == "ltp":
try:
unmount = target.run(
"if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/ ;fi"
)
except Exception:
unmount = target.run(
"if df -h |grep caliper_nfs ; then fuser -km /mnt/caliper_nfs/ ;fi"
)
unmount = target.run(
"if df -h |grep caliper_nfs ; then umount /mnt/caliper_nfs/ ;fi"
)
print_format()
return 0
#
import os
import logging
import tempfile
import socket
import glob
import shutil
from caliper.client.shared import error, autotemp
from caliper.server import utils
from caliper.server.hosts import remote_host
from caliper.client.shared.settings import settings
enable_master_ssh = settings.get_value('SERVER',
'enable_master_ssh',
type=bool,
default=False)
def _make_ssh_cmd_default(user="******",
port=22,
opts='',
hosts_file='/dev/null',
connect_timeout=30,
alive_interval=300):
base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no "
"-o UserKnownHostsFile=%s -o BatchMode=yes "
"-o ConnectTimeout=%d -o ServerAliveInterval=%d "
"-l %s -p %d")
assert isinstance(connect_timeout, (int, long))
assert connect_timeout > 0
def get_client_autodir_paths(cls, host):
return settings.get_value('auto', 'client_autodir_paths', type=list)
def run_all_cases(target_exec_dir, target, kind_bench, bench_name,
run_file):
"""
function: run one benchmark which was selected in the configuration files
"""
try:
# get the abspath, which is filename of run config for the benchmark
bench_conf_file = os.path.join(
caliper_path.config_files.tests_cfg_dir,
kind_bench, run_file)
# get the config sections for the benchmrk
configRun, sections_run = server_utils.read_config_file(
bench_conf_file)
except AttributeError as e:
raise AttributeError
except Exception:
raise
logging.debug("the sections to run are: %s" % sections_run)
if not os.path.exists(Folder.exec_dir):
os.mkdir(Folder.exec_dir)
log_bench = os.path.join(Folder.exec_dir, bench_name)
logfile = log_bench + "_output.log"
tmp_log_file = log_bench + "_output_tmp.log"
if os.path.exists(logfile):
os.remove(logfile)
starttime = datetime.datetime.now()
if os.path.exists(Folder.caliper_log_file):
sections = bench_name + " EXECUTION"
fp = open(Folder.caliper_log_file,"r")
f = fp.readlines()
fp.close()
op = open(Folder.caliper_log_file,"w")
for line in f:
if not(sections in line):
op.write(line)
op.close()
result = subprocess.call("echo '$$ %s EXECUTION START: %s' >> %s"
% (bench_name,
str(starttime)[:19],
Folder.caliper_log_file),
shell=True)
bench_test = "ltp"
if bench_name == bench_test:
tar_ip = settings.get_value('CLIENT', 'ip', type=str)
target.run("if [[ ! -e /mnt/caliper_nfs ]]; then mkdir -p /mnt/caliper_nfs; fi")
# fix me , now that we create the folder, why not we mount it directly here
try:
tar_mask = ".".join(tar_ip.split(".")[0:3])
p1 = subprocess.Popen(["ifconfig"], stdout=subprocess.PIPE)
p2 = subprocess.Popen(["grep", tar_mask], stdin=p1.stdout, stdout=subprocess.PIPE)
p1.stdout.close()
output,err = p2.communicate()
output = output.strip()
host_ip = output.split("inet addr:")[1].split(" ")[0]
except Exception:
logging.debug("Unable to get the host_ip" )
try:
mount_cmd = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" % (host_ip) )
except Exception:
try:
umount_cmd = target.run("umount /mnt/caliper_nfs/")
mount_cmd = target.run("mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" % (host_ip) )
except Exception:
logging.debug("Unable to mount")
return result
readme_file=log_bench+"_README"
resultltp = subprocess.call("touch %s"
%(readme_file),shell=True)
resultltp = subprocess.call("echo 'The categorization of ltp in caliper is\nCATEGORY\t\t\t\t\t\tSCENARIOS OF LTP\n\n[command]\t\t\t\t\t\tcommands\n[cpu]\t\t\t\t\t\tsched,cpuhotplug\n[memory]\t\t\t\t\t\tmm.numa,hugetlb\n[dio]\t\t\t\t\t\tdio,io,dma_thread_diotest,timers\n[filesystem]\t\t\t\t\t\tfilecaps,fs,fs_bind,fs_ext4,fs_perms_simple,fs_readonly\n[kernel/\t\t\t\t\t\tsyscalls,controllers,pty,containers,admin_tools,modules,can\n[proc]\t\t\t\t\t\tipc,hyperthreading,nptl,cap_bounds,connectors,pipes\n\n\nltp_output.log contains the screenshot of complete ltp execution and ltp_parser.log contains the information regarding the number of tests executed and among them which all have passed failed or skipped.\n\nFor more information regarding a particular category please see ltp_<category>_output.log which contains the output screen and parser log for that particular category' >> %s"
%(readme_file),shell=True)
# for each command in run config file, read the config for the benchmark
for i in range(0, len(sections_run)):
flag = 0
try:
category = configRun.get(sections_run[i], 'category')
command = configRun.get(sections_run[i], 'command')
except Exception:
logging.debug("no value for the %s" % sections_run[i])
continue
if bench_name == bench_test:
subsection = sections_run[i].split(" ")[1]
subsection_file = log_bench + "_" + subsection + "_output.log"
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
server_run_command = get_server_command(kind_bench, sections_run[i])
logging.debug("Get the server command is: %s" % server_run_command)
# run the command of the benchmarks
try:
flag = run_kinds_commands(sections_run[i], server_run_command,
tmp_log_file, kind_bench,
target, command)
except Exception, e:
logging.info(e)
crash_handle.main()
if bench_name == bench_test:
move_logs = subprocess.call("cp /opt/caliper_nfs/ltp_log/* %s "
% (Folder.exec_dir), shell=True)
server_utils.file_copy(logfile, tmp_log_file, 'a+')
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
run_flag = server_utils.get_fault_tolerance_config(
'fault_tolerance', 'run_error_continue')
if run_flag == 1:
continue
else:
return result
else:
if bench_name == bench_test:
move_logs = subprocess.call("cp /opt/caliper_nfs/ltp_log/* %s "
% (Folder.exec_dir), shell=True)
if os.path.exists(subsection_file):
server_utils.file_copy(tmp_log_file,subsection_file, 'a+')
server_utils.file_copy(logfile, tmp_log_file, 'a+')
if flag != 1:
logging.info("There is wrong when running the command \"%s\""
% command)
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
crash_handle.main()
run_flag = server_utils.get_fault_tolerance_config(
'fault_tolerance', 'run_error_continue')
if run_flag == 1:
if bench_name != bench_test:
continue
else:
return result
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
msg = "Caliper does not support this kind of arch machine"
raise error.ServUnsupportedArchError(msg)
def get_host_name(host):
try:
arch_result = host.run("/bin/uname -a")
except error.CmdError, e:
raise error.ServRunError(e.args[0], e.args[1])
else:
returncode = arch_result.exit_status
if returncode == 0:
output = arch_result.stdout
try:
machine_name = settings.get_value('CLIENT',
'Platform_name',
type=str)
except:
machine_name = output.split(" ")[1]
return machine_name
else:
msg = "Caliper does not support this kind of arch machine"
raise error.ServUnsupportedArchError(msg)
def get_host_hardware_info(host):
hardware_info = {}
try:
cpu_type = host.run("grep 'model name' /proc/cpuinfo |uniq \
|awk -F : '{print $2}' |sed 's/^[ \t]*//g'\
|sed 's/ \+/ /g'")
from caliper.server.hosts import host_factory
from caliper.client.shared import error
from caliper.client.shared.settings import settings
from caliper.client.shared.caliper_path import folder_ope as Folder
path = "./HardwareInfo"
out_path = "./hardware_yaml"
lscpu_list = ['Architecture','Socket\(s\)','Cpu_Type','Core\(s\) per socket','Thread\(s\) per core','Model name','CPU\(s\)','NUMA node\(s\)','BogoMIPS','Byte Order']
lsb_release_list = ['Distributor ID','Description','Release','Codename']
lspci_list = ['Ethernet controller [0200]']
def get_remote_host():
try:
client_ip = settings.get_value('CLIENT', 'ip', type=str)
except Exception, e:
client_ip = '127.0.0.1'
try:
port = settings.get_value('CLIENT', 'port', type=int)
except Exception, e:
port = 22
try:
user = settings.get_value('CLIENT', 'user', type=str)
except Exception, e:
user = os.getlogin()
try:
password = settings.get_value('CLIENT', 'password', type=str)
except Exception, e:
raise error.ServRunError(e.args[0], e.args[1])
remote_host = host_factory.create_host(client_ip, user, password, port)
return remote_host
def network_populate(dic,contents):
def get_remote_host():
try:
client_ip = settings.get_value('CLIENT', 'ip', type=str)
except Exception, e:
client_ip = '127.0.0.1'
#
import os
import logging
import tempfile
import socket
import glob
import shutil
from caliper.client.shared import error, autotemp
from caliper.server import utils
from caliper.server.hosts import remote_host
from caliper.client.shared.settings import settings
enable_master_ssh = settings.get_value('SERVER', 'enable_master_ssh',
type=bool, default=False)
def _make_ssh_cmd_default(user="******", port=22, opts='',
hosts_file='/dev/null', connect_timeout=30,
alive_interval=300):
base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no "
"-o UserKnownHostsFile=%s -o BatchMode=yes "
"-o ConnectTimeout=%d -o ServerAliveInterval=%d "
"-l %s -p %d")
assert isinstance(connect_timeout, (int, long))
assert connect_timeout > 0
if isinstance(port, str):
port = int(port)
return base_command % (opts, hosts_file,
connect_timeout, alive_interval, user, port)
def run_all_cases(target_exec_dir, target, kind_bench, bench_name, run_file):
"""
function: run one benchmark which was selected in the configuration files
"""
try:
# get the abspath, which is filename of run config for the benchmark
bench_conf_file = os.path.join(caliper_path.config_files.tests_cfg_dir,
kind_bench, run_file)
# get the config sections for the benchmrk
configRun, sections_run = server_utils.read_config_file(
bench_conf_file)
except AttributeError as e:
raise AttributeError
except Exception:
raise
logging.debug("the sections to run are: %s" % sections_run)
if not os.path.exists(Folder.exec_dir):
os.mkdir(Folder.exec_dir)
log_bench = os.path.join(Folder.exec_dir, bench_name)
logfile = log_bench + "_output.log"
tmp_log_file = log_bench + "_output_tmp.log"
if os.path.exists(logfile):
os.remove(logfile)
starttime = datetime.datetime.now()
if os.path.exists(Folder.caliper_log_file):
sections = bench_name + " EXECUTION"
fp = open(Folder.caliper_log_file, "r")
f = fp.readlines()
fp.close()
op = open(Folder.caliper_log_file, "w")
for line in f:
if not (sections in line):
op.write(line)
op.close()
result = subprocess.call(
"echo '$$ %s EXECUTION START: %s' >> %s" %
(bench_name, str(starttime)[:19], Folder.caliper_log_file),
shell=True)
bench_test = "ltp"
if bench_name == bench_test:
tar_ip = settings.get_value('CLIENT', 'ip', type=str)
target.run(
"if [[ ! -e /mnt/caliper_nfs ]]; then mkdir -p /mnt/caliper_nfs; fi"
)
# fix me , now that we create the folder, why not we mount it directly here
try:
tar_mask = ".".join(tar_ip.split(".")[0:3])
p1 = subprocess.Popen(["ifconfig"], stdout=subprocess.PIPE)
p2 = subprocess.Popen(["grep", tar_mask],
stdin=p1.stdout,
stdout=subprocess.PIPE)
p1.stdout.close()
output, err = p2.communicate()
output = output.strip()
host_ip = output.split("inet addr:")[1].split(" ")[0]
except Exception:
logging.debug("Unable to get the host_ip")
try:
mount_cmd = target.run(
"mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" %
(host_ip))
except Exception:
try:
umount_cmd = target.run("umount /mnt/caliper_nfs/")
mount_cmd = target.run(
"mount -t nfs %s:/opt/caliper_nfs /mnt/caliper_nfs" %
(host_ip))
except Exception:
logging.debug("Unable to mount")
return result
readme_file = log_bench + "_README"
resultltp = subprocess.call("touch %s" % (readme_file), shell=True)
resultltp = subprocess.call(
"echo 'The categorization of ltp in caliper is\nCATEGORY\t\t\t\t\t\tSCENARIOS OF LTP\n\n[command]\t\t\t\t\t\tcommands\n[cpu]\t\t\t\t\t\tsched,cpuhotplug\n[memory]\t\t\t\t\t\tmm.numa,hugetlb\n[dio]\t\t\t\t\t\tdio,io,dma_thread_diotest,timers\n[filesystem]\t\t\t\t\t\tfilecaps,fs,fs_bind,fs_ext4,fs_perms_simple,fs_readonly\n[kernel/\t\t\t\t\t\tsyscalls,controllers,pty,containers,admin_tools,modules,can\n[proc]\t\t\t\t\t\tipc,hyperthreading,nptl,cap_bounds,connectors,pipes\n\n\nltp_output.log contains the screenshot of complete ltp execution and ltp_parser.log contains the information regarding the number of tests executed and among them which all have passed failed or skipped.\n\nFor more information regarding a particular category please see ltp_<category>_output.log which contains the output screen and parser log for that particular category' >> %s"
% (readme_file),
shell=True)
# for each command in run config file, read the config for the benchmark
for i in range(0, len(sections_run)):
flag = 0
try:
category = configRun.get(sections_run[i], 'category')
command = configRun.get(sections_run[i], 'command')
except Exception:
logging.debug("no value for the %s" % sections_run[i])
continue
if bench_name == bench_test:
subsection = sections_run[i].split(" ")[1]
subsection_file = log_bench + "_" + subsection + "_output.log"
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
server_run_command = get_server_command(kind_bench, sections_run[i])
logging.debug("Get the server command is: %s" % server_run_command)
# run the command of the benchmarks
try:
flag = run_kinds_commands(sections_run[i], server_run_command,
tmp_log_file, kind_bench, target,
command)
except Exception, e:
logging.info(e)
crash_handle.main()
if bench_name == bench_test:
move_logs = subprocess.call(
"cp /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir),
shell=True)
server_utils.file_copy(logfile, tmp_log_file, 'a+')
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
run_flag = server_utils.get_fault_tolerance_config(
'fault_tolerance', 'run_error_continue')
if run_flag == 1:
continue
else:
return result
else:
if bench_name == bench_test:
move_logs = subprocess.call(
"cp /opt/caliper_nfs/ltp_log/* %s " % (Folder.exec_dir),
shell=True)
if os.path.exists(subsection_file):
server_utils.file_copy(tmp_log_file, subsection_file, 'a+')
server_utils.file_copy(logfile, tmp_log_file, 'a+')
if flag != 1:
logging.info("There is wrong when running the command \"%s\"" %
command)
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
crash_handle.main()
run_flag = server_utils.get_fault_tolerance_config(
'fault_tolerance', 'run_error_continue')
if run_flag == 1:
if bench_name != bench_test:
continue
else:
return result
if os.path.exists(tmp_log_file):
os.remove(tmp_log_file)
break
if not server_ip:
if len(server_ips) > 1:
try:
server_ips.remove("127.0.0.1")
except Exception:
raise e
server_ip = server_ips[0]
strinfo = re.compile('\$SERVER_IP')
post_commands = strinfo.sub(server_ip, commands)
commands = post_commands
if re.findall('\$CLIENT_IP', commands):
try:
client_ip = settings.get_value('CLIENT', 'ip', type=str)
except Exception, e:
client_ip = '127.0.0.1'
strinfo = re.compile('\$CLIENT_IP')
post_commands = strinfo.sub(client_ip, commands)
commands = post_commands
if commands[0] == '\'' and commands[-1] == '\'':
actual_commands = commands[1:-1]
elif commands[0] == '\"' and commands[-1] == '\"':
actual_commands = commands[1:-1]
else:
actual_commands = commands
if actual_commands == '':
return ''
return actual_commands
# -*- coding:utf-8 -*-
import os
import logging
import tempfile
import socket
import glob
import shutil
from caliper.client.shared import error, autotemp
from caliper.server import utils
from caliper.server.hosts import remote_host
from caliper.client.shared.settings import settings
enable_master_ssh = settings.get_value('TestNode',
'enable_master_ssh',
type=bool,
default=False)
def _make_ssh_cmd_default(user="******",
port=22,
opts='',
hosts_file='/dev/null',
connect_timeout=30,
alive_interval=300):
base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no "
"-o UserKnownHostsFile=%s -o BatchMode=yes "
"-o ConnectTimeout=%d -o ServerAliveInterval=%d "
"-l %s -p %d")
assert isinstance(connect_timeout, (int, long))
assert connect_timeout > 0
break
if not server_ip:
if len(server_ips) > 1:
try:
server_ips.remove("127.0.0.1")
except Exception:
raise e
server_ip = server_ips[0]
strinfo = re.compile('\$SERVER_IP')
post_commands = strinfo.sub(server_ip, commands)
commands = post_commands
if re.findall('\$CLIENT_IP', commands):
try:
client_ip = settings.get_value('CLIENT', 'ip', type=str)
except Exception, e:
client_ip = '127.0.0.1'
strinfo = re.compile('\$CLIENT_IP')
post_commands = strinfo.sub(client_ip, commands)
commands = post_commands
if commands[0] == '\'' and commands[-1] == '\'':
actual_commands = commands[1:-1]
elif commands[0] == '\"' and commands[-1] == '\"':
actual_commands = commands[1:-1]
else:
actual_commands = commands
if actual_commands == '':
return ''
return actual_commands
