def get_actual_commands(commands, target):
if commands is None or commands == '':
return None
post_commands = commands
if re.findall('\$SERVER_IP', commands):
try:
server_ip = settings.get_value('SERVER', 'ip', type=str)
except Exception, e:
server_ips = server_utils.get_local_ip()
server_ip = ""
for each_ip in server_ips:
items = each_ip.split(".")[0:2]
pre = '.'.join(items)
if target.ip.startswith(pre):
server_ip = each_ip
break
if not server_ip:
if len(server_ips) > 1:
try:
server_ips.remove("127.0.0.1")
except Exception:
raise e
server_ip = server_ips[0]
strinfo = re.compile('\$SERVER_IP')
post_commands = strinfo.sub(server_ip, commands)
def get_actual_commands(commands, target):
if commands is None or commands == '':
return None
post_commands = commands
if re.findall('\$SERVER_IP', commands):
try:
server_ip = settings.get_value('SERVER', 'ip', type=str)
except Exception, e:
server_ips = server_utils.get_local_ip()
server_ip = ""
for each_ip in server_ips:
items = each_ip.split(".")[0:2]
pre = '.'.join(items)
if target.ip.startswith(pre):
server_ip = each_ip
break
if not server_ip:
if len(server_ips) > 1:
try:
server_ips.remove("127.0.0.1")
except Exception:
raise e
server_ip = server_ips[0]
strinfo = re.compile('\$SERVER_IP')
post_commands = strinfo.sub(server_ip, commands)
def build_for_target(target):
if server_utils.get_target_ip(target) == server_utils.get_local_ip():
return
target_arch = server_utils.get_host_arch(target)
try:
build_caliper(target_arch)
except Exception, e:
raise
def build_for_target(target,f_option,clear):
#f_option is set if -f is used
# Create the temperory build folders
GEN_DIR = caliper_path.GEN_DIR
WS_GEN_DIR = os.path.join(FOLDER.workspace, 'binary')
benchs_dir = os.path.join(TMP_DIR, 'benchmarks')
if not os.path.exists(benchs_dir):
try:
os.makedirs(benchs_dir, 0755)
except Exception:
os.mkdir(benchs_dir, 0755)
# Improvement Point: Right now all the benchmarks are copied, we can only
# copy the selected benchmarks to save the time.
if os.path.exists(benchs_dir):
shutil.rmtree(benchs_dir)
shutil.copytree(caliper_path.BENCHS_DIR, benchs_dir)
if not os.path.exists(caliper_path.FRONT_END_DIR):
shutil.copytree(caliper_path.FRONT_TMP_DIR,
caliper_path.FRONT_END_DIR)
if f_option == 0:
if os.path.exists(FOLDER.caliper_log_file):
os.remove(FOLDER.caliper_log_file)
if os.path.exists(FOLDER.summary_file):
os.remove(FOLDER.summary_file)
if not os.path.exists(FOLDER.build_dir):
create_folder(FOLDER.build_dir)
if not os.path.exists(FOLDER.exec_dir):
create_folder(FOLDER.exec_dir)
if not os.path.exists(FOLDER.results_dir):
create_folder(FOLDER.results_dir)
if not os.path.exists(FOLDER.yaml_dir):
create_folder(FOLDER.yaml_dir)
if not os.path.exists(FOLDER.html_dir):
create_folder(FOLDER.html_dir)
if not os.path.exists(caliper_path.HTML_DATA_DIR_INPUT):
os.makedirs(caliper_path.HTML_DATA_DIR_INPUT)
if not os.path.exists(caliper_path.HTML_DATA_DIR_OUTPUT):
os.makedirs(caliper_path.HTML_DATA_DIR_OUTPUT)
if server_utils.get_target_ip(target) in server_utils.get_local_ip():
return build_for_local()
try:
host_arch = server_utils.get_local_machine_arch()
except Exception, e:
logging.debug(" Error in get_local_machine_arch()")
raise e
def run_client_command(cmd_sec_name, tmp_logfile, kind_bench,
target, command):
fp = open(tmp_logfile, "a+")
start_log = "%%%%%% %s test start %%%%%% \n" % cmd_sec_name
fp.write(start_log)
fp.write("<<<BEGIN TEST>>>\n")
tags = "[test: " + cmd_sec_name + "]\n"
fp.write(tags)
logs = "log: " + get_actual_commands(command, target) + "\n"
fp.write(logs)
start = time.time()
flag = 0
logging.debug("the client running command is %s" % command)
# get the execution location in the remote host
is_localhost = 0
if server_utils.get_target_ip(target) in server_utils.get_local_ip():
is_localhost = 1
if (is_localhost == 1):
arch = server_utils.get_local_machine_arch()
host_exec_dir = os.path.join(caliper_path.GEN_DIR, arch)
else:
host_current_pwd = target.run("pwd").stdout.split("\n")[0]
arch = server_utils.get_host_arch(target)
host_exec_dir = os.path.join(host_current_pwd, 'caliper',
"binary", arch)
try:
logging.debug("begining to execute the command of %s on remote host"
% command)
if (is_localhost == 1):
logging.debug("client command in localhost is: %s" % command)
[out, returncode] = run_commands(host_exec_dir, kind_bench,
command, fp, fp)
else:
logging.debug("client command in localhost is: %s" % command)
[out, returncode] = run_remote_commands(host_exec_dir, kind_bench,
command, target, fp, fp)
except error.ServRunError, e:
fp.write("[status]: FAIL\n")
sys.stdout.write(e)
flag = -1
def run(self):
global GEN_DIR, BUILD_MAPPING_FILE, BUILD_MAPPING_DIR
GEN_DIR = caliper_path.GEN_DIR
if self.target_arch:
arch = self.target_arch
else:
arch = 'x86_64'
# get the config file
case_file = os.path.join(TEST_CASE_DIR, 'cases_config.json')
fp = open(case_file, 'r')
case_list = json.load(fp)
BUILD_MAPPING_DIR = os.path.join(BUILD_MAPPING_DIR, arch)
if not os.path.exists(BUILD_MAPPING_DIR):
try:
os.makedirs(BUILD_MAPPING_DIR)
except:
pass
# set_signals()
# check and delete those binaries if it is already built if -c is used
if self.clear:
logging.info("=" * 55)
logging.info(
"WARNING: Please wait, dont run any other instance of caliper")
for section in self.sections:
BUILD_MAPPING_FILE = os.path.join(BUILD_MAPPING_DIR,
section + '.yaml')
with client_utils.SimpleFlock(BUILD_MAPPING_FILE, 60):
fp = open(BUILD_MAPPING_FILE)
dic = yaml.load(fp)
fp.close()
if type(dic) != dict:
dic = {}
if section in dic.keys():
for file in dic[section]['binaries']:
try:
shutil.rmtree(file)
except:
pass
dic[section]['binaries'] = []
dic[section]['ProcessID'] = 0
fp = open(BUILD_MAPPING_FILE, 'w')
fp.write(yaml.dump(dic, default_flow_style=False))
fp.close()
logging.info("It is safe to run caliper now")
logging.info("=" * 55)
for section in self.sections:
BUILD = 0
BUILD_MAPPING_FILE = os.path.join(BUILD_MAPPING_DIR,
section + '.yaml')
reset_binary_mapping()
try:
# Lock the file and modify it if this is the first process which is building the tool
with client_utils.SimpleFlock(BUILD_MAPPING_FILE, 60):
fp = open(BUILD_MAPPING_FILE)
dic = yaml.load(fp)
if type(dic) != dict:
dic = {}
fp.close()
if section not in dic.keys():
dic[section] = {}
dic[section]['binaries'] = []
dic[section]['ProcessID'] = os.getpid()
BUILD = 1
fp = open(BUILD_MAPPING_FILE, 'w')
fp.write(yaml.dump(dic, default_flow_style=False))
fp.close()
# checking if binary field is empty, empty means that the previous build is a failure
if not dic[section]['binaries']:
BUILD = 1
except Exception as e:
logging.debug(e)
sys.exit(1)
BUILD = 1
if BUILD == 1:
logging.info("=" * 55)
logging.info("Building %s" % section)
build_dir = os.path.join(caliper_path.BENCHS_DIR, section,
'tests')
build_config = os.path.join(TEST_CASE_DIR,
'project_config.cfg')
log_name = "%s.log" % section
log_file = os.path.join('/tmp', log_name)
if not os.path.exists(build_dir):
download_section(section)
os.chdir(build_dir)
try:
section_build_path = os.path.join(
caliper_path.CALIPER_TMP_DIR, 'binary', section)
run_path = os.path.join('/tmp', section)
if caliper_path.client_ip in server_utils.get_local_ip(
) and os.path.exists(section_build_path):
result = subprocess.call(
"echo '$$ %s build Successful' >> %s" %
(section, log_file),
shell=True)
if not os.path.exists(run_path):
shutil.copytree(section_build_path, run_path)
logging.info("%s is already build" % section)
continue
else:
result = subprocess.call(
'ansible-playbook -i %s site.yml --extra-vars "hosts=%s" -u %s>> %s 2>&1'
% (build_config, self.host, getpass.getuser(),
log_file),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
except Exception as e:
result = e
else:
if caliper_path.client_ip in server_utils.get_local_ip():
try:
if not os.path.exists(section_build_path):
shutil.copytree(run_path, section_build_path)
except Exception, e:
pass
if result:
logging.info("Building %s Failed" % section)
logging.info("=" * 55)
record_log(log_file, arch, 0)
else:
logging.info("Building %s Successful" % section)
logging.info("=" * 55)
record_log(log_file, arch, 1)