def vm_check(hostname, host_ip, distro, internet):
"""
Check whether virtual machine is up and fine
"""
# pylint: disable=too-many-return-statements
vm_host = ssh_host.SSHHost(host_ip)
command = "hostname"
retval = vm_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
host_ip,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
current_hostname = retval.cr_stdout.strip()
if current_hostname != hostname:
logging.error("wrong host name of the virtual machine [%s], expected "
"[%s], got [%s]", host_ip, hostname, current_hostname)
return -1
vm_host = ssh_host.SSHHost(hostname)
command = "hostname"
retval = vm_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
current_hostname = retval.cr_stdout.strip()
if current_hostname != hostname:
logging.error("wrong host name of the virtual machine [%s], expected "
"[%s], got [%s]", hostname, hostname, current_hostname)
return -1
if vm_host.sh_distro() != distro:
logging.error("wrong distro of the virtual machine [%s], expected "
"[%s], got [%s]", hostname, distro, vm_host.sh_distro())
return -1
if internet:
if vm_host.sh_check_internet():
logging.error("virtual machine [%s] can not access Internet",
hostname)
return -1
return 0
def main():
"""
Install Exascaler monitoring
"""
# pylint: disable=unused-variable,too-many-branches
argc = len(sys.argv)
if argc == 2:
if sys.argv[1] == "-h" or sys.argv[1] == "--help":
usage()
sys.exit(0)
else:
utils.eprint("Unkown options \"%s\"" % sys.argv[1])
usage()
sys.exit(-1)
elif argc > 2:
utils.eprint("too many options")
usage()
sys.exit(-1)
local_host = ssh_host.SSHHost("localhost", local=True)
missing_dependencies = dependency_find(local_host)
if len(missing_dependencies):
ret = dependency_install(local_host)
if ret:
sys.exit(-1)
from pyesmon import esmon_install_nodeps
esmon_install_nodeps.main()
def iso_path_in_config(local_host):
"""
Return the ISO path in the config file
"""
local_host = ssh_host.SSHHost("localhost", local=True)
command = (r"grep -v ^\# /etc/esmon_install.conf | "
"grep ^iso_path: | awk '{print $2}'")
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on localhost, "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
retval.cr_exit_status, retval.cr_stdout, retval.cr_stderr)
return None
lines = retval.cr_stdout.splitlines()
if len(lines) != 1:
logging.error("unexpected iso path in config file: %s", lines)
return None
return lines[0]
def parse_host_configs(config, config_fpath, hosts):
"""
Parse the host_configs
"""
host_configs = esmon_common.config_value(config, "ssh_hosts")
if host_configs is None:
logging.info("can NOT find [ssh_hosts] in the config file [%s]",
config_fpath)
return 0
for host_config in host_configs:
host_id = host_config["host_id"]
if host_id is None:
logging.error(
"can NOT find [host_id] in the config of a "
"SSH host, please correct file [%s]", config_fpath)
return -1
hostname = esmon_common.config_value(host_config, "hostname")
if hostname is None:
logging.error(
"can NOT find [hostname] in the config of SSH host "
"with ID [%s], please correct file [%s]", host_id,
config_fpath)
return -1
mapping_dict = {esmon_common.ESMON_CONFIG_CSTR_NONE: None}
ssh_identity_file = esmon_common.config_value(
host_config,
esmon_common.CSTR_SSH_IDENTITY_FILE,
mapping_dict=mapping_dict)
if host_id in hosts:
logging.error(
"multiple SSH hosts with the same ID [%s], please "
"correct file [%s]", host_id, config_fpath)
return -1
host = ssh_host.SSHHost(hostname, ssh_identity_file)
hosts[host_id] = host
return 0
def vm_start(workspace, server_host, hostname, network_configs, ips,
template_hostname, image_dir, distro, internet, disk_number):
"""
Start virtual machine, if vm is bad, clone it
"""
# pylint: disable=too-many-arguments
host_ip = ips[0]
ret = vm_check(hostname, host_ip, distro, internet)
if ret == 0:
return 0
if vm_is_shut_off(server_host, hostname):
command = ("virsh start %s" % (hostname))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
vm_host = ssh_host.SSHHost(hostname)
ret = vm_host.sh_wait_up()
if ret == 0:
ret = vm_check(hostname, host_ip, distro, internet)
if ret == 0:
return 0
ret = vm_clone(workspace, server_host, hostname, network_configs, ips,
template_hostname, image_dir, distro, internet, disk_number)
if ret:
logging.error("failed to create virtual machine [%s] based on "
"template [%s]", hostname, template_hostname)
return -1
return 0
def esmon_do_test(workspace, config, config_fpath):
"""
Run the tests
"""
# pylint: disable=too-many-return-statements,too-many-locals
# pylint: disable=too-many-branches,too-many-statements
esmon_virt_config_fpath = esmon_common.config_value(
config, esmon_common.CSTR_ESMON_VIRT)
if esmon_virt_config_fpath is None:
logging.error(
"no [esmon_virt] is configured, "
"please correct file [%s]", config_fpath)
return -1
ret = esmon_virt.esmon_virt(workspace, esmon_virt_config_fpath)
if ret:
logging.error("failed to install the virtual machines")
return -1
ssh_host_configs = esmon_common.config_value(config,
esmon_common.CSTR_SSH_HOSTS)
if ssh_host_configs is None:
logging.error(
"can NOT find [%s] in the config file, "
"please correct file [%s]", esmon_common.CSTR_SSH_HOSTS,
config_fpath)
return -1
hosts = {}
for host_config in ssh_host_configs:
host_id = host_config[esmon_common.CSTR_HOST_ID]
if host_id is None:
logging.error(
"can NOT find [%s] in the config of a "
"SSH host, please correct file [%s]",
esmon_common.CSTR_HOST_ID, config_fpath)
return -1
hostname = esmon_common.config_value(host_config,
esmon_common.CSTR_HOSTNAME)
if hostname is None:
logging.error(
"can NOT find [%s] in the config of SSH host "
"with ID [%s], please correct file [%s]",
esmon_common.CSTR_HOSTNAME, host_id, config_fpath)
return -1
local = esmon_common.config_value(host_config,
esmon_common.CSTR_LOCAL_HOST)
if local is None:
logging.debug(
"can NOT find [%s] in the config of SSH host "
"with ID [%s], use [false] as default value",
esmon_common.CSTR_LOCAL_HOST, host_id)
local = False
mapping_dict = {esmon_common.ESMON_CONFIG_CSTR_NONE: None}
ssh_identity_file = esmon_common.config_value(
host_config,
esmon_common.CSTR_SSH_IDENTITY_FILE,
mapping_dict=mapping_dict)
if host_id in hosts:
logging.error(
"multiple SSH hosts with the same ID [%s], please "
"correct file [%s]", host_id, config_fpath)
return -1
host = ssh_host.SSHHost(hostname, ssh_identity_file, local=local)
hosts[host_id] = host
install_server_hostid = esmon_common.config_value(
config, esmon_common.CSTR_INSTALL_SERVER)
if install_server_hostid is None:
logging.error(
"can NOT find [%s] in the config file [%s], "
"please correct it", esmon_common.CSTR_INSTALL_SERVER,
config_fpath)
return -1
if install_server_hostid not in hosts:
logging.error(
"SSH host with ID [%s] is NOT configured in "
"[ssh_hosts], please correct file [%s]", install_server_hostid,
config_fpath)
return -1
install_server = hosts[install_server_hostid]
collect_interval = esmon_common.config_value(
config, esmon_common.CSTR_COLLECT_INTERVAL)
if collect_interval is None:
logging.error(
"can NOT find [%s] in the config file, "
"please correct file [%s]", esmon_common.CSTR_COLLECT_INTERVAL,
config_fpath)
return -1
continuous_query_periods = \
esmon_common.config_value(config, esmon_common.CSTR_CONTINUOUS_QUERY_PERIODS)
if continuous_query_periods is None:
logging.error(
"can NOT find [%s] in the config file, "
"please correct file [%s]",
esmon_common.CSTR_CONTINUOUS_QUERY_PERIODS, config_fpath)
return -1
lustre_default_version = \
esmon_common.config_value(config, esmon_common.CSTR_LUSTRE_DEFAULT_VERSION)
if lustre_default_version is None:
logging.error("[%s] is not configured, please correct file [%s]",
esmon_common.CSTR_LUSTRE_DEFAULT_VERSION, config_fpath)
return -1
lustre_exp_ost = \
esmon_common.config_value(config, esmon_common.CSTR_LUSTRE_EXP_OST)
if lustre_exp_ost is None:
lustre_exp_ost = False
logging.info(
"[%s] is not configured, setting it to [False] by default",
esmon_common.CSTR_LUSTRE_EXP_OST)
lustre_exp_mdt = \
esmon_common.config_value(config, esmon_common.CSTR_LUSTRE_EXP_MDT)
if lustre_exp_mdt is None:
lustre_exp_mdt = False
logging.info(
"[%s] is not configured, setting it to [False] by default",
esmon_common.CSTR_LUSTRE_EXP_MDT)
server_config = esmon_common.config_value(config, esmon_common.CSTR_SERVER)
if server_config is None:
logging.error(
"can NOT find [%s] in the config file, "
"please correct file [%s]", esmon_common.CSTR_SERVER, config_fpath)
return -1
agent_configs = esmon_common.config_value(config, esmon_common.CSTR_AGENTS)
if agent_configs is None:
logging.error(
"can NOT find [%s] in the config file, "
"please correct file [%s]", esmon_common.CSTR_AGENTS, config_fpath)
return -1
ret, agents_reinstall = \
esmon_config.install_config_value(config,
esmon_common.CSTR_AGENTS_REINSTALL)
if ret:
return -1
local_host = ssh_host.SSHHost("localhost", local=True)
command = "ls esmon-*.iso"
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
current_dir = os.getcwd()
iso_names = retval.cr_stdout.split()
if len(iso_names) != 1:
logging.error(
"found unexpected ISOs [%s] under currect directory [%s]",
iso_names, current_dir)
return -1
iso_name = iso_names[0]
iso_path = current_dir + "/" + iso_name
command = "mkdir -p %s" % workspace
retval = install_server.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
install_server.sh_hostname, retval.cr_exit_status,
retval.cr_stdout, retval.cr_stderr)
return -1
ret = install_server.sh_send_file(iso_path, workspace)
if ret:
logging.error(
"failed to send ESMON ISO [%s] on local host to "
"directory [%s] on host [%s]", iso_path, workspace,
install_server.sh_hostname)
return -1
host_iso_path = workspace + "/" + iso_name
install_config = {}
install_config[esmon_common.CSTR_ISO_PATH] = host_iso_path
install_config[esmon_common.CSTR_SSH_HOSTS] = ssh_host_configs
install_config[esmon_common.CSTR_AGENTS_REINSTALL] = agents_reinstall
install_config[esmon_common.CSTR_AGENTS] = agent_configs
install_config[esmon_common.CSTR_SERVER] = server_config
install_config[esmon_common.CSTR_COLLECT_INTERVAL] = collect_interval
install_config[
esmon_common.CSTR_CONTINUOUS_QUERY_PERIODS] = continuous_query_periods
install_config[
esmon_common.CSTR_LUSTRE_DEFAULT_VERSION] = lustre_default_version
install_config[esmon_common.CSTR_LUSTRE_EXP_OST] = lustre_exp_ost
install_config[esmon_common.CSTR_LUSTRE_EXP_MDT] = lustre_exp_mdt
install_config_string = yaml.dump(install_config, default_flow_style=False)
install_config_fpath = workspace + "/" + esmon_common.ESMON_INSTALL_CONFIG_FNAME
with open(install_config_fpath, "wt") as install_config_file:
install_config_file.write(install_config_string)
skip_install_test = esmon_common.config_value(
config, esmon_common.CSTR_SKIP_INSTALL_TEST)
if skip_install_test is None:
logging.debug("no [%s] is configured, use [false] as default value",
esmon_common.CSTR_SKIP_INSTALL_TEST)
skip_install_test = False
if not skip_install_test:
ret = esmon_test_install(workspace, install_server, host_iso_path)
if ret:
return -1
ret = esmon_test_lustre(workspace, hosts, config, config_fpath,
install_config, install_config_fpath)
if ret:
logging.error("failed to test Lustre")
return -1
return 0
def esmon_io_loading(workspace, config, confpath):
"""
Start the I/O
"""
# pylint: disable=too-many-locals,unused-argument,too-many-return-statements
# pylint: disable=too-many-branches,too-many-statements
ssh_host_configs = esmon_common.config_value(config,
esmon_common.CSTR_SSH_HOSTS)
if ssh_host_configs is None:
logging.error(
"can NOT find [%s] in the config file, "
"please correct file [%s]", esmon_common.CSTR_SSH_HOSTS, confpath)
return -1
hosts = {}
for host_config in ssh_host_configs:
host_id = host_config["host_id"]
if host_id is None:
logging.error(
"can NOT find [host_id] in the config of a "
"SSH host, please correct file [%s]", confpath)
return -1
hostname = esmon_common.config_value(host_config, "hostname")
if hostname is None:
logging.error(
"can NOT find [hostname] in the config of SSH host "
"with ID [%s], please correct file [%s]", host_id, confpath)
return -1
local = esmon_common.config_value(host_config,
esmon_common.CSTR_LOCAL_HOST)
if local is None:
logging.debug(
"can NOT find [%s] in the config of SSH host "
"with ID [%s], use [false] as default value",
esmon_common.CSTR_LOCAL_HOST, host_id)
local = False
mapping_dict = {esmon_common.ESMON_CONFIG_CSTR_NONE: None}
ssh_identity_file = esmon_common.config_value(
host_config,
esmon_common.CSTR_SSH_IDENTITY_FILE,
mapping_dict=mapping_dict)
if host_id in hosts:
logging.error(
"multiple SSH hosts with the same ID [%s], please "
"correct file [%s]", host_id, confpath)
return -1
host = ssh_host.SSHHost(hostname, ssh_identity_file, local=local)
hosts[host_id] = host
# Parse the Lustre client configuration.
lustre_configs = esmon_common.config_value(config,
esmon_common.CSTR_LUSTRES)
if lustre_configs is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_LUSTRES, confpath)
return -1
for lustre_config in lustre_configs:
# Parse general configs of Lustre file system
fsname = esmon_common.config_value(lustre_config,
esmon_common.CSTR_FSNAME)
if fsname is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_FSNAME, confpath)
return -1
lazy_prepare = esmon_common.config_value(
lustre_config, esmon_common.CSTR_LAZY_PREPARE)
if lazy_prepare is None:
lazy_prepare = False
logging.info(
"no [%s] is configured for fs [%s], using default value false",
esmon_common.CSTR_LAZY_PREPARE, fsname)
return -1
lustre_fs = lustre.LustreFilesystem(fsname)
lustre_hosts = {}
# Parse OST configs
ost_configs = esmon_common.config_value(lustre_config,
esmon_common.CSTR_OSTS)
if ost_configs is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_OSTS, confpath)
return -1
for ost_config in ost_configs:
ost_index = esmon_common.config_value(ost_config,
esmon_common.CSTR_INDEX)
if ost_index is None:
logging.error(
"no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_INDEX, confpath)
return -1
host_id = esmon_common.config_value(ost_config,
esmon_common.CSTR_HOST_ID)
if host_id is None:
logging.error(
"no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_HOST_ID, confpath)
return -1
if host_id not in hosts:
logging.error(
"no host with ID [%s] is configured in hosts, "
"please correct file [%s]", host_id, confpath)
return -1
device = esmon_common.config_value(ost_config,
esmon_common.CSTR_DEVICE)
if device is None:
logging.error(
"no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_DEVICE, confpath)
return -1
host = hosts[host_id]
lustre_host = lustre.LustreServerHost(
host.sh_hostname,
identity_file=host.sh_identity_file,
local=host.sh_local,
host_id=host_id)
if host_id not in lustre_hosts:
lustre_hosts[host_id] = lustre_host
mnt = "/mnt/%s_ost_%s" % (fsname, ost_index)
lustre.LustreOST(lustre_fs, ost_index, lustre_host, device, mnt)
# Parse client configs
client_configs = esmon_common.config_value(lustre_config,
esmon_common.CSTR_CLIENTS)
if client_configs is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_CLIENTS, confpath)
return -1
for client_config in client_configs:
host_id = esmon_common.config_value(client_config,
esmon_common.CSTR_HOST_ID)
if host_id is None:
logging.error(
"no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_HOST_ID, confpath)
return -1
if host_id not in hosts:
logging.error(
"no host with [%s] is configured in hosts, "
"please correct file [%s]", host_id, confpath)
return -1
mnt = esmon_common.config_value(client_config,
esmon_common.CSTR_MNT)
if mnt is None:
logging.error(
"no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_MNT, confpath)
return -1
host = hosts[host_id]
lustre_host = lustre.LustreServerHost(
host.sh_hostname,
identity_file=host.sh_identity_file,
local=host.sh_local,
host_id=host_id)
if host_id not in lustre_hosts:
lustre_hosts[host_id] = lustre_host
lustre.LustreClient(lustre_fs, host, mnt)
ret = esmon_launch_ioload_daemon(lustre_fs)
if ret:
return ret
return 0
def esmon_vm_install(workspace, config, config_fpath):
"""
Start to test with ESMON
"""
# pylint: disable=too-many-return-statements,too-many-locals
# pylint: disable=too-many-branches,too-many-statements
ssh_host_configs = esmon_common.config_value(config, esmon_common.CSTR_SSH_HOSTS)
if ssh_host_configs is None:
logging.error("can NOT find [%s] in the config file, "
"please correct file [%s]",
esmon_common.CSTR_SSH_HOSTS, config_fpath)
return -1
hosts = {}
for host_config in ssh_host_configs:
host_id = host_config[esmon_common.CSTR_HOST_ID]
if host_id is None:
logging.error("can NOT find [%s] in the config of a "
"SSH host, please correct file [%s]",
esmon_common.CSTR_HOST_ID, config_fpath)
return -1
hostname = esmon_common.config_value(host_config,
esmon_common.CSTR_HOSTNAME)
if hostname is None:
logging.error("can NOT find [%s] in the config of SSH host "
"with ID [%s], please correct file [%s]",
esmon_common.CSTR_HOSTNAME, host_id, config_fpath)
return -1
local = esmon_common.config_value(host_config,
esmon_common.CSTR_LOCAL_HOST)
if local is None:
logging.debug("can NOT find [%s] in the config of SSH host "
"with ID [%s], use [false] as default value",
esmon_common.CSTR_LOCAL_HOST, host_id)
local = False
mapping_dict = {esmon_common.ESMON_CONFIG_CSTR_NONE: None}
ssh_identity_file = esmon_common.config_value(host_config,
esmon_common.CSTR_SSH_IDENTITY_FILE,
mapping_dict=mapping_dict)
if host_id in hosts:
logging.error("multiple SSH hosts with the same ID [%s], please "
"correct file [%s]", host_id, config_fpath)
return -1
host = ssh_host.SSHHost(hostname, ssh_identity_file, local=local)
hosts[host_id] = host
template_configs = esmon_common.config_value(config, esmon_common.CSTR_TEMPLATES)
if template_configs is None:
logging.error("can NOT find [%s] in the config file, "
"please correct file [%s]",
esmon_common.CSTR_TEMPLATES, config_fpath)
return -1
templates = {}
for template_config in template_configs:
template_hostname = esmon_common.config_value(template_config,
esmon_common.CSTR_HOSTNAME)
if template_hostname is None:
logging.error("can NOT find [%s] in the config of a "
"SSH host, please correct file [%s]",
esmon_common.CSTR_HOSTNAME, config_fpath)
return -1
reinstall = esmon_common.config_value(template_config,
esmon_common.CSTR_REINSTALL)
if reinstall is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_REINSTALL, config_fpath)
return -1
internet = esmon_common.config_value(template_config,
esmon_common.CSTR_INTERNET)
if internet is None:
internet = False
logging.debug("no [%s] is configured, will "
"not add internet support", esmon_common.CSTR_INTERNET)
ram_size = esmon_common.config_value(template_config, esmon_common.CSTR_RAM_SIZE)
if ram_size is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_RAM_SIZE, config_fpath)
return -1
disk_sizes = esmon_common.config_value(template_config,
esmon_common.CSTR_DISK_SIZES)
if disk_sizes is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_DISK_SIZES, config_fpath)
return -1
network_configs = esmon_common.config_value(template_config,
esmon_common.CSTR_NETWORK_CONFIGS)
if network_configs is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_NETWORK_CONFIGS, config_fpath)
return -1
iso = esmon_common.config_value(template_config, esmon_common.CSTR_ISO)
if iso is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_ISO, config_fpath)
return -1
distro = esmon_common.config_value(template_config, esmon_common.CSTR_DISTRO)
if distro is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_DISTRO, config_fpath)
return -1
image_dir = esmon_common.config_value(template_config, esmon_common.CSTR_IMAGE_DIR)
if image_dir is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_IMAGE_DIR, config_fpath)
return -1
server_host_id = esmon_common.config_value(template_config,
esmon_common.CSTR_SERVER_HOST_ID)
if server_host_id is None:
logging.error("no [%s] is configured, please correct file [%s]",
esmon_common.CSTR_SERVER_HOST_ID, config_fpath)
return -1
if server_host_id not in hosts:
logging.error("SSH host with ID [%s] is NOT configured in "
"[%s], please correct file [%s]",
server_host_id, esmon_common.CSTR_SSH_HOSTS,
config_fpath)
return -1
server_host = hosts[server_host_id]
command = "mkdir -p %s" % workspace
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
template = VirtTemplate(server_host, iso, template_hostname, internet,
network_configs, image_dir, distro, ram_size,
disk_sizes)
templates[template_hostname] = template
state = server_host.sh_virsh_dominfo_state(template_hostname)
if not reinstall and state is not None:
logging.debug("skipping reinstall of template [%s] according to config",
template_hostname)
continue
ret = vm_install(workspace, server_host, iso,
template_hostname, internet,
network_configs, image_dir,
distro, ram_size, disk_sizes)
if ret:
logging.error("failed to create virtual machine template [%s]",
template_hostname)
return -1
vm_host_configs = esmon_common.config_value(config, "vm_hosts")
if vm_host_configs is None:
logging.error("no [vm_hosts] is configured, "
"please correct file [%s]", config_fpath)
return -1
vm_hosts = []
hosts_string = """127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
"""
for vm_host_config in vm_host_configs:
hostname = esmon_common.config_value(vm_host_config, esmon_common.CSTR_HOSTNAME)
if hostname is None:
logging.error("no [hostname] is configured for a vm_host, "
"please correct file [%s]", config_fpath)
return -1
ips = esmon_common.config_value(vm_host_config, esmon_common.CSTR_HOST_IPS)
if ips is None:
logging.error("no [%s] is configured for a vm_host, "
"please correct file [%s]", esmon_common.CSTR_HOST_IPS,
config_fpath)
return -1
template_hostname = esmon_common.config_value(vm_host_config,
esmon_common.CSTR_TEMPLATE_HOSTNAME)
if template_hostname is None:
logging.error("can NOT find [%s] in the config of a "
"SSH host, please correct file [%s]",
esmon_common.CSTR_TEMPLATE_HOSTNAME, config_fpath)
return -1
if template_hostname not in templates:
logging.error("template with hostname [%s] is NOT configured in "
"[%s], please correct file [%s]",
template_hostname, esmon_common.CSTR_TEMPLATES, config_fpath)
return -1
template = templates[template_hostname]
reinstall = esmon_common.config_value(vm_host_config, "reinstall")
state = template.vt_server_host.sh_virsh_dominfo_state(hostname)
if reinstall is None:
reinstall = False
if state is None:
reinstall = True
if not reinstall:
ret = vm_start(workspace,
template.vt_server_host,
hostname,
template.vt_network_configs,
ips,
template.vt_template_hostname,
template.vt_image_dir,
template.vt_distro,
template.vt_internet,
len(template.vt_disk_sizes))
if ret:
logging.error("virtual machine [%s] can't be started",
hostname)
return -1
else:
ret = vm_clone(workspace,
template.vt_server_host,
hostname,
template.vt_network_configs,
ips,
template.vt_template_hostname,
template.vt_image_dir,
template.vt_distro,
template.vt_internet,
len(template.vt_disk_sizes))
if ret:
logging.error("failed to create virtual machine [%s] based on "
"template [%s]", hostname,
template.vt_template_hostname)
return -1
host_ip = ips[0]
vm_host = ssh_host.SSHHost(hostname)
hosts_string += ("%s %s\n" % (host_ip, hostname))
vm_hosts.append(vm_host)
host_configs = esmon_common.config_value(config, esmon_common.CSTR_HOSTS)
if host_configs is None:
logging.error("can NOT find [%s] in the config file, "
"please correct file [%s]",
esmon_common.CSTR_HOSTS, config_fpath)
return -1
for host_config in host_configs:
hostname = esmon_common.config_value(host_config, esmon_common.CSTR_HOSTNAME)
if hostname is None:
logging.error("can NOT find [%s] in the config file, "
"please correct file [%s]",
esmon_common.CSTR_HOSTNAME, config_fpath)
return -1
host_ip = esmon_common.config_value(host_config, esmon_common.CSTR_IP)
if host_ip is None:
logging.error("can NOT find [%s] in the config file, "
"please correct file [%s]",
esmon_common.CSTR_IP, config_fpath)
return -1
hosts_string += ("%s %s\n" % (host_ip, hostname))
hosts_fpath = workspace + "/hosts"
with open(hosts_fpath, "wt") as hosts_file:
hosts_file.write(hosts_string)
for host in vm_hosts:
ret = host.sh_send_file(hosts_fpath, "/etc")
if ret:
logging.error("failed to send hosts file [%s] on local host to "
"directory [%s] on host [%s]",
hosts_fpath, workspace,
host.sh_hostname)
return -1
# Clear the known_hosts, otherwise the reinstalled hosts can't be
# accessed by other hosts
command = "> /root/.ssh/known_hosts"
retval = host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
return 0
def vm_install(workspace, server_host, iso_path, hostname,
internet, network_configs, image_dir, distro,
ram_size, disk_sizes):
"""
Install virtual machine from ISO
"""
# pylint: disable=too-many-arguments,too-many-locals
# pylint: disable=too-many-return-statements,too-many-statements
# pylint: disable=too-many-branches
ret = vm_delete(server_host, hostname)
if ret:
return -1
network_config = network_configs[0]
host_ip = network_config["ip"]
command = ("ping -c 1 %s" % host_ip)
retval = server_host.sh_run(command)
if retval.cr_exit_status == 0:
logging.error("IP [%s] is already used by a host", host_ip)
return -1
command = ("ping -c 1 %s" % hostname)
retval = server_host.sh_run(command)
if retval.cr_exit_status == 0:
logging.error("host [%s] is already up", hostname)
return -1
mnt_path = "/mnt/" + utils.random_word(8)
command = ("mkdir -p %s && mount -o loop %s %s" %
(mnt_path, iso_path, mnt_path))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
ks_config = """# Kickstart file automatically generated by ESMON.
install
reboot
cdrom
lang en_US.UTF-8
keyboard us
"""
ks_config += """rootpw password
firewall --disabled
authconfig --enableshadow --passalgo=sha512
selinux --disabled
timezone --utc Asia/Shanghai
bootloader --location=mbr --driveorder=sda --append="crashkernel=auto net.ifnames=0 biosdevname=0"
zerombr
clearpart --all --initlabel
part / --fstype=ext4 --grow --size=500 --ondisk=sda --asprimary
repo --name="Media" --baseurl=file:///mnt/source --cost=100
%packages
@Core
%end
%post --log=/var/log/anaconda/post-install.log
#!/bin/bash
# Configure hostname, somehow virt-install --name doesn't work
"""
if distro == ssh_host.DISTRO_RHEL6:
ks_config += 'echo NETWORKING=yes > /etc/sysconfig/network\n'
ks_config += ('echo HOSTNAME=%s >> /etc/sysconfig/network\n' %
(hostname))
elif distro == ssh_host.DISTRO_RHEL7:
ks_config += "echo %s > /etc/hostname\n" % (hostname)
else:
logging.error("wrong distro [%s]", distro)
return -1
ks_config += "# Configure network\n"
eth_number = 0
ens_number = 3
for network_config in network_configs:
# net.ifnames=0 biosdevname=0 will be added to GRUB_CMDLINE_LINUX, so the
# interface name will always be eth*
ks_config += "# Network eth%d\n" % eth_number
ks_config += ("rm -f /etc/sysconfig/network-scripts/ifcfg-ens%d\n" %
ens_number)
ks_config += ("cat << EOF > /etc/sysconfig/network-scripts/ifcfg-eth%d\n" %
eth_number)
ks_config += "DEVICE=eth%d\n" % eth_number
ks_config += 'IPADDR="%s"\n' % network_config["ip"]
ks_config += 'NETMASK="%s"\n' % network_config["netmask"]
if "gateway" in network_config:
ks_config += 'GATEWAY=\"%s"\n' % network_config["gateway"]
ks_config += """ONBOOT=yes
BOOTPROTO="static"
TYPE=Ethernet
IPV6INIT=no
NM_CONTROLLED=no
EOF
"""
eth_number += 1
ens_number += 1
ks_config += "%end\n"
local_host_dir = workspace + "/" + hostname
os.mkdir(local_host_dir)
ks_fname = "%s.ks" % hostname
ks_fpath = local_host_dir + "/" + ks_fname
with open(ks_fpath, "wt") as fout:
fout.write(ks_config)
host_ks_fpath = workspace + "/" + ks_fname
ret = server_host.sh_send_file(ks_fpath, workspace)
if ret:
logging.error("failed to send file [%s] on local host to "
"directory [%s] on host [%s]",
ks_fpath, workspace,
server_host.sh_hostname)
return -1
command = ("virt-install --vcpus=1 --os-type=linux "
"--hvm --connect=qemu:///system "
"--accelerate --serial pty -v --nographics --noautoconsole --wait=-1 ")
command += "--ram=%s " % ram_size
for network_config in network_configs:
command += ("--network=%s " % (network_config["virt_install_option"]))
command += ("--name=%s " % (hostname))
command += ("--initrd-inject=%s " % (host_ks_fpath))
disk_index = 0
for disk_size in disk_sizes:
command += ("--disk path=%s/%s_%d.img,size=%s " %
(image_dir, hostname, disk_index, disk_size))
disk_index += 1
command += ("--location %s " % (mnt_path))
command += ("--disk=%s,device=cdrom,perms=ro " % (iso_path))
command += ("--extra-args='console=tty0 console=ttyS0,115200n8 "
"ks=file:/%s'" % (ks_fname))
if distro == ssh_host.DISTRO_RHEL6:
install_timeout = 300
elif distro == ssh_host.DISTRO_RHEL7:
install_timeout = 1200
retval = server_host.sh_run(command, timeout=install_timeout)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
ret = server_host.sh_run("which sshpass")
if ret.cr_exit_status != 0:
# sshpass rely on epel-release on centos6
command = ("yum install epel-release -y")
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = ("yum install sshpass -y")
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
# Remove the record in known_hosts, otherwise ssh will fail
command = ('sed -i "/%s /d" /root/.ssh/known_hosts' % (host_ip))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
# When virt-install finished, the virtual machine starts to reboot
# so wait a little bit here until the host is up. Need
# StrictHostKeyChecking=no, otherwise exit code will be 6 (ENOENT)
expect_stdout = hostname + "\n"
command = ("sshpass -p password ssh -o StrictHostKeyChecking=no "
"root@%s hostname" % (host_ip))
ret = server_host.sh_wait_update(command, expect_exit_status=0,
expect_stdout=expect_stdout)
if ret:
logging.error("failed to wait host [%s] up", hostname)
return -1
command = ("sshpass -p password ssh root@%s "
"\"mkdir /root/.ssh && chmod 600 /root/.ssh\"" % (host_ip))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = ("sshpass -p password scp /root/.ssh/* root@%s:/root/.ssh" % (host_ip))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
vm_host = ssh_host.SSHHost(host_ip)
command = "> /root/.ssh/known_hosts"
retval = vm_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
vm_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = "hostname"
retval = vm_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
vm_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
real_hostname = retval.cr_stdout.strip()
if real_hostname != hostname:
logging.error("wrong hostname, expected [%s], got [%s]",
hostname, real_hostname)
return -1
if internet:
ret = vm_host.sh_enable_dns()
if ret:
logging.error("failed to enable dns on host [%s]")
return -1
command = "yum install rsync -y"
retval = vm_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
vm_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
# Do not check the return status, because the connection could be stopped
command = "init 0"
vm_host.sh_run(command)
command = ("umount %s" % (mnt_path))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
ret = -1
command = ("rmdir %s" % (mnt_path))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
# Need to wait until VM shut off, otherwise "virsh change-media" won't
# change the XML file
ret = utils.wait_condition(vm_check_shut_off, [server_host, hostname])
if ret:
logging.error("failed when waiting host [%s] on [%s] shut off",
hostname, server_host.sh_hostname)
return ret
# Find the CDROM device
command = ("virsh domblklist %s --details | grep cdrom | "
"awk '{print $3}'" % (hostname))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
cdroms = retval.cr_stdout.splitlines()
if len(cdroms) != 1:
logging.error("unexpected cdroms: [%s]",
retval.cr_stdout)
return -1
cdrom = cdroms[0]
command = ("virsh change-media %s %s --eject" % (hostname, cdrom))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
return 0
def vm_clone(workspace, server_host, hostname, network_configs, ips,
template_hostname, image_dir, distro, internet, disk_number):
"""
Create virtual machine
"""
# pylint: disable=too-many-arguments,too-many-locals,too-many-return-statements
# pylint: disable=too-many-branches,too-many-statements
host_ip = ips[0]
ret = vm_delete(server_host, hostname)
if ret:
return -1
command = ("ping -c 1 %s" % host_ip)
retval = server_host.sh_run(command)
if retval.cr_exit_status == 0:
logging.error("IP [%s] already used by a host", host_ip)
return -1
command = ("ping -c 1 %s" % hostname)
retval = server_host.sh_run(command)
if retval.cr_exit_status == 0:
logging.error("host [%s] already up", hostname)
return -1
active = True
state = server_host.sh_virsh_dominfo_state(template_hostname)
if state is None:
logging.error("template [%s] doesn't exist on host [%s]",
template_hostname, server_host.sh_hostname)
return -1
elif state == "shut off":
active = False
if active:
command = ("virsh destroy %s" % template_hostname)
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
file_options = ""
for disk_index in range(disk_number):
file_options += (" --file %s/%s_%d.img" %
(image_dir, hostname, disk_index))
command = ("rm -f %s/%s_%d.img" %
(image_dir, hostname, disk_index))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = ("virt-clone --original %s --name %s%s" %
(template_hostname, hostname, file_options))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
local_host_dir = workspace + "/" + hostname
os.mkdir(local_host_dir)
# net.ifnames=0 biosdevname=0 has been added to grub, so the interface
# name will always be eth*
eth_number = 0
for eth_ip in ips:
network_config = network_configs[eth_number]
ifcfg = 'DEVICE="eth%d"\n' % eth_number
ifcfg += 'IPADDR="%s"\n' % eth_ip
ifcfg += 'NETMASK="%s"\n' % network_config["netmask"]
if "gateway" in network_config:
ifcfg += 'GATEWAY=\"%s"\n' % network_config["gateway"]
ifcfg += """ONBOOT=yes
BOOTPROTO="static"
TYPE=Ethernet
IPV6INIT=no
NM_CONTROLLED=no
"""
ifcfg_fname = "ifcfg-eth%d" % eth_number
ifcfg_fpath = local_host_dir + "/" + ifcfg_fname
with open(ifcfg_fpath, "wt") as fout:
fout.write(ifcfg)
host_ifcfg_fpath = workspace + "/" + ifcfg_fname
ret = server_host.sh_send_file(ifcfg_fpath, workspace)
if ret:
logging.error("failed to send file [%s] on local host to "
"directory [%s] on host [%s]",
ifcfg_fpath, workspace,
server_host.sh_hostname)
return -1
ret = server_host.sh_run("which virt-copy-in")
if ret.cr_exit_status != 0:
command = ("yum install libguestfs-tools-c -y")
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = ("virt-copy-in -d %s %s "
"/etc/sysconfig/network-scripts" % (hostname, host_ifcfg_fpath))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
eth_number += 1
host_rules_fpath = workspace + "/70-persistent-net.rules"
command = ("> %s" % host_rules_fpath)
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = ("virt-copy-in -d %s %s "
"/etc/udev/rules.d" % (hostname, host_rules_fpath))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
if distro == ssh_host.DISTRO_RHEL6:
network_string = 'NETWORKING=yes\n'
network_string += 'HOSTNAME=%s\n' % hostname
network_fname = "network"
network_fpath = local_host_dir + "/" + network_fname
with open(network_fpath, "wt") as fout:
fout.write(network_string)
host_network_fpath = workspace + "/" + network_fname
ret = server_host.sh_send_file(network_fpath, workspace)
if ret:
logging.error("failed to send file [%s] on local host to "
"directory [%s] on host [%s]",
network_fpath, workspace,
server_host.sh_hostname)
return -1
command = ("virt-copy-in -d %s %s "
"/etc/sysconfig" % (hostname, host_network_fpath))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
else:
host_hostname_fpath = workspace + "/hostname"
command = ("echo %s > %s" % (hostname, host_hostname_fpath))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = ("virt-copy-in -d %s %s "
"/etc" % (hostname, host_hostname_fpath))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
command = ("virsh start %s" % hostname)
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
# Remove the record in known_hosts, otherwise ssh will fail
command = ('sed -i "/%s /d" /root/.ssh/known_hosts' % (host_ip))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
# Remove the record in known_hosts, otherwise ssh will fail
command = ('sed -i "/%s /d" /root/.ssh/known_hosts' % (hostname))
retval = server_host.sh_run(command)
if retval.cr_exit_status:
logging.error("failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]",
command,
server_host.sh_hostname,
retval.cr_exit_status,
retval.cr_stdout,
retval.cr_stderr)
return -1
vm_host = ssh_host.SSHHost(host_ip)
ret = vm_host.sh_wait_up()
if ret:
logging.error("failed to wait host [%s] up",
host_ip)
return -1
ret = vm_check(hostname, host_ip, distro, internet)
if ret:
return -1
return 0
def esmon_do_build(current_dir, relative_workspace, config, config_fpath):
"""
Build the ISO
"""
# pylint: disable=too-many-locals,too-many-return-statements
# pylint: disable=too-many-branches,too-many-statements
hosts = {}
if parse_host_configs(config, config_fpath, hosts):
logging.error("failed to parse host configs")
return -1
centos6_host_config = esmon_common.config_value(config, "centos6_host")
if centos6_host_config is None:
logging.info(
"can NOT find [centos6_host] in the config file [%s], "
"diableing CentOS6 support", config_fpath)
centos6_host = None
else:
centos6_host_id = esmon_common.config_value(centos6_host_config,
"host_id")
if centos6_host_id is None:
logging.error(
"can NOT find [host_id] in the config of [centos6_host], "
"please correct file [%s]", config_fpath)
return -1
if centos6_host_id not in hosts:
logging.error(
"SSH host with ID [%s] is NOT configured in "
"[ssh_hosts], please correct file [%s]", centos6_host_id,
config_fpath)
return -1
centos6_host = hosts[centos6_host_id]
local_host = ssh_host.SSHHost("localhost", local=True)
distro = local_host.sh_distro()
if distro != ssh_host.DISTRO_RHEL7:
logging.error("build can only be launched on RHEL7/CentOS7 host")
return -1
iso_cached_dir = current_dir + "/../iso_cached_dir"
collectd_git_path = current_dir + "/../" + "collectd.git"
rpm_dir = iso_cached_dir + "/RPMS"
command = ("mkdir -p %s" % (rpm_dir))
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
collectd_git_url = esmon_common.config_value(config, "collectd_git_url")
if collectd_git_url is None:
collectd_git_url = "https://github.com/DDNStorage/collectd.git"
logging.info(
"can NOT find [collectd_git_url] in the config, "
"use default value [%s]", collectd_git_url)
collectd_git_branch = esmon_common.config_value(config,
"collectd_git_branch")
if collectd_git_branch is None:
collectd_git_branch = "master-ddn"
logging.info(
"can NOT find [collectd_git_branch] in the config, "
"use default value [%s]", collectd_git_branch)
ret = esmon_common.clone_src_from_git(collectd_git_path, collectd_git_url,
collectd_git_branch)
if ret:
logging.error(
"failed to clone Collectd branch [%s] from [%s] to "
"directory [%s]", collectd_git_branch, collectd_git_url,
collectd_git_path)
return -1
command = ("cd %s && git rev-parse --short HEAD" % collectd_git_path)
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
collectd_git_version = retval.cr_stdout.strip()
command = (r"cd %s && grep Version contrib/redhat/collectd.spec | "
r"grep -v \# | awk '{print $2}'" % collectd_git_path)
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
collectd_version_string = retval.cr_stdout.strip()
collectd_version = collectd_version_string.replace('%{?rev}',
collectd_git_version)
collectd_tarball_name = "collectd-" + collectd_version
command = (r"cd %s && grep Release contrib/redhat/collectd.spec | "
r"grep -v \# | awk '{print $2}'" % collectd_git_path)
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
collectd_release_string = retval.cr_stdout.strip()
collectd_release = collectd_release_string.replace('%{?dist}', '')
collectd_version_release = collectd_version + "-" + collectd_release
if centos6_host is not None:
centos6_workspace = ESMON_BUILD_LOG_DIR + "/" + relative_workspace
ret = host_build(centos6_workspace, centos6_host, local_host,
collectd_git_path, iso_cached_dir,
collectd_version_release, collectd_tarball_name,
ssh_host.DISTRO_RHEL6)
if ret:
logging.error("failed to prepare RPMs of CentOS6 on host [%s]",
centos6_host.sh_hostname)
return -1
# The build host of CentOS7 could potentially be another host, not local
# host
local_workspace = current_dir + "/" + relative_workspace
ret = host_build(local_workspace, local_host, local_host,
collectd_git_path, iso_cached_dir,
collectd_version_release, collectd_tarball_name,
ssh_host.DISTRO_RHEL7)
if ret:
logging.error("failed to prepare RPMs of CentOS7 on local host")
return -1
local_distro_rpm_dir = (
"%s/%s/%s" % (iso_cached_dir, RPM_STRING, ssh_host.DISTRO_RHEL7))
local_server_rpm_dir = ("%s/%s" % (local_distro_rpm_dir, SERVER_STRING))
command = ("mkdir -p %s" % local_server_rpm_dir)
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
server_rpms = {}
name = "grafana-6.0.2-1.x86_64.rpm"
url = ("https://dl.grafana.com/oss/release/grafana-6.0.2-1.x86_64.rpm")
server_rpms[name] = url
name = "influxdb-1.7.4.x86_64.rpm"
url = ("https://dl.influxdata.com/influxdb/releases/"
"influxdb-1.7.4.x86_64.rpm")
server_rpms[name] = url
for name, url in server_rpms.iteritems():
fpath = ("%s/%s" % (local_server_rpm_dir, name))
command = "test -e %s" % fpath
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.debug("file [%s] doesn't exist, downloading it", fpath)
command = ("cd %s && wget --no-check-certificate %s" %
(local_server_rpm_dir, url))
retval = local_host.sh_run(command, timeout=3600)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status,
retval.cr_stdout, retval.cr_stderr)
return -1
server_existing_files = os.listdir(local_server_rpm_dir)
for server_rpm in server_rpms.iterkeys():
server_existing_files.remove(server_rpm)
for extra_fname in server_existing_files:
logging.warning(
"find unknown file [%s] under directory [%s], removing",
extra_fname, local_server_rpm_dir)
command = ("rm -fr %s/%s" % (local_server_rpm_dir, extra_fname))
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status,
retval.cr_stdout, retval.cr_stderr)
return -1
ret = esmon_download_grafana_plugins(local_host, iso_cached_dir)
if ret:
logging.error("failed to download Grafana plugins")
return -1
dependent_existing_files = os.listdir(iso_cached_dir)
for panel_name in esmon_common.GRAFANA_PLUGIN_GITS.iterkeys():
dependent_existing_files.remove(panel_name)
dependent_existing_files.remove("RPMS")
for extra_fname in dependent_existing_files:
logging.warning(
"find unknown file [%s] under directory [%s], removing",
extra_fname, iso_cached_dir)
command = ("rm -fr %s/%s" % (iso_cached_dir, extra_fname))
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status,
retval.cr_stdout, retval.cr_stderr)
return -1
command = ("cd %s && rm esmon-*.tar.bz2 esmon-*.tar.gz -f && "
"sh autogen.sh && "
"./configure --with-cached-iso=%s && "
"make" % (current_dir, iso_cached_dir))
retval = local_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
local_host.sh_hostname, retval.cr_exit_status, retval.cr_stdout,
retval.cr_stderr)
return -1
if centos6_host is not None:
command = ("rm -fr %s" % (centos6_workspace))
retval = centos6_host.sh_run(command)
if retval.cr_exit_status:
logging.error(
"failed to run command [%s] on host [%s], "
"ret = [%d], stdout = [%s], stderr = [%s]", command,
centos6_host.sh_hostname, retval.cr_exit_status,
retval.cr_stdout, retval.cr_stderr)
return -1
return 0