def connect_PerfRepo(self, mapping_file, url=None, username=None, password=None, max_retries=3):
if not self._perf_repo_api.connected():
if url is None:
url = lnst_config.get_option("perfrepo", "url")
if username is None:
username = lnst_config.get_option("perfrepo", "username")
if password is None:
password = lnst_config.get_option("perfrepo", "password")
if not url:
logging.warn("No PerfRepo URL specified in config file")
if not username:
logging.warn("No PerfRepo username specified in config file")
if not password:
logging.warn("No PerfRepo password specified in config file")
if url and username and password:
self._perf_repo_api.connect(url, username, password, max_retries)
root = Path(None, self._ctl._recipe_path).get_root()
path = Path(root, mapping_file)
self._perf_repo_api.load_mapping(path)
if not self._perf_repo_api.connected():
if PerfRepoRESTAPI is None:
logging.warn("Python PerfRepo library not found.")
logging.warn("Connection to PerfRepo incomplete, further "\
"PerfRepo commands will be ignored.")
return self._perf_repo_api
def connect_PerfRepo(self, mapping_file, url=None, username=None, password=None, max_retries=3):
if not self._perf_repo_api.connected():
if url is None:
url = lnst_config.get_option("perfrepo", "url")
if username is None:
username = lnst_config.get_option("perfrepo", "username")
if password is None:
password = lnst_config.get_option("perfrepo", "password")
if not url:
logging.warn("No PerfRepo URL specified in config file")
if not username:
logging.warn("No PerfRepo username specified in config file")
if not password:
logging.warn("No PerfRepo password specified in config file")
if url and username and password:
self._perf_repo_api.connect(url, username, password, max_retries)
root = Path(None, self._ctl._recipe_path).get_root()
path = Path(root, mapping_file)
self._perf_repo_api.load_mapping(path)
if not self._perf_repo_api.connected():
if PerfRepoRESTAPI is None:
logging.warn("Python PerfRepo library not found.")
logging.warn("Connection to PerfRepo incomplete, further "\
"PerfRepo commands will be ignored.")
return self._perf_repo_api
def __init__(self, recipe_path, log_ctl,
res_serializer=None, pool_checks=True,
defined_aliases=None, overriden_aliases=None,
reduce_sync=False):
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
self._recipe_path = recipe_path
self._msg_dispatcher = MessageDispatcher(log_ctl)
self._reduce_sync = reduce_sync
self._parser = RecipeParser(recipe_path)
self.remove_saved_machine_config()
sp = SlavePool(lnst_config.get_option('environment', 'pool_dirs'),
check_process_running("libvirtd"), pool_checks)
self._slave_pool = sp
self._machines = {}
self._network_bridges = {}
self._tasks = []
mac_pool_range = lnst_config.get_option('environment', 'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
self._parser.set_machines(self._machines)
self._parser.set_aliases(defined_aliases, overriden_aliases)
self._recipe = self._parser.parse()
modules_dirs = lnst_config.get_option('environment', 'module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
self._resource_table = {}
self._resource_table["module"] = self._load_test_modules(modules_dirs)
self._resource_table["tools"] = self._load_test_tools(tools_dirs)
def __init__(self, m_id, hostname=None, libvirt_domain=None, rpcport=None,
security=None):
self._id = m_id
self._hostname = hostname
self._slave_desc = None
self._connection = None
self._configured = False
self._system_config = {}
self._security = security
self._security["identity"] = lnst_config.get_option("security",
"identity")
self._security["privkey"] = lnst_config.get_option("security",
"privkey")
self._domain_ctl = None
self._network_bridges = None
self._libvirt_domain = libvirt_domain
if libvirt_domain:
self._domain_ctl = VirtDomainCtl(libvirt_domain)
if rpcport:
self._port = rpcport
else:
self._port = lnst_config.get_option('environment', 'rpcport')
self._msg_dispatcher = None
self._mac_pool = None
self._interfaces = []
self._namespaces = []
self._services = []
self._bg_cmds = {}
self._device_database = {}
def __init__(self,
recipe_path,
log_ctl,
res_serializer=None,
pool_checks=True,
packet_capture=False,
defined_aliases=None,
overriden_aliases=None,
reduce_sync=False,
restrict_pools=[]):
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
self._recipe_path = Path(None, recipe_path).abs_path()
self._msg_dispatcher = MessageDispatcher(log_ctl)
self._packet_capture = packet_capture
self._reduce_sync = reduce_sync
self._parser = RecipeParser(recipe_path)
self.remove_saved_machine_config()
self._machines = {}
self._network_bridges = {}
self._tasks = []
mac_pool_range = lnst_config.get_option('environment',
'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
self._parser.set_machines(self._machines)
self._parser.set_aliases(defined_aliases, overriden_aliases)
self._recipe = self._parser.parse()
conf_pools = lnst_config.get_pools()
pools = {}
if len(restrict_pools) > 0:
for pool_name in restrict_pools:
if pool_name in conf_pools:
pools[pool_name] = conf_pools[pool_name]
elif len(restrict_pools) == 1 and os.path.isdir(pool_name):
pools = {"cmd_line_pool": pool_name}
else:
raise NetTestError("Pool %s does not exist!" % pool_name)
else:
pools = conf_pools
sp = SlavePool(pools, pool_checks)
self._slave_pool = sp
mreq = self._get_machine_requirements()
sp.set_machine_requirements(mreq)
modules_dirs = lnst_config.get_option('environment', 'module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
self._resource_table = {}
self._resource_table["module"] = self._load_test_modules(modules_dirs)
self._resource_table["tools"] = self._load_test_tools(tools_dirs)
def connect_PerfRepo(self, url=None, username=None, password=None):
if not self._perf_repo_api.connected():
if url is None:
url = lnst_config.get_option("perfrepo", "url")
if username is None:
url = lnst_config.get_option("perfrepo", "username")
if password is None:
url = lnst_config.get_option("perfrepo", "password")
self._perf_repo_api.connect(url, username, password)
return self._perf_repo_api
def __init__(self, recipe_path, log_ctl,
res_serializer=None, pool_checks=True,
packet_capture=False,
defined_aliases=None, overriden_aliases=None,
reduce_sync=False, restrict_pools=[]):
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
self._recipe_path = Path(None, recipe_path).abs_path()
self._msg_dispatcher = MessageDispatcher(log_ctl)
self._packet_capture = packet_capture
self._reduce_sync = reduce_sync
self._parser = RecipeParser(recipe_path)
self.remove_saved_machine_config()
self._machines = {}
self._network_bridges = {}
self._tasks = []
mac_pool_range = lnst_config.get_option('environment', 'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
self._parser.set_machines(self._machines)
self._parser.set_aliases(defined_aliases, overriden_aliases)
self._recipe = self._parser.parse()
conf_pools = lnst_config.get_pools()
pools = {}
if len(restrict_pools) > 0:
for pool_name in restrict_pools:
if pool_name in conf_pools:
pools[pool_name] = conf_pools[pool_name]
elif len(restrict_pools) == 1 and os.path.isdir(pool_name):
pools = {"cmd_line_pool": pool_name}
else:
raise NetTestError("Pool %s does not exist!" % pool_name)
else:
pools = conf_pools
sp = SlavePool(pools, pool_checks)
self._slave_pool = sp
mreq = self._get_machine_requirements()
sp.set_machine_requirements(mreq)
modules_dirs = lnst_config.get_option('environment', 'module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
self._resource_table = {}
self._resource_table["module"] = self._load_test_modules(modules_dirs)
self._resource_table["tools"] = self._load_test_tools(tools_dirs)
def __init__(self, command_context, netconfig, log_ctl):
self._packet_captures = {}
self._netconfig = netconfig
self._command_context = command_context
self._log_ctl = log_ctl
self._capture_files = {}
self._copy_targets = {}
self._copy_sources = {}
self._system_config = {}
self._cache = ResourceCache(lnst_config.get_option("cache", "dir"),
lnst_config.get_option("cache", "expiration_period"))
self._resource_table = {}
def __init__(self, command_context, log_ctl, if_manager):
self._packet_captures = {}
self._if_manager = if_manager
self._command_context = command_context
self._log_ctl = log_ctl
self._capture_files = {}
self._copy_targets = {}
self._copy_sources = {}
self._system_config = {}
self._cache = ResourceCache(lnst_config.get_option("cache", "dir"),
lnst_config.get_option("cache", "expiration_period"))
self._resource_table = {'module': {}, 'tools': {}}
def is_nm_managed(dev_config, if_manager):
if lnst_config.get_option("environment", "use_nm") and\
check_process_running("NetworkManager"):
return type_class_mapping[dev_config["type"]].is_nm_managed(dev_config,
if_manager)
else:
return False
def hello(self, recipe_path):
self.machine_cleanup()
self.restore_nm_option()
logging.info("Recieved a controller connection.")
self.clear_resource_table()
self._cache.del_old_entries()
self.reset_file_transfers()
self._if_manager.rescan_devices()
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
self._log_ctl.set_recipe(recipe_path, expand=date)
sleep(1)
slave_desc = {}
if check_process_running("NetworkManager"):
logging.warning("=============================================")
logging.warning("NetworkManager is running on a slave machine!")
if lnst_config.get_option("environment", "use_nm"):
logging.warning("Support of NM is still experimental!")
else:
logging.warning("Usage of NM is disabled!")
logging.warning("=============================================")
slave_desc["nm_running"] = True
else:
slave_desc["nm_running"] = False
k_release, _ = exec_cmd("uname -r", False, False, False)
r_release, _ = exec_cmd("cat /etc/redhat-release", False, False, False)
slave_desc["kernel_release"] = k_release.strip()
slave_desc["redhat_release"] = r_release.strip()
return ("hello", slave_desc)
def disable_nm(self):
logging.warning("=====================================================")
logging.warning("Disabling use of NetworkManager on controller request")
logging.warning("=====================================================")
val = lnst_config.get_option("environment", "use_nm")
lnst_config.set_option("environment", "use_nm", False)
return val
def is_nm_managed(dev_config, if_manager):
if lnst_config.get_option("environment", "use_nm") and\
check_process_running("NetworkManager"):
return type_class_mapping[dev_config["type"]].is_nm_managed(dev_config,
if_manager)
else:
return False
def __init__(self, command_context, netconfig, log_ctl):
self._packet_captures = {}
self._netconfig = netconfig
self._command_context = command_context
self._log_ctl = log_ctl
self._capture_files = {}
self._copy_targets = {}
self._copy_sources = {}
self._system_config = {}
self._cache = ResourceCache(
lnst_config.get_option("cache", "dir"),
lnst_config.get_option("cache", "expiration_period"))
self._resource_table = {}
def __init__(self, log_ctl):
die_when_parent_die()
self._cmd_context = NetTestCommandContext()
port = lnst_config.get_option("environment", "rpcport")
logging.info("Using RPC port %d." % port)
self._server_handler = ServerHandler(("", port))
self._if_manager = InterfaceManager(self._server_handler)
self._server_handler.set_if_manager(self._if_manager)
self._net_namespaces = {}
self._methods = SlaveMethods(self._cmd_context, log_ctl,
self._if_manager, self._net_namespaces,
self._server_handler, self)
self.register_die_signal(signal.SIGHUP)
self.register_die_signal(signal.SIGINT)
self.register_die_signal(signal.SIGTERM)
self._finished = False
self._log_ctl = log_ctl
self._server_handler.add_connection('netlink',
self._if_manager.get_nl_socket())
def hello(self, recipe_path):
self.machine_cleanup()
self.restore_nm_option()
logging.info("Recieved a controller connection.")
self.clear_resource_table()
self._cache.del_old_entries()
self.reset_file_transfers()
self._if_manager.rescan_devices()
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
self._log_ctl.set_recipe(recipe_path, expand=date)
sleep(1)
slave_desc = {}
if check_process_running("NetworkManager"):
logging.warning("=============================================")
logging.warning("NetworkManager is running on a slave machine!")
if lnst_config.get_option("environment", "use_nm"):
logging.warning("Support of NM is still experimental!")
else:
logging.warning("Usage of NM is disabled!")
logging.warning("=============================================")
slave_desc["nm_running"] = True
else:
slave_desc["nm_running"] = False
k_release, _ = exec_cmd("uname -r", False, False, False)
r_release, _ = exec_cmd("cat /etc/redhat-release", False, False, False)
slave_desc["kernel_release"] = k_release.strip()
slave_desc["redhat_release"] = r_release.strip()
slave_desc["lnst_version"] = lnst_config.version
return ("hello", slave_desc)
def enable_nm(self):
logging.warning("====================================================")
logging.warning("Enabling use of NetworkManager on controller request")
logging.warning("====================================================")
val = lnst_config.get_option("environment", "use_nm")
lnst_config.set_option("environment", "use_nm", True)
return val
def __init__(self, log_ctl):
die_when_parent_die()
self._cmd_context = NetTestCommandContext()
port = lnst_config.get_option("environment", "rpcport")
logging.info("Using RPC port %d." % port)
self._server_handler = ServerHandler(("", port))
self._if_manager = InterfaceManager(self._server_handler)
self._net_namespaces = {}
self._methods = SlaveMethods(self._cmd_context, log_ctl,
self._if_manager, self._net_namespaces,
self._server_handler, self)
self.register_die_signal(signal.SIGHUP)
self.register_die_signal(signal.SIGINT)
self.register_die_signal(signal.SIGTERM)
self._finished = False
self._log_ctl = log_ctl
self._server_handler.add_connection('netlink',
self._if_manager.get_nl_socket())
def __init__(self, m_id, hostname=None, libvirt_domain=None, rpcport=None):
self._id = m_id
self._hostname = hostname
self._slave_desc = None
self._connection = None
self._configured = False
self._system_config = {}
self._domain_ctl = None
self._network_bridges = None
self._libvirt_domain = libvirt_domain
if libvirt_domain:
self._domain_ctl = VirtDomainCtl(libvirt_domain)
if rpcport:
self._port = rpcport
else:
self._port = lnst_config.get_option('environment', 'rpcport')
self._msg_dispatcher = None
self._mac_pool = None
self._interfaces = []
self._namespaces = []
self._bg_cmds = {}
def __init__(self, pools, pool_checks=True):
self._map = {}
self._pools = {}
self._pool = {}
self._machine_matches = []
self._network_matches = []
self._allow_virt = lnst_config.get_option("environment",
"allow_virtual")
self._allow_virt &= check_process_running("libvirtd")
self._pool_checks = pool_checks
self._mapper = SetupMapper()
self._mreqs = None
logging.info("Checking machine pool availability.")
for pool_name, pool_dir in pools.items():
self._pools[pool_name] = {}
self.add_dir(pool_name, pool_dir)
if len(self._pools[pool_name]) == 0:
del self._pools[pool_name]
self._mapper.set_pools(self._pools)
logging.info("Finished loading pools.")
def __init__(self, pools, pool_checks=True):
self._map = {}
self._pools = {}
self._pool = {}
self._machine_matches = []
self._network_matches = []
self._allow_virt = lnst_config.get_option("environment",
"allow_virtual")
self._allow_virt &= check_process_running("libvirtd")
self._pool_checks = pool_checks
self._mapper = SetupMapper()
self._mreqs = None
logging.info("Checking machine pool availability.")
for pool_name, pool_dir in pools.items():
self._pools[pool_name] = {}
self.add_dir(pool_name, pool_dir)
if len(self._pools[pool_name]) == 0:
del self._pools[pool_name]
self._mapper.set_pools(self._pools)
logging.info("Finished loading pools.")
def __init__(self,
recipe_path,
log_ctl,
cleanup=False,
res_serializer=None,
pool_checks=True):
self._docleanup = cleanup
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
self._recipe_path = recipe_path
self._msg_dispatcher = MessageDispatcher(log_ctl)
sp = SlavePool(lnst_config.get_option('environment', 'pool_dirs'),
check_process_running("libvirtd"), pool_checks)
self._slave_pool = sp
self._machines = {}
self._network_bridges = {}
self._recipe = recipe = {}
recipe["networks"] = {}
recipe["machines"] = {}
recipe["switches"] = {}
mac_pool_range = lnst_config.get_option('environment',
'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
parser = RecipeParse(recipe_path)
parser.set_target(self._recipe)
parser.set_machines(self._machines)
parser.register_event_handler("provisioning_requirements_ready",
self._prepare_provisioning)
parser.register_event_handler("interface_config_ready",
self._prepare_interface)
modules_dirs = lnst_config.get_option('environment', 'module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
self._resource_table = {}
self._resource_table["module"] = self._load_test_modules(modules_dirs)
self._resource_table["tools"] = self._load_test_tools(tools_dirs)
self._parser = parser
def NetConfigDeviceAllCleanup():
if check_process_running("NetworkManager") and \
lnst_config.get_option("environment", "use_nm"):
for dev_type in nm_type_class_mapping:
nm_type_class_mapping[dev_type].type_cleanup()
else:
for dev_type in type_class_mapping:
type_class_mapping[dev_type].type_cleanup()
def NetConfigDeviceAllCleanup():
if check_process_running("NetworkManager") and \
lnst_config.get_option("environment", "use_nm"):
for dev_type in nm_type_class_mapping:
nm_type_class_mapping[dev_type].type_cleanup()
else:
for dev_type in type_class_mapping:
type_class_mapping[dev_type].type_cleanup()
def remove_saved_machine_config(cls):
#removes previously saved configuration
cfg = None
try:
with open("/tmp/.lnst_machine_conf", "rb") as f:
cfg = cPickle.load(f)
except:
logging.info("No previous configuration found.")
return
if cfg:
logging.info("Cleaning up leftover configuration from previous "\
"config_only run.")
for hostname, machine in cfg["machines"].iteritems():
port = lnst_config.get_option("environment", "rpcport")
if test_tcp_connection(hostname, port):
s = socket.create_connection((hostname, port))
rpc_con = CtlSecSocket(s)
try:
rpc_con.handshake(machine["security"])
except SecSocketException:
logging.error("Failed authentication for machine %s" %\
hostname)
continue
rpc_msg = {
"type": "command",
"method_name": "machine_cleanup",
"args": []
}
logging.debug("Calling cleanup on slave '%s'" % hostname)
rpc_con.send_msg(rpc_msg)
while True:
msg = rpc_con.recv_msg()
if msg['type'] == 'result':
break
rpc_con.close()
if "libvirt_dom" in machine:
libvirt_dom = machine["libvirt_dom"]
domain_ctl = VirtDomainCtl(libvirt_dom)
logging.info("Detaching dynamically created interfaces.")
for i in machine["interfaces"]:
try:
domain_ctl.detach_interface(i)
except:
pass
logging.info("Removing dynamically created bridges.")
for br in cfg["bridges"]:
try:
net_ctl = VirtNetCtl(br)
net_ctl.cleanup()
except:
pass
os.remove("/tmp/.lnst_machine_conf")
def __init__(self,
recipe_path,
log_ctl,
res_serializer=None,
pool_checks=True,
packet_capture=False,
defined_aliases=None,
overriden_aliases=None,
reduce_sync=False):
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
self._recipe_path = Path(None, recipe_path).abs_path()
self._msg_dispatcher = MessageDispatcher(log_ctl)
self._packet_capture = packet_capture
self._reduce_sync = reduce_sync
self._parser = RecipeParser(recipe_path)
self.remove_saved_machine_config()
sp = SlavePool(lnst_config.get_option('environment', 'pool_dirs'),
pool_checks)
self._slave_pool = sp
self._machines = {}
self._network_bridges = {}
self._tasks = []
mac_pool_range = lnst_config.get_option('environment',
'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
self._parser.set_machines(self._machines)
self._parser.set_aliases(defined_aliases, overriden_aliases)
self._recipe = self._parser.parse()
mreq = self._get_machine_requirements()
sp.set_machine_requirements(mreq)
modules_dirs = lnst_config.get_option('environment', 'module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
self._resource_table = {}
self._resource_table["module"] = self._load_test_modules(modules_dirs)
self._resource_table["tools"] = self._load_test_tools(tools_dirs)
def connect_PerfRepo(self, mapping_file, url=None, username=None, password=None):
if not self._perf_repo_api.connected():
if url is None:
url = lnst_config.get_option("perfrepo", "url")
if username is None:
username = lnst_config.get_option("perfrepo", "username")
if password is None:
password = lnst_config.get_option("perfrepo", "password")
self._perf_repo_api.connect(url, username, password)
root = Path(None, self._ctl._recipe_path).get_root()
path = Path(root, mapping_file)
self._perf_repo_api.load_mapping(path)
if not self._perf_repo_api.connected():
logging.warn("Connection to PerfRepo incomplete, further "\
"PerfRepo commands will be ignored.")
return self._perf_repo_api
def restore_nm_option(self):
val = lnst_config.get_option("environment", "use_nm")
if val == self._bkp_nm_opt_val:
return val
logging.warning("=========================================")
logging.warning("Restoring use_nm option to original value")
logging.warning("=========================================")
lnst_config.set_option("environment", "use_nm", self._bkp_nm_opt_val)
return val
def get_result_xml(self):
impl = getDOMImplementation()
doc = impl.createDocument(None, "results", None)
xslt_url = lnst_config.get_option("environment", "xslt_url")
proc_inst = doc.createProcessingInstruction('xml-stylesheet',
'type="text/xsl" href="'+xslt_url+'"')
top_el = doc.documentElement
doc.insertBefore(proc_inst, top_el)
for recipe in self._results:
recipe_el = doc.createElement("recipe")
recipe_el.setAttribute("name", recipe["name"])
recipe_el.setAttribute("result", recipe["result"])
top_el.appendChild(recipe_el)
if recipe["result"] == "FAIL" and \
"err_msg" in recipe and recipe["err_msg"] != "":
err_el = doc.createElement("error_message")
err_text = doc.createTextNode(recipe["err_msg"])
err_el.appendChild(err_text)
recipe_el.appendChild(err_el)
for task in recipe["tasks"]:
cmd_seq_el = doc.createElement("task")
recipe_el.appendChild(cmd_seq_el)
for cmd, cmd_res in task:
command_el = doc.createElement("command")
cmd_seq_el.appendChild(command_el)
for key in cmd:
if key == "options":
continue
command_el.setAttribute(key, str(cmd[key]))
result_el = doc.createElement("result")
command_el.appendChild(result_el)
if cmd_res["passed"]:
res = "PASS"
else:
res = "FAIL"
result_el.setAttribute("result", res)
if "msg" in cmd_res and cmd_res["msg"]:
msg_el = doc.createElement("message")
msg_text = doc.createTextNode(cmd_res["msg"])
msg_el.appendChild(msg_text)
result_el.appendChild(msg_el)
if "res_data" in cmd_res and cmd_res["res_data"]:
res_data_el = doc.createElement("result_data")
serialize_obj(cmd_res["res_data"], doc, res_data_el)
command_el.appendChild(res_data_el)
return doc.toprettyxml()
def restore_nm_option(self):
val = lnst_config.get_option("environment", "use_nm")
if val == self._bkp_nm_opt_val:
return val
logging.warning("=========================================")
logging.warning("Restoring use_nm option to original value")
logging.warning("=========================================")
lnst_config.set_option("environment", "use_nm", self._bkp_nm_opt_val)
return val
def remove_saved_machine_config(cls):
#removes previously saved configuration
cfg = None
try:
with open("/tmp/.lnst_machine_conf", "rb") as f:
cfg = cPickle.load(f)
except:
logging.info("No previous configuration found.")
return
if cfg:
logging.info("Cleaning up leftover configuration from previous "\
"config_only run.")
for hostname, machine in cfg["machines"].iteritems():
port = lnst_config.get_option("environment", "rpcport")
if test_tcp_connection(hostname, port):
s = socket.create_connection((hostname, port))
rpc_con = CtlSecSocket(s)
try:
rpc_con.handshake(machine["security"])
except SecSocketException:
logging.error("Failed authentication for machine %s" %\
hostname)
continue
rpc_msg= {"type": "command",
"method_name": "machine_cleanup",
"args": []}
logging.debug("Calling cleanup on slave '%s'" % hostname)
rpc_con.send_msg(rpc_msg)
while True:
msg = rpc_con.recv_msg()
if msg['type'] == 'result':
break
rpc_con.close()
if "libvirt_dom" in machine:
libvirt_dom = machine["libvirt_dom"]
domain_ctl = VirtDomainCtl(libvirt_dom)
logging.info("Detaching dynamically created interfaces.")
for i in machine["interfaces"]:
try:
domain_ctl.detach_interface(i)
except:
pass
logging.info("Removing dynamically created bridges.")
for br in cfg["bridges"]:
try:
net_ctl = VirtNetCtl(br)
net_ctl.cleanup()
except:
pass
os.remove("/tmp/.lnst_machine_conf")
def get_nm_version():
if not check_process_running("NetworkManager") or not lnst_config.get_option("environment", "use_nm"):
return ""
bus = dbus.SystemBus()
nm_obj = bus.get_object(NM_BUS, OBJ_PRE)
nm_if = dbus.Interface(nm_obj, IF_PRE)
props = dbus.Interface(nm_obj, "org.freedesktop.DBus.Properties")
return props.Get(IF_PRE, "Version")
def get_result_html(self):
xslt_url = lnst_config.get_option("environment", "xslt_url")
xslt = etree.parse(xslt_url)
xml = self._generate_xml().toprettyxml()
etree_xml = etree.fromstring(xml)
transform = etree.XSLT(xslt)
transformed_xml = transform(etree_xml)
return "<!DOCTYPE html>\n" + str(transformed_xml)
def get_result_html(self):
xslt_url = lnst_config.get_option("environment", "xslt_url")
xslt = etree.parse(xslt_url)
xml = self._generate_xml().toprettyxml()
etree_xml = etree.fromstring(xml)
transform = etree.XSLT(xslt)
transformed_xml = transform(etree_xml)
return "<!DOCTYPE html>\n" + str(transformed_xml)
def get_nm_version():
if not check_process_running("NetworkManager") or\
not lnst_config.get_option("environment", "use_nm"):
return ""
bus = dbus.SystemBus()
nm_obj = bus.get_object(NM_BUS, OBJ_PRE)
nm_if = dbus.Interface(nm_obj, IF_PRE)
props = dbus.Interface(nm_obj, "org.freedesktop.DBus.Properties")
return props.Get(IF_PRE, "Version")
def __init__(self, recipe_path, log_ctl, cleanup=False,
res_serializer=None, pool_checks=True):
self._docleanup = cleanup
self._res_serializer = res_serializer
self._remote_capture_files = {}
self._log_ctl = log_ctl
self._recipe_path = recipe_path
self._msg_dispatcher = MessageDispatcher(log_ctl)
sp = SlavePool(lnst_config.get_option('environment', 'pool_dirs'),
check_process_running("libvirtd"), pool_checks)
self._slave_pool = sp
self._machines = {}
self._network_bridges = {}
self._recipe = recipe = {}
recipe["networks"] = {}
recipe["machines"] = {}
recipe["switches"] = {}
mac_pool_range = lnst_config.get_option('environment', 'mac_pool_range')
self._mac_pool = MacPool(mac_pool_range[0], mac_pool_range[1])
parser = RecipeParse(recipe_path)
parser.set_target(self._recipe)
parser.set_machines(self._machines)
parser.register_event_handler("provisioning_requirements_ready",
self._prepare_provisioning)
parser.register_event_handler("interface_config_ready",
self._prepare_interface)
modules_dirs = lnst_config.get_option('environment', 'module_dirs')
tools_dirs = lnst_config.get_option('environment', 'tool_dirs')
self._resource_table = {}
self._resource_table["module"] = self._load_test_modules(modules_dirs)
self._resource_table["tools"] = self._load_test_tools(tools_dirs)
self._parser = parser
def __init__(self, command_context, log_ctl, if_manager, net_namespaces,
server_handler, slave_server):
self._packet_captures = {}
self._if_manager = if_manager
self._command_context = command_context
self._log_ctl = log_ctl
self._net_namespaces = net_namespaces
self._server_handler = server_handler
self._slave_server = slave_server
self._capture_files = {}
self._copy_targets = {}
self._copy_sources = {}
self._system_config = {}
self._cache = ResourceCache(lnst_config.get_option("cache", "dir"),
lnst_config.get_option("cache", "expiration_period"))
self._resource_table = {'module': {}, 'tools': {}}
self._bkp_nm_opt_val = lnst_config.get_option("environment", "use_nm")
def __init__(self, command_context, log_ctl, if_manager, net_namespaces,
server_handler, slave_server):
self._packet_captures = {}
self._if_manager = if_manager
self._command_context = command_context
self._log_ctl = log_ctl
self._net_namespaces = net_namespaces
self._server_handler = server_handler
self._slave_server = slave_server
self._capture_files = {}
self._copy_targets = {}
self._copy_sources = {}
self._system_config = {}
self._cache = ResourceCache(
lnst_config.get_option("cache", "dir"),
lnst_config.get_option("cache", "expiration_period"))
self._resource_table = {'module': {}, 'tools': {}}
self._bkp_nm_opt_val = lnst_config.get_option("environment", "use_nm")
def _prepare_machine(self, m_id):
machine = self._machines[m_id]
address = socket.gethostbyname(machine.get_hostname())
self._log_ctl.add_slave(m_id, address)
port = lnst_config.get_option('environment', 'rpcport')
machine.set_rpc(self._msg_dispatcher, port)
machine.set_mac_pool(self._mac_pool)
machine.set_network_bridges(self._network_bridges)
recipe_name = os.path.basename(self._recipe_path)
machine.configure(recipe_name, self._docleanup)
machine.sync_resources(self._resource_table)
def _prepare_machine(self, m_id):
machine = self._machines[m_id]
address = socket.gethostbyname(machine.get_hostname())
self._log_ctl.add_slave(m_id, address)
port = lnst_config.get_option('environment', 'rpcport')
machine.set_rpc(self._msg_dispatcher, port)
machine.set_mac_pool(self._mac_pool)
machine.set_network_bridges(self._network_bridges)
recipe_name = os.path.basename(self._recipe_path)
machine.configure(recipe_name, self._docleanup)
machine.sync_resources(self._resource_table)
def add_file(self, filepath):
if os.path.isfile(filepath) and re.search("\.xml$", filepath, re.I):
dirname, basename = os.path.split(filepath)
m_id = re.sub("\.[xX][mM][lL]$", "", basename)
parser = SlaveMachineParser(filepath)
xml_data = parser.parse()
machine_spec = self._process_machine_xml_data(m_id, xml_data)
# Check if there isn't any machine with the same
# hostname or libvirt_domain already in the pool
for pm_id, m in self._pool.iteritems():
pm = m["params"]
rm = machine_spec["params"]
if pm["hostname"] == rm["hostname"]:
msg = "You have the same machine listed twice in " "your pool ('%s' and '%s')." % (m_id, pm_id)
raise SlaveMachineError(msg)
if "libvirt_domain" in rm and "libvirt_domain" in pm and pm["libvirt_domain"] == rm["libvirt_domain"]:
msg = "You have the same libvirt_domain listed twice in " "your pool ('%s' and '%s')." % (
m_id,
pm_id,
)
raise SlaveMachineError(msg)
if self._pool_checks:
available = False
hostname = machine_spec["params"]["hostname"]
if "rpc_port" in machine_spec["params"]:
port = machine_spec["params"]["rpc_port"]
else:
port = lnst_config.get_option("environment", "rpcport")
logging.debug("Querying machine '%s': %s:%s" % (m_id, hostname, port))
if test_tcp_connection(hostname, port):
available = True
if "libvirt_domain" in machine_spec["params"] and not self._allow_virt:
logging.debug("libvirtd not running. Removing " "libvirt_domain from machine '%s'" % m_id)
del machine_spec["params"]["libvirt_domain"]
if available:
self._pool[m_id] = machine_spec
return (m_id, available)
def __init__(self, schema_file, xml_path):
# locate the schema file
# try git path
dirname = os.path.dirname(sys.argv[0])
schema_path = os.path.join(dirname, schema_file)
if not os.path.exists(schema_path):
# try configuration
res_dir = lnst_config.get_option("environment", "resource_dir")
schema_path = os.path.join(res_dir, schema_file)
if not os.path.exists(schema_path):
raise Exception("The recipe schema file was not found. " + "Your LNST installation is corrupt!")
self._template_proc = XmlTemplates()
self._path = xml_path
relaxng_doc = etree.parse(schema_path)
self._schema = etree.RelaxNG(relaxng_doc)
def is_nm_managed_by_name(dev_name):
if not check_process_running("NetworkManager") or not lnst_config.get_option("environment", "use_nm"):
return False
bus = dbus.SystemBus()
nm_obj = bus.get_object(NM_BUS, OBJ_PRE)
nm_if = dbus.Interface(nm_obj, IF_PRE)
try:
device_obj_path = nm_if.GetDeviceByIpIface(dev_name)
except:
# There is a higher possibility that if the interface doesn't exist
# it's a software interface that can be created by NM so we say that it's
# managed and check existance of physical interfaces sepparately
return True
dev = bus.get_object(NM_BUS, device_obj_path)
dev_props = dbus.Interface(dev, "org.freedesktop.DBus.Properties")
return dev_props.Get(IF_PRE + ".Device", "Managed")
def hello(self, recipe_path):
logging.info("Recieved a controller connection.")
self.clear_resource_table()
self._cache.del_old_entries()
self.reset_file_transfers()
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
self._log_ctl.set_recipe(recipe_path, expand=date)
sleep(1)
if check_process_running("NetworkManager"):
logging.warning("=============================================")
logging.warning("NetworkManager is running on a slave machine!")
if lnst_config.get_option("environment", "use_nm"):
logging.warning("Support of NM is still experimental!")
else:
logging.warning("Usage of NM is disabled!")
logging.warning("=============================================")
return "hello"
def __init__(self, schema_file, xml_path):
# locate the schema file
# try git path
dirname = os.path.dirname(sys.argv[0])
schema_path = os.path.join(dirname, schema_file)
if not os.path.exists(schema_path):
# try configuration
res_dir = lnst_config.get_option("environment", "resource_dir")
schema_path = os.path.join(res_dir, schema_file)
if not os.path.exists(schema_path):
raise Exception("The recipe schema file was not found. " + \
"Your LNST installation is corrupt!")
self._template_proc = XmlTemplates()
self._path = xml_path
relaxng_doc = etree.parse(schema_path)
self._schema = etree.RelaxNG(relaxng_doc)
def hello(self, recipe_path):
logging.info("Recieved a controller connection.")
self.clear_resource_table()
self._cache.del_old_entries()
self.reset_file_transfers()
date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
self._log_ctl.set_recipe(recipe_path, expand=date)
sleep(1)
if check_process_running("NetworkManager"):
logging.warning("=============================================")
logging.warning("NetworkManager is running on a slave machine!")
if lnst_config.get_option("environment", "use_nm"):
logging.warning("Support of NM is still experimental!")
else:
logging.warning("Usage of NM is disabled!")
logging.warning("=============================================")
return "hello"
def is_nm_managed_by_name(dev_name):
if not check_process_running("NetworkManager") or\
not lnst_config.get_option("environment", "use_nm"):
return False
bus = dbus.SystemBus()
nm_obj = bus.get_object(NM_BUS, OBJ_PRE)
nm_if = dbus.Interface(nm_obj, IF_PRE)
try:
device_obj_path = nm_if.GetDeviceByIpIface(dev_name)
except:
#There is a higher possibility that if the interface doesn't exist
#it's a software interface that can be created by NM so we say that it's
#managed and check existance of physical interfaces sepparately
return True
dev = bus.get_object(NM_BUS, device_obj_path)
dev_props = dbus.Interface(dev, "org.freedesktop.DBus.Properties")
return dev_props.Get(IF_PRE + ".Device", "Managed")
def load_presets_from_config():
for preset_name in PRESETS:
preset = lnst_config.get_option("colours", preset_name)
if preset == None:
continue
fg, bg, bf = preset
extended_re = "^extended\([0-9]+\)$"
if fg == "default":
fg = None
elif not re.match(extended_re, fg) and fg not in COLOURS.keys():
raise Exception("Colour '%s' not supported" % fg)
if bg == "default":
bg = None
elif not re.match(extended_re, bg) and bg not in COLOURS.keys():
raise Exception("Colour '%s' not supported" % bg)
PRESETS[preset_name] = [fg, bg, bool_it(bf)]
def load_presets_from_config():
for preset_name in PRESETS:
preset = lnst_config.get_option("colours", preset_name)
if preset == None:
continue
fg, bg, bf = preset
extended_re = "^extended\([0-9]+\)$"
if fg == "default":
fg = None
elif not re.match(extended_re, fg) and fg not in COLOURS.keys():
raise Exception("Colour '%s' not supported" % fg)
if bg == "default":
bg = None
elif not re.match(extended_re, bg) and bg not in COLOURS.keys():
raise Exception("Colour '%s' not supported" % bg)
PRESETS[preset_name] = [fg, bg, bool_it(bf)]
def add_file(self, filepath):
if os.path.isfile(filepath) and re.search("\.xml$", filepath, re.I):
dom_init = XmlDomTreeInit()
dom = dom_init.parse_file(filepath)
dirname, basename = os.path.split(filepath)
parser = SlaveMachineParse()
parser.set_include_root(dirname)
parser.disable_events()
machine = {"params": {}, "interfaces": {}}
m_id = re.sub("\.[xX][mM][lL]$", "", basename)
parser.set_machine(m_id, machine)
slavemachine = dom.getElementsByTagName("slavemachine")[0]
parser.parse(slavemachine)
if self._pool_checks:
hostname = machine["params"]["hostname"]
if "rpcport" in machine:
port = machine["params"]["rpcport"]
else:
port = lnst_config.get_option('environment', 'rpcport')
logging.info("Querying slave machine %s." % m_id)
if not test_tcp_connection(hostname, port):
msg = "Machine '%s' not responding. Skipping." % m_id
logging.warning(msg)
return
if 'libvirt_domain' in machine['params'] and \
not self._allow_virt:
msg = "libvird not running. Skipping machine '%s'." % m_id
logging.warning(msg)
logging.info("Adding slave machine %s to slave pool." % m_id)
self._pool[m_id] = machine
def __init__(self, m_id, hostname=None, libvirt_domain=None, rpcport=None):
self._id = m_id
self._hostname = hostname
self._connection = None
self._configured = False
self._system_config = {}
self._domain_ctl = None
self._network_bridges = None
self._libvirt_domain = libvirt_domain
if libvirt_domain:
self._domain_ctl = VirtDomainCtl(libvirt_domain)
if rpcport:
self._port = rpcport
else:
self._port = lnst_config.get_option('environment', 'rpcport')
self._msg_dispatcher = None
self._mac_pool = None
self._interfaces = []
def add_file(self, filepath):
if os.path.isfile(filepath) and re.search("\.xml$", filepath, re.I):
dom_init = XmlDomTreeInit()
dom = dom_init.parse_file(filepath)
dirname, basename = os.path.split(filepath)
parser = SlaveMachineParse()
parser.set_include_root(dirname)
parser.disable_events()
machine = {"params": {}, "interfaces": {}}
m_id = re.sub("\.[xX][mM][lL]$", "", basename)
parser.set_machine(m_id, machine)
slavemachine = dom.getElementsByTagName("slavemachine")[0]
parser.parse(slavemachine)
if self._pool_checks:
hostname = machine["params"]["hostname"]
if "rpcport" in machine:
port = machine["params"]["rpcport"]
else:
port = lnst_config.get_option("environment", "rpcport")
logging.info("Querying slave machine %s." % m_id)
if not test_tcp_connection(hostname, port):
msg = "Machine '%s' not responding. Skipping." % m_id
logging.warning(msg)
return
if "libvirt_domain" in machine["params"] and not self._allow_virt:
msg = "libvird not running. Skipping machine '%s'." % m_id
logging.warning(msg)
logging.info("Adding slave machine %s to slave pool." % m_id)
self._pool[m_id] = machine
def add_dir(self, pool_name, dir_path):
logging.info("Processing pool '%s', directory '%s'" %
(pool_name, dir_path))
pool = self._pools[pool_name]
try:
dentries = os.listdir(dir_path)
except OSError:
logging.warn("Directory '%s' does not exist for pool '%s'" %
(dir_path, pool_name))
return
for dirent in dentries:
m_id, m = self.add_file(pool_name, dir_path, dirent)
if m_id != None and m != None:
pool[m_id] = m
if len(pool) == 0:
logging.warn("No machines found in pool '%s', directory '%s'" %
(pool_name, dir_path))
max_len = 0
for m_id in pool.keys():
if len(m_id) > max_len:
max_len = len(m_id)
if self._pool_checks:
check_sockets = {}
for m_id, m in sorted(pool.iteritems()):
hostname = m["params"]["hostname"]
if "rpc_port" in m["params"]:
port = m["params"]["rpc_port"]
else:
port = lnst_config.get_option('environment', 'rpcport')
logging.debug("Querying machine '%s': %s:%s" %\
(m_id, hostname, port))
s = socket.socket()
s.settimeout(0)
try:
s.connect((hostname, port))
except socket.error as msg:
# if the error is other than EINPROGRESS, e.g. the stack
# could not resolve name, the machine should become unavailable
try:
en = msg.errno
except AttributeError:
en = 0
if en != errno.EINPROGRESS:
pool[m_id]["available"] = False
s.close()
logging.debug("Bypassing machine '%s' (%s)" %
(m_id, msg))
continue
check_sockets[s] = m_id
while len(check_sockets) > 0:
rl, wl, el = select.select([], check_sockets.keys(), [])
for s in wl:
err = s.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
m_id = check_sockets[s]
if err == 0:
pool[m_id]["available"] = True
s.shutdown(socket.SHUT_RDWR)
s.close()
del check_sockets[s]
else:
pool[m_id]["available"] = False
s.close()
del check_sockets[s]
else:
for m_id in pool.keys():
pool[m_id]["available"] = True
for m_id in sorted(list(pool.keys())):
m = pool[m_id]
if m["available"]:
if 'libvirt_domain' in m['params']:
libvirt_msg = " libvirt_domain: %s" %\
m['params']['libvirt_domain']
else:
libvirt_msg = ""
msg = "%s%s [%s] %s" % (m_id, (max_len - len(m_id)) * " ",
decorate_with_preset(
"UP", "pass"), libvirt_msg)
else:
msg = "%s%s [%s]" % (m_id, (max_len - len(m_id)) * " ",
decorate_with_preset("DOWN", "fail"))
del pool[m_id]
logging.info(msg)
def add_dir(self, pool_name, dir_path):
logging.info("Processing pool '%s', directory '%s'" % (pool_name,
dir_path))
pool = self._pools[pool_name]
try:
dentries = os.listdir(dir_path)
except OSError:
logging.warn("Directory '%s' does not exist for pool '%s'" %
(dir_path,
pool_name))
return
for dirent in dentries:
m_id, m = self.add_file(pool_name, dir_path, dirent)
if m_id != None and m != None:
pool[m_id] = m
if len(pool) == 0:
logging.warn("No machines found in pool '%s', directory '%s'" %
(pool_name,
dir_path))
max_len = 0
for m_id in pool.keys():
if len(m_id) > max_len:
max_len = len(m_id)
if self._pool_checks:
check_sockets = {}
for m_id, m in sorted(pool.iteritems()):
hostname = m["params"]["hostname"]
if "rpc_port" in m["params"]:
port = m["params"]["rpc_port"]
else:
port = lnst_config.get_option('environment', 'rpcport')
logging.debug("Querying machine '%s': %s:%s" %\
(m_id, hostname, port))
s = socket.socket()
s.settimeout(0)
try:
s.connect((hostname, port))
except socket.error as msg:
# if the error is other than EINPROGRESS, e.g. the stack
# could not resolve name, the machine should become unavailable
try:
en = msg.errno
except AttributeError:
en = 0
if en != errno.EINPROGRESS:
pool[m_id]["available"] = False
s.close()
logging.debug("Bypassing machine '%s' (%s)" %
(m_id, msg))
continue
check_sockets[s] = m_id
while len(check_sockets) > 0:
rl, wl, el = select.select([], check_sockets.keys(), [])
for s in wl:
err = s.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
m_id = check_sockets[s]
if err == 0:
pool[m_id]["available"] = True
s.shutdown(socket.SHUT_RDWR)
s.close()
del check_sockets[s]
else:
pool[m_id]["available"] = False
s.close()
del check_sockets[s]
else:
for m_id in pool.keys():
pool[m_id]["available"] = True
for m_id in sorted(list(pool.keys())):
m = pool[m_id]
if m["available"]:
if 'libvirt_domain' in m['params']:
libvirt_msg = " libvirt_domain: %s" %\
m['params']['libvirt_domain']
else:
libvirt_msg = ""
msg = "%s%s [%s] %s" % (m_id, (max_len - len(m_id)) * " ",
decorate_with_preset("UP", "pass"),
libvirt_msg)
else:
msg = "%s%s [%s]" % (m_id, (max_len - len(m_id)) * " ",
decorate_with_preset("DOWN", "fail"))
del pool[m_id]
logging.info(msg)
def add_dir(self, dir_path):
logging.info("Processing pool dir '%s'" % dir_path)
pool_dir = self._pool_dirs[dir_path]
dentries = os.listdir(dir_path)
for dirent in dentries:
m_id, m = self.add_file(dir_path, dirent)
if m_id != None and m != None:
pool_dir[m_id] = m
if len(pool_dir) == 0:
logging.warn("No machines found in this directory")
max_len = 0
for m_id in pool_dir.keys():
if len(m_id) > max_len:
max_len = len(m_id)
if self._pool_checks:
check_sockets = {}
for m_id, m in pool_dir.iteritems():
hostname = m["params"]["hostname"]
if "rpc_port" in m["params"]:
port = m["params"]["rpc_port"]
else:
port = lnst_config.get_option('environment', 'rpcport')
logging.debug("Querying machine '%s': %s:%s" %\
(m_id, hostname, port))
s = socket.socket()
s.settimeout(0)
try:
s.connect((hostname, port))
except:
pass
check_sockets[s] = m_id
while len(check_sockets) > 0:
rl, wl, el = select.select([], check_sockets.keys(), [])
for s in wl:
err = s.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
m_id = check_sockets[s]
if err == 0:
pool_dir[m_id]["available"] = True
del check_sockets[s]
else:
pool_dir[m_id]["available"] = False
del check_sockets[s]
else:
for m_id in pool_dir.keys():
pool_dir[m_id]["available"] = True
for m_id in list(pool_dir.keys()):
m = pool_dir[m_id]
if m["available"]:
if 'libvirt_domain' in m['params']:
libvirt_msg = " libvirt_domain: %s" %\
m['params']['libvirt_domain']
else:
libvirt_msg = ""
msg = "%s%s [%s] %s" % (m_id, (max_len - len(m_id)) * " ",
decorate_with_preset(
"UP", "pass"), libvirt_msg)
else:
msg = "%s%s [%s]" % (m_id, (max_len - len(m_id)) * " ",
decorate_with_preset("DOWN", "fail"))
del pool_dir[m_id]
logging.info(msg)
