def run_shell_with_input(cmd, _input):
"""
Run the shell command `cmd` and supply `_input` as stdin.
Parameters
----------
cmd : str
_input : str
Returns
-------
str
Raises
------
MyCalledProcessError
"""
log.info("'%s <<<\"%s\"'", cmd, _input)
cmd = shlex.split(cmd)
p = subprocess.Popen(cmd,
close_fds=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
stdout, stderr = p.communicate(_input.encode())
if p.returncode == 0:
return stdout
else:
raise MyCalledProcessError(p.returncode, p.args, output=stderr)
def init_req_socket(self):
addr = 'tcp://{}:{}'.format(self.server_addr, Protocol.PORT_DEFAULT_SERVICE)
log.info("connecting to: %s ...", addr)
self.svc = self.context.socket(zmq.REQ)
# self.svc.setsockopt(zmq.REQ_CORRELATE, 1)
# self.svc.setsockopt(zmq.REQ_RELAXED, 1)
self.svc.connect(addr)
def start_management_node(self):
"""
Start the management switch and connect all other nodes to it.
Also store a reference in the :py:class:`.NetworkManager`.
Returns
-------
ManagementNode
"""
network_backend_bootstrapper = NetworkBackends.get_current_network_backend_bootstrapper(
)
if config.is_management_switch_enabled():
# late import needed
from miniworld.management.network.manager import NetworkManager
log.info("creating management node/switch ...")
if network_backend_bootstrapper.management_node_type is None:
log.info("Network Backend has no management node")
return None
management_node = network_backend_bootstrapper.management_node_type(
network_backend_bootstrapper)
management_node.start(switch=True,
bridge_dev_name=config.get_bridge_tap_name())
for node in self.nodes:
management_node.connect_to_emu_node(singletons.network_backend,
node)
NetworkManager.management_node = management_node
return management_node
def factory():
if config.is_protocol_zeromq_mode_mcast():
log.info("distance matrix distribution via publish-subscribe pattern")
return ZeroMQCServerPubSub
else:
log.info("distance matrix distribution via request-reply pattern")
return ZeroMQServerRouter
def get_all_connections(self):
"""
Get all the connections the nodes will have to each other.
Returns
-------
dict<int, set<int>>
The connections each node has. Sorted by id asc. id number.
Fully staffed matrix.
"""
# we need a lock here, otherwise all threads (each node) try to read the config file simultaneously!
# used by each :py:class:`.EmulationNode`
with static_lock:
if self._all_connections is not None:
return self._all_connections
core_scenarios = scenario_config.get_core_scenarios()
all_connections = defaultdict(set)
for _, core_scenario_path in core_scenarios:
connections_dict = CoreConfigFileParser.parse_core_config_file(
core_scenario_path)
for k, v in connections_dict.items():
all_connections[k].update(v)
all_connections = DictUtil.to_fully_staffed_matrix(
all_connections)
self._all_connections = all_connections
log.info("connections: %s", pformat(all_connections))
return all_connections
def create_n_check_tmp_files(delete_first=False, create_ramdisk=False):
""" Create tmp dirs if not existing yet.
Raises
------
AlreadyRunning
"""
if os.path.exists(PATH_TMP):
if delete_first:
try:
shutil.rmtree(PATH_LOGS)
except BaseException:
pass
# umount ramdisk
try_umount_ramdisk()
else:
# Ticket #12
raise AlreadyRunning("Another instance of %s might be running! Did you invoke the cleanup script '%s'?" % (PROJECT_NAME, PATH_CLEANUP_SCRIPT))
if not os.path.exists(PATH_TMP):
log.debug("Creating directory '%s'!", PATH_TMP)
os.makedirs(PATH_TMP)
if create_ramdisk:
if not os.path.ismount(PATH_TMP):
log.info("creating ramdisk at '%s'", PATH_TMP)
# TODO: better way for ramdisk creation?
run_shell("sudo mount -t ramfs ramfs {}".format(PATH_TMP))
else:
log.info("ramdisk still exists at '%s' ... ", PATH_TMP)
def try_umount_ramdisk():
# umount ramdisk
try:
log.info("unmounting ramdisk (if one is mounted)")
umount_ramdisk()
except BaseException:
pass
def start(self, cnt_peers):
"""
Start the server by expecting the clients to register in the first state.
This method controls the whole communication expect the distribution of the distance matrix.
Parameters
----------
cnt_peers : int
Returns
-------
"""
log.info("waiting for %d servers ...", cnt_peers)
self.cnt_peers = cnt_peers
self.handle_state_register()
# wait until the scenario config is set, because the tunnel addresses
# (received in the next step) are written to it
log.info("waiting until scenario config is set ...")
self.wait_for_scenario_config.wait()
self.handle_state_information_exchange()
# TODO: let NetworkBackend inject communication steps for needed information
self.handle_state_start_nodes()
def handle_select(rlist, xlist):
if xlist:
raise ZeroMQException("Unknown error occurred during a select() call")
if self.reset_socket in rlist:
self.reset_socket.recv()
self.reset()
return True
if self.sub_socket in rlist:
local_distance_matrix = self.recv_distance_matrix()
if config.is_debug():
log.info("received distance matrix: %s", local_distance_matrix)
log.info("step ...")
singletons.simulation_manager.step(1, distance_matrix=local_distance_matrix)
self.send_sync()
# wait for sync reply or error
rlist, _, xlist = zmq.select([self.reset_socket, self.svc], [], [])
# rlist, _, xlist = zmq.select([self.svc], [], [])
if handle_select(rlist, xlist):
return True
# self.sync()
if config.is_debug():
log.debug("stepped ...")
if self.svc in rlist:
self.recv_sync()
return True
def start_bg_checker_thread(self):
return
self.bg_checker_thread = ExceptionStopThread.run_fun_threaded_n_log_exception(
target=self.check_error_codes,
tkwargs=dict(name="BG Process Checker"))
self.bg_checker_thread.start()
log.info("starting bg process checker thread ...")
def myprint(self, str):
"""
Include the server id in log messages
Parameters
----------
str : str
"""
log.info('%s: %s', self.server_id, str)
def _handle_state_distance_matrix(self, distance_matrix):
# # sync initially
if singletons.simulation_manager.current_step == 0:
log.info("syncing clients initially ...")
self.sync_subscribers()
self.send_distance_matrix(distance_matrix)
self.sync_subscribers()
def __init__(self, *args, **kwargs):
super(ZeroMQCServerPubSub, self).__init__(*args, **kwargs)
# create the publish socket
self.pub_socket = self.context.socket(zmq.PUB)
addr = "tcp://*:{}".format(Protocol.PORT_PUB_SERVICE)
self.pub_socket.bind(addr)
log.info("listening on '%s'", addr)
def handle_state_start_nodes(self):
"""
Start the nodes and sync them afterwards. Finally notify that :py:meth:`.handle_state_distance_matrix` can continue.
"""
log.info("state: %s", States.STATE_START_NODES)
expect_state = self.get_expecter_state(States.STATE_START_NODES, 1)
ResponderArgument(self.router_socket, self.protocol, expect_state, '')()
self.wait_for_nodes_started.set()
def __init__(self, *args, **kwargs):
super(ZeroMQClientSub, self).__init__(*args, **kwargs)
self.sub_socket = self.context.socket(zmq.SUB)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, b'')
addr = "tcp://{}:{}".format(self.server_addr, Protocol.PORT_PUB_SERVICE)
self.sub_socket.connect(addr)
log.info("connecting to: %s ...", addr)
def handle_state_distance_matrix(self, distance_matrix):
if self.last_step_time is not None:
et = time.time() - self.last_step_time
log.info("took %0.2f seconds (previous distribution + sync)", et)
self.last_step_time = time.time()
self.wait_for_nodes_started.wait()
self._handle_state_distance_matrix(distance_matrix)
def start(self):
log.info("starting log writer (file: '%s')", self.log_filename)
if self.fh_logfile is None:
self.fh_logfile = open(
PathUtil.get_log_file_path(self.log_filename), "w")
self.writer_thread = ExceptionStopThread.run_fun_threaded_n_log_exception(
target=self.check, tkwargs=dict(name="LogWriter"))
self.writer_thread.daemon = True
self.writer_thread.start()
def send_distance_matrix(self, distance_matrix):
"""
Send the distance matrix via the publish socket.
Parameters
----------
distance_matrix : DistanceMatrix
"""
data = self.serialize(DistanceMatrix.transform_distance_matrix(distance_matrix))
log.info("sending %f kbytes ...", len(data) / 1024.0)
self.pub_socket.send(data)
def print_overall_node_status(self):
""" Print the nodes not ready yet """
while not self.event_nodes_started.is_set():
nodes_not_ready = self.nodes_not_ready()
if nodes_not_ready:
log.info("waiting for nodes: %s ...",
', '.join(map(str, nodes_not_ready)))
self.event_nodes_started.wait(TIME_NODE_STATUS_REFRESH)
else:
break
def after_simulation_step(self, simulation_manager, step_cnt, network_backend, emulation_nodes, **kwargs):
"""
Notify about events, if registered for events.
"""
# notify script about new events
event_hook_script_path = scenario_config.get_network_backend_event_hook_script()
if event_hook_script_path and os.path.exists(event_hook_script_path):
bridge_list = ' '.join(self.event_monitor.new_bridges)
log.info("notified event_hook_script: %s" % check_output([event_hook_script_path, bridge_list],
cwd=dirname(event_hook_script_path)))
self.reset_simulation_step()
def handle_state_register(self):
"""
Wait until all clients have been registered and sync them afterwards.
"""
# show registered clients ...
def register_fun(expecter, idx, cnt):
log.info("%d/%d clients registered", idx, cnt)
# let clients register and sync them
log.info("state: %s", States.STATE_REGISTER)
expect_state = self.get_expecter_state(States.STATE_REGISTER, 0, after_response_fun=register_fun)
ResponderServerID(self.router_socket, self.protocol, expect_state)()
def apply_nic_check_commands(self, check_commands_per_node):
es = singletons.event_system
# check the connectivity
if self.is_connectivity_checks_enabled:
with es.event_init(es.EVENT_NETWORK_CHECK) as ev:
log.info("Doing connectivity checks ...")
# NOTE: do not stress the network
self.run_emulation_node_commands(check_commands_per_node, ev)
else:
with es.event_init(es.EVENT_NETWORK_CHECK) as ev:
ev.finish()
def get_nic_check_commands(self, connections):
"""
Do link checking connection-based, there check which nodes are reachable!
In the distributed mode, we perform link checking on both sides!
Parameters
----------
connections
Returns
-------
"""
log.info("assuming bidirectional links ...")
check_commands_per_node = defaultdict(list)
for emulation_nodes, list_ifaces in connections.items():
if not emulation_nodes.filter_real_emulation_nodes():
break
# NOTE: for the distributed mode, we have to check which node is local and which is remote
# because connections might not be fully staffed, we need to perform the check from the local node to the remote node!
emulation_node_x, emulation_node_y = emulation_nodes.sort_by_locality(
)
for interfaces in list_ifaces:
if not interfaces.filter_normal_interfaces():
break
interface_x, interface_y = interfaces
def add_check_cmd(ip_addr):
ping_cmd = self.connectivity_checker_fun(
ip_addr, self.network_timeout)
check_commands_per_node[emulation_node_x].append(ping_cmd)
# for CentralNode let each node ping all allocated ips
if EmulationNodes([emulation_node_y]).filter_central_nodes():
for ip_addr in self.ips.values():
add_check_cmd(ip_addr)
# check which ip is allocated to emulation_node_y
else:
ip_addr = self.ips[self.get_key_ip_dict(
emulation_node_y, interface_y)]
add_check_cmd(ip_addr)
return check_commands_per_node
def set_global_config(*args, **kwargs):
""" Set the global config.
Returns
-------
dict
The config as JSON.
"""
_config = JSONConfig.read_json_config(*args, **kwargs)
from miniworld.log import log
log.info("setting global config file '%s'", *args, **kwargs)
config.data = deepcopy(_config)
return _config
def handle_state_information_exchange(self):
"""
Let clients send their tunnel Ip addr and send each the scenario config.
"""
log.info("state: %s", States.STATE_EXCHANGE)
expect_state = self.get_expecter_state(States.STATE_EXCHANGE, 3)
expect_state.expect_for_all()
tunnel_addresses_dict = expect_state.get_message_part_per_node_id(arg_nr=1)
server_score_dict = expect_state.get_message_part_per_node_id(arg_nr=2)
singletons.node_distribution_strategy.server_score = server_score_dict
log.info("server scores: %s", pformat(server_score_dict))
# set tunnel addresses in scenario config
Scenario.scenario_config.set_network_backend_bridged_tunnel_endpoints(tunnel_addresses_dict)
# create scenario configs
node_ids = miniworld.Scenario.scenario_config.get_local_node_ids()
# distribute nodes among emulation server
server_node_mapping = singletons.node_distribution_strategy.distribute_emulation_nodes(list(node_ids),
self.cnt_peers)
log.info("nodes mapping: %s", pformat(server_node_mapping))
log.info("nodes per server: %s", pformat({k: len(v) for k, v in server_node_mapping.items()}))
# set server to node mapping before creating the scenario configs
miniworld.Scenario.scenario_config.set_distributed_server_node_mapping(server_node_mapping)
# create for each server a custom scenario config
scenario_configs = singletons.simulation_manager.create_scenario_configs(server_node_mapping)
responder = ResponderPerServerID(self.router_socket, self.protocol, expect_state, scenario_configs)
responder.respond()
def _start(self):
# TODO: use ShellCommandSerializer
tunnel_cmd = IPRoute2Commands.get_gretap_tunnel_cmd(self.get_tunnel_name(), self.remote_ip,
self.get_tunnel_id())
log.info("creating gretap tunnel for %s<->%s" % (self.emulation_node_x.id, self.emulation_node_y.id))
self.add_command(self.EVENT_TUNNEL_ADD, tunnel_cmd)
log.info("changing NIC MTU to '%d' ... ", GRETAP_MTU)
self.get_local_emulation_node().virtualization_layer.set_link_mtus(mtu=GRETAP_MTU)
# set tunnel dev group
super(GreTapTunnel, self)._start()
def factory():
"""
Use the config system to choose the zeromq client type.
Returns
-------
type
"""
if config.is_protocol_zeromq_mode_mcast():
log.info("distance matrix distribution via publish-subscribe pattern")
return ZeroMQClientSub
else:
log.info("distance matrix distribution via request-reply pattern")
return ZeroMQClientReq
def create_n_connect_central_nodes(self, interfaces):
"""
Parameters
----------
interfaces
Returns
-------
dict<int, CentralNode>
"""
from miniworld.model.singletons.Singletons import singletons
# create CentralNode s but only if there is a HubWiFi interface
# TODO: REMOVE
cnt = 0
central_nodes_dict = {}
# connect local devices
for _if in filter(lambda x: is_central_node_interface(x), interfaces):
if cnt == 1:
raise ValueError("Only one '%s' interface supported at the moment!" % HubWiFi)
# TODO: REFACTOR!
# TODO: #54: make amount of nodes configurable
count_central_nodes = 1
for i in range(0, count_central_nodes):
central_node = self.network_backend_bootstrapper.central_node_type(
self.network_backend_bootstrapper, id=i + 1)
# central_node.id = self.get_br_name(central_node.id, central_node.interface)
# TODO: #54 make configurable!
log.debug("creating CentralNode with id: %s", central_node.id)
central_node.start(switch=False, bridge_dev_name=central_node.id)
central_nodes_dict[central_node.id] = central_node
# remember new bridges
self.event_monitor.add_new_bridge(central_node.id)
cnt += 1
# connect via server boundaries (overlay)
node_ids = singletons.simulation_manager.get_emulation_node_ids()
for x, y in zip(node_ids, node_ids[1:]):
emulation_node_x = singletons.simulation_manager.get_emulation_node_for_idx(x)
emulation_node_y = singletons.simulation_manager.get_emulation_node_for_idx(y)
log.info("connecting %s<->%s", emulation_node_x, emulation_node_y)
self.connection_across_servers(self, emulation_node_x, emulation_node_y)
return central_nodes_dict
def ip_config(self):
# NOTE: we need to reuse the existing configurators due to their internal state!
if scenario_config.is_network_links_auto_ipv4():
log.info("using ip provisioner: %s" % scenario_config.get_network_provisioner_name())
new_connections = self.get_new_connections_with_interfaces_since_last_distance_matrix_change()
with open(PathUtil.get_log_file_path("%s.txt" % self.__class__.__name__), "a") as f:
if self.net_configurator.needs_reconfiguration(singletons.simulation_manager.current_step):
# only connection setup and check for new connections
log.info("%s: configuring network ...", self.net_configurator.__class__.__name__)
commands_per_node = self.net_configurator.get_nic_configuration_commands(new_connections)
self.net_configurator.apply_nic_configuration_commands(commands_per_node)
f.write("setup_commands: %s\n" % pformat(dict(commands_per_node)))
def sync_subscribers(self):
"""
Sync the subscribers and provide some debugging information about the sync progress (if debug mode)
"""
log.info("syncing subscribers ...")
def fun(expecter, idx, cnt):
log.debug("%d/%d clients synced ...", idx, cnt)
# add some debug info
after_response_fun = fun if config.is_debug() else lambda x, y, z: None
expect_distance_matrix = self.get_expecter_state(States.STATE_DISTANCE_MATRIX, 1, after_response_fun)
# sync clients and send each his distance matrix
ResponderArgument(self.router_socket, self.protocol, expect_distance_matrix, '')()
log.info("syncing subscribers [done]")
