def run_shell_with_input(cmd, _input): """ Run the shell command `cmd` and supply `_input` as stdin. Parameters ---------- cmd : str _input : str Returns ------- str Raises ------ MyCalledProcessError """ log.info("'%s <<<\"%s\"'", cmd, _input) cmd = shlex.split(cmd) p = subprocess.Popen(cmd, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) stdout, stderr = p.communicate(_input.encode()) if p.returncode == 0: return stdout else: raise MyCalledProcessError(p.returncode, p.args, output=stderr)
def init_req_socket(self): addr = 'tcp://{}:{}'.format(self.server_addr, Protocol.PORT_DEFAULT_SERVICE) log.info("connecting to: %s ...", addr) self.svc = self.context.socket(zmq.REQ) # self.svc.setsockopt(zmq.REQ_CORRELATE, 1) # self.svc.setsockopt(zmq.REQ_RELAXED, 1) self.svc.connect(addr)
def start_management_node(self): """ Start the management switch and connect all other nodes to it. Also store a reference in the :py:class:`.NetworkManager`. Returns ------- ManagementNode """ network_backend_bootstrapper = NetworkBackends.get_current_network_backend_bootstrapper( ) if config.is_management_switch_enabled(): # late import needed from miniworld.management.network.manager import NetworkManager log.info("creating management node/switch ...") if network_backend_bootstrapper.management_node_type is None: log.info("Network Backend has no management node") return None management_node = network_backend_bootstrapper.management_node_type( network_backend_bootstrapper) management_node.start(switch=True, bridge_dev_name=config.get_bridge_tap_name()) for node in self.nodes: management_node.connect_to_emu_node(singletons.network_backend, node) NetworkManager.management_node = management_node return management_node
def factory(): if config.is_protocol_zeromq_mode_mcast(): log.info("distance matrix distribution via publish-subscribe pattern") return ZeroMQCServerPubSub else: log.info("distance matrix distribution via request-reply pattern") return ZeroMQServerRouter
def get_all_connections(self): """ Get all the connections the nodes will have to each other. Returns ------- dict<int, set<int>> The connections each node has. Sorted by id asc. id number. Fully staffed matrix. """ # we need a lock here, otherwise all threads (each node) try to read the config file simultaneously! # used by each :py:class:`.EmulationNode` with static_lock: if self._all_connections is not None: return self._all_connections core_scenarios = scenario_config.get_core_scenarios() all_connections = defaultdict(set) for _, core_scenario_path in core_scenarios: connections_dict = CoreConfigFileParser.parse_core_config_file( core_scenario_path) for k, v in connections_dict.items(): all_connections[k].update(v) all_connections = DictUtil.to_fully_staffed_matrix( all_connections) self._all_connections = all_connections log.info("connections: %s", pformat(all_connections)) return all_connections
def create_n_check_tmp_files(delete_first=False, create_ramdisk=False): """ Create tmp dirs if not existing yet. Raises ------ AlreadyRunning """ if os.path.exists(PATH_TMP): if delete_first: try: shutil.rmtree(PATH_LOGS) except BaseException: pass # umount ramdisk try_umount_ramdisk() else: # Ticket #12 raise AlreadyRunning("Another instance of %s might be running! Did you invoke the cleanup script '%s'?" % (PROJECT_NAME, PATH_CLEANUP_SCRIPT)) if not os.path.exists(PATH_TMP): log.debug("Creating directory '%s'!", PATH_TMP) os.makedirs(PATH_TMP) if create_ramdisk: if not os.path.ismount(PATH_TMP): log.info("creating ramdisk at '%s'", PATH_TMP) # TODO: better way for ramdisk creation? run_shell("sudo mount -t ramfs ramfs {}".format(PATH_TMP)) else: log.info("ramdisk still exists at '%s' ... ", PATH_TMP)
def try_umount_ramdisk(): # umount ramdisk try: log.info("unmounting ramdisk (if one is mounted)") umount_ramdisk() except BaseException: pass
def start(self, cnt_peers): """ Start the server by expecting the clients to register in the first state. This method controls the whole communication expect the distribution of the distance matrix. Parameters ---------- cnt_peers : int Returns ------- """ log.info("waiting for %d servers ...", cnt_peers) self.cnt_peers = cnt_peers self.handle_state_register() # wait until the scenario config is set, because the tunnel addresses # (received in the next step) are written to it log.info("waiting until scenario config is set ...") self.wait_for_scenario_config.wait() self.handle_state_information_exchange() # TODO: let NetworkBackend inject communication steps for needed information self.handle_state_start_nodes()
def handle_select(rlist, xlist): if xlist: raise ZeroMQException("Unknown error occurred during a select() call") if self.reset_socket in rlist: self.reset_socket.recv() self.reset() return True if self.sub_socket in rlist: local_distance_matrix = self.recv_distance_matrix() if config.is_debug(): log.info("received distance matrix: %s", local_distance_matrix) log.info("step ...") singletons.simulation_manager.step(1, distance_matrix=local_distance_matrix) self.send_sync() # wait for sync reply or error rlist, _, xlist = zmq.select([self.reset_socket, self.svc], [], []) # rlist, _, xlist = zmq.select([self.svc], [], []) if handle_select(rlist, xlist): return True # self.sync() if config.is_debug(): log.debug("stepped ...") if self.svc in rlist: self.recv_sync() return True
def start_bg_checker_thread(self): return self.bg_checker_thread = ExceptionStopThread.run_fun_threaded_n_log_exception( target=self.check_error_codes, tkwargs=dict(name="BG Process Checker")) self.bg_checker_thread.start() log.info("starting bg process checker thread ...")
def myprint(self, str): """ Include the server id in log messages Parameters ---------- str : str """ log.info('%s: %s', self.server_id, str)
def _handle_state_distance_matrix(self, distance_matrix): # # sync initially if singletons.simulation_manager.current_step == 0: log.info("syncing clients initially ...") self.sync_subscribers() self.send_distance_matrix(distance_matrix) self.sync_subscribers()
def __init__(self, *args, **kwargs): super(ZeroMQCServerPubSub, self).__init__(*args, **kwargs) # create the publish socket self.pub_socket = self.context.socket(zmq.PUB) addr = "tcp://*:{}".format(Protocol.PORT_PUB_SERVICE) self.pub_socket.bind(addr) log.info("listening on '%s'", addr)
def handle_state_start_nodes(self): """ Start the nodes and sync them afterwards. Finally notify that :py:meth:`.handle_state_distance_matrix` can continue. """ log.info("state: %s", States.STATE_START_NODES) expect_state = self.get_expecter_state(States.STATE_START_NODES, 1) ResponderArgument(self.router_socket, self.protocol, expect_state, '')() self.wait_for_nodes_started.set()
def __init__(self, *args, **kwargs): super(ZeroMQClientSub, self).__init__(*args, **kwargs) self.sub_socket = self.context.socket(zmq.SUB) self.sub_socket.setsockopt(zmq.SUBSCRIBE, b'') addr = "tcp://{}:{}".format(self.server_addr, Protocol.PORT_PUB_SERVICE) self.sub_socket.connect(addr) log.info("connecting to: %s ...", addr)
def handle_state_distance_matrix(self, distance_matrix): if self.last_step_time is not None: et = time.time() - self.last_step_time log.info("took %0.2f seconds (previous distribution + sync)", et) self.last_step_time = time.time() self.wait_for_nodes_started.wait() self._handle_state_distance_matrix(distance_matrix)
def start(self): log.info("starting log writer (file: '%s')", self.log_filename) if self.fh_logfile is None: self.fh_logfile = open( PathUtil.get_log_file_path(self.log_filename), "w") self.writer_thread = ExceptionStopThread.run_fun_threaded_n_log_exception( target=self.check, tkwargs=dict(name="LogWriter")) self.writer_thread.daemon = True self.writer_thread.start()
def send_distance_matrix(self, distance_matrix): """ Send the distance matrix via the publish socket. Parameters ---------- distance_matrix : DistanceMatrix """ data = self.serialize(DistanceMatrix.transform_distance_matrix(distance_matrix)) log.info("sending %f kbytes ...", len(data) / 1024.0) self.pub_socket.send(data)
def print_overall_node_status(self): """ Print the nodes not ready yet """ while not self.event_nodes_started.is_set(): nodes_not_ready = self.nodes_not_ready() if nodes_not_ready: log.info("waiting for nodes: %s ...", ', '.join(map(str, nodes_not_ready))) self.event_nodes_started.wait(TIME_NODE_STATUS_REFRESH) else: break
def after_simulation_step(self, simulation_manager, step_cnt, network_backend, emulation_nodes, **kwargs): """ Notify about events, if registered for events. """ # notify script about new events event_hook_script_path = scenario_config.get_network_backend_event_hook_script() if event_hook_script_path and os.path.exists(event_hook_script_path): bridge_list = ' '.join(self.event_monitor.new_bridges) log.info("notified event_hook_script: %s" % check_output([event_hook_script_path, bridge_list], cwd=dirname(event_hook_script_path))) self.reset_simulation_step()
def handle_state_register(self): """ Wait until all clients have been registered and sync them afterwards. """ # show registered clients ... def register_fun(expecter, idx, cnt): log.info("%d/%d clients registered", idx, cnt) # let clients register and sync them log.info("state: %s", States.STATE_REGISTER) expect_state = self.get_expecter_state(States.STATE_REGISTER, 0, after_response_fun=register_fun) ResponderServerID(self.router_socket, self.protocol, expect_state)()
def apply_nic_check_commands(self, check_commands_per_node): es = singletons.event_system # check the connectivity if self.is_connectivity_checks_enabled: with es.event_init(es.EVENT_NETWORK_CHECK) as ev: log.info("Doing connectivity checks ...") # NOTE: do not stress the network self.run_emulation_node_commands(check_commands_per_node, ev) else: with es.event_init(es.EVENT_NETWORK_CHECK) as ev: ev.finish()
def get_nic_check_commands(self, connections): """ Do link checking connection-based, there check which nodes are reachable! In the distributed mode, we perform link checking on both sides! Parameters ---------- connections Returns ------- """ log.info("assuming bidirectional links ...") check_commands_per_node = defaultdict(list) for emulation_nodes, list_ifaces in connections.items(): if not emulation_nodes.filter_real_emulation_nodes(): break # NOTE: for the distributed mode, we have to check which node is local and which is remote # because connections might not be fully staffed, we need to perform the check from the local node to the remote node! emulation_node_x, emulation_node_y = emulation_nodes.sort_by_locality( ) for interfaces in list_ifaces: if not interfaces.filter_normal_interfaces(): break interface_x, interface_y = interfaces def add_check_cmd(ip_addr): ping_cmd = self.connectivity_checker_fun( ip_addr, self.network_timeout) check_commands_per_node[emulation_node_x].append(ping_cmd) # for CentralNode let each node ping all allocated ips if EmulationNodes([emulation_node_y]).filter_central_nodes(): for ip_addr in self.ips.values(): add_check_cmd(ip_addr) # check which ip is allocated to emulation_node_y else: ip_addr = self.ips[self.get_key_ip_dict( emulation_node_y, interface_y)] add_check_cmd(ip_addr) return check_commands_per_node
def set_global_config(*args, **kwargs): """ Set the global config. Returns ------- dict The config as JSON. """ _config = JSONConfig.read_json_config(*args, **kwargs) from miniworld.log import log log.info("setting global config file '%s'", *args, **kwargs) config.data = deepcopy(_config) return _config
def handle_state_information_exchange(self): """ Let clients send their tunnel Ip addr and send each the scenario config. """ log.info("state: %s", States.STATE_EXCHANGE) expect_state = self.get_expecter_state(States.STATE_EXCHANGE, 3) expect_state.expect_for_all() tunnel_addresses_dict = expect_state.get_message_part_per_node_id(arg_nr=1) server_score_dict = expect_state.get_message_part_per_node_id(arg_nr=2) singletons.node_distribution_strategy.server_score = server_score_dict log.info("server scores: %s", pformat(server_score_dict)) # set tunnel addresses in scenario config Scenario.scenario_config.set_network_backend_bridged_tunnel_endpoints(tunnel_addresses_dict) # create scenario configs node_ids = miniworld.Scenario.scenario_config.get_local_node_ids() # distribute nodes among emulation server server_node_mapping = singletons.node_distribution_strategy.distribute_emulation_nodes(list(node_ids), self.cnt_peers) log.info("nodes mapping: %s", pformat(server_node_mapping)) log.info("nodes per server: %s", pformat({k: len(v) for k, v in server_node_mapping.items()})) # set server to node mapping before creating the scenario configs miniworld.Scenario.scenario_config.set_distributed_server_node_mapping(server_node_mapping) # create for each server a custom scenario config scenario_configs = singletons.simulation_manager.create_scenario_configs(server_node_mapping) responder = ResponderPerServerID(self.router_socket, self.protocol, expect_state, scenario_configs) responder.respond()
def _start(self): # TODO: use ShellCommandSerializer tunnel_cmd = IPRoute2Commands.get_gretap_tunnel_cmd(self.get_tunnel_name(), self.remote_ip, self.get_tunnel_id()) log.info("creating gretap tunnel for %s<->%s" % (self.emulation_node_x.id, self.emulation_node_y.id)) self.add_command(self.EVENT_TUNNEL_ADD, tunnel_cmd) log.info("changing NIC MTU to '%d' ... ", GRETAP_MTU) self.get_local_emulation_node().virtualization_layer.set_link_mtus(mtu=GRETAP_MTU) # set tunnel dev group super(GreTapTunnel, self)._start()
def factory(): """ Use the config system to choose the zeromq client type. Returns ------- type """ if config.is_protocol_zeromq_mode_mcast(): log.info("distance matrix distribution via publish-subscribe pattern") return ZeroMQClientSub else: log.info("distance matrix distribution via request-reply pattern") return ZeroMQClientReq
def create_n_connect_central_nodes(self, interfaces): """ Parameters ---------- interfaces Returns ------- dict<int, CentralNode> """ from miniworld.model.singletons.Singletons import singletons # create CentralNode s but only if there is a HubWiFi interface # TODO: REMOVE cnt = 0 central_nodes_dict = {} # connect local devices for _if in filter(lambda x: is_central_node_interface(x), interfaces): if cnt == 1: raise ValueError("Only one '%s' interface supported at the moment!" % HubWiFi) # TODO: REFACTOR! # TODO: #54: make amount of nodes configurable count_central_nodes = 1 for i in range(0, count_central_nodes): central_node = self.network_backend_bootstrapper.central_node_type( self.network_backend_bootstrapper, id=i + 1) # central_node.id = self.get_br_name(central_node.id, central_node.interface) # TODO: #54 make configurable! log.debug("creating CentralNode with id: %s", central_node.id) central_node.start(switch=False, bridge_dev_name=central_node.id) central_nodes_dict[central_node.id] = central_node # remember new bridges self.event_monitor.add_new_bridge(central_node.id) cnt += 1 # connect via server boundaries (overlay) node_ids = singletons.simulation_manager.get_emulation_node_ids() for x, y in zip(node_ids, node_ids[1:]): emulation_node_x = singletons.simulation_manager.get_emulation_node_for_idx(x) emulation_node_y = singletons.simulation_manager.get_emulation_node_for_idx(y) log.info("connecting %s<->%s", emulation_node_x, emulation_node_y) self.connection_across_servers(self, emulation_node_x, emulation_node_y) return central_nodes_dict
def ip_config(self): # NOTE: we need to reuse the existing configurators due to their internal state! if scenario_config.is_network_links_auto_ipv4(): log.info("using ip provisioner: %s" % scenario_config.get_network_provisioner_name()) new_connections = self.get_new_connections_with_interfaces_since_last_distance_matrix_change() with open(PathUtil.get_log_file_path("%s.txt" % self.__class__.__name__), "a") as f: if self.net_configurator.needs_reconfiguration(singletons.simulation_manager.current_step): # only connection setup and check for new connections log.info("%s: configuring network ...", self.net_configurator.__class__.__name__) commands_per_node = self.net_configurator.get_nic_configuration_commands(new_connections) self.net_configurator.apply_nic_configuration_commands(commands_per_node) f.write("setup_commands: %s\n" % pformat(dict(commands_per_node)))
def sync_subscribers(self): """ Sync the subscribers and provide some debugging information about the sync progress (if debug mode) """ log.info("syncing subscribers ...") def fun(expecter, idx, cnt): log.debug("%d/%d clients synced ...", idx, cnt) # add some debug info after_response_fun = fun if config.is_debug() else lambda x, y, z: None expect_distance_matrix = self.get_expecter_state(States.STATE_DISTANCE_MATRIX, 1, after_response_fun) # sync clients and send each his distance matrix ResponderArgument(self.router_socket, self.protocol, expect_distance_matrix, '')() log.info("syncing subscribers [done]")