def validate(self, *, output_dir=None): """Validate the network parameters (latency, bandwidth ...) Performs flent, ping tests to validate the constraints set by :py:meth:`enoslib.service.netem.Netem.deploy`. Reports are available in the tmp directory used by enos. Args: roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` inventory_path (str): path to an inventory output_dir (str): directory where validation files will be stored. Default to :py:const:`enoslib.constants.TMP_DIRNAME`. """ logger.debug("Checking the constraints") if not output_dir: output_dir = os.path.join(os.getcwd(), TMP_DIRNAME) output_dir = os.path.abspath(output_dir) _check_tmpdir(output_dir) _playbook = os.path.join(SERVICE_PATH, "netem.yml") options = self._build_options({ "enos_action": "tc_validate", "tc_output_dir": output_dir }) run_ansible([_playbook], roles=self.roles, extra_vars=options)
def validate_network(roles=None, inventory_path=None, output_dir=None, extra_vars=None): """Validate the network parameters (latency, bandwidth ...) Performs flent, ping tests to validate the constraints set by :py:func:`emulate_network`. Reports are available in the tmp directory used by enos. Args: roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` inventory_path (str): path to an inventory output_dir (str): directory where validation files will be stored. Default to :py:const:`enoslib.constants.TMP_DIRNAME`. """ logger.debug('Checking the constraints') if not output_dir: output_dir = os.path.join(os.getcwd(), TMP_DIRNAME) if not extra_vars: extra_vars = {} output_dir = os.path.abspath(output_dir) _check_tmpdir(output_dir) utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') options = {'enos_action': 'tc_validate', 'tc_output_dir': output_dir} options.update(extra_vars) run_ansible([utils_playbook], roles=roles, inventory_path=inventory_path, extra_vars=options)
def _check_networks(roles, networks, inventory, fake_interfaces=None, fake_networks=None): """Checks the network interfaces on the nodes Beware, this has a side effect on each Host in env['rsc']. """ def get_devices(facts): """Extract the network devices information from the facts.""" devices = [] for interface in facts['ansible_interfaces']: ansible_interface = 'ansible_' + interface # filter here (active/ name...) if 'ansible_' + interface in facts: interface = facts[ansible_interface] devices.append(interface) return devices wait_ssh(inventory) tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME) _check_tmpdir(tmpdir) fake_interfaces = fake_interfaces or [] fake_networks = fake_networks or [] utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') facts_file = os.path.join(tmpdir, 'facts.json') options = { 'enos_action': 'check_network', 'facts_file': facts_file, 'fake_interfaces': fake_interfaces } run_ansible([utils_playbook], inventory, extra_vars=options, on_error_continue=False) # Read the file # Match provider networks to interface names for each host with open(facts_file) as f: facts = json.load(f) for _, host_facts in facts.items(): host_nets = _map_device_on_host_networks(networks, get_devices(host_facts)) # Add the mapping : networks <-> nic name host_facts['networks'] = host_nets # Finally update the env with this information # generate the extra_mapping for the fake interfaces extra_mapping = dict(zip(fake_networks, fake_interfaces)) _update_hosts(roles, facts, extra_mapping=extra_mapping)
def reset_network(roles, inventory): """Reset the network constraints (latency, bandwidth ...) Remove any filter that have been applied to shape the traffic. Args: roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` inventory (str): path to the inventory """ logger.debug('Reset the constraints') tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME) _check_tmpdir(tmpdir) utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') options = {'enos_action': 'tc_reset', 'tc_output_dir': tmpdir} run_ansible([utils_playbook], inventory, extra_vars=options)
def destroy(self): """Reset the network constraints (latency, bandwidth ...) Remove any filter that have been applied to shape the traffic. """ logger.debug("Reset the constraints") tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME) _check_tmpdir(tmpdir) _playbook = os.path.join(SERVICE_PATH, "netem.yml") options = self._build_options({ "enos_action": "tc_reset", "tc_output_dir": tmpdir }) run_ansible([_playbook], roles=self.roles, extra_vars=options)
def validate_network(roles, inventory): """Validate the network parameters (latency, bandwidth ...) Performs flent, ping tests to validate the constraints set by :py:func:`emulate_network`. Reports are available in the tmp directory used by enos. Args: roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` inventory (str): path to the inventory """ logging.debug('Checking the constraints') tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME) _check_tmpdir(tmpdir) utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') options = {'enos_action': 'tc_validate', 'tc_output_dir': tmpdir} run_ansible([utils_playbook], inventory, extra_vars=options)
def emulate_network(roles, inventory, network_constraints): """Emulate network links. Read ``network_constraints`` and apply ``tc`` rules on all the nodes. Constraints are applied between groups of machines. Theses groups are described in the ``network_constraints`` variable and must be found in the inventory file. The newtwork constraints support ``delay``, ``rate`` and ``loss``. Args: roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` inventory (str): path to the inventory network_constraints (dict): network constraints to apply Examples: * Using defaults The following will apply the network constraints between every groups. For instance the constraints will be applied for the communication between "n1" and "n3" but not between "n1" and "n2". Note that using default leads to symetric constraints. .. code-block:: python roles = { "grp1": ["n1", "n2"], "grp2": ["n3", "n4"], "grp3": ["n3", "n4"], } tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", } emulate_network(roles, inventory, tc) If you want to control more precisely which groups need to be taken into account, you can use ``except`` or ``groups`` key .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "except": "grp3" } emulate_network(roles, inventory, tc) is equivalent to .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "groups": ["grp1", "grp2"] } emulate_network(roles, inventory, tc) * Using ``src`` and ``dst`` The following will enforce a symetric constraint between ``grp1`` and ``grp2``. .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "constraints": [{ "src": "grp1" "dst": "grp2" "delay": "10ms" "symetric": True }] } emulate_network(roles, inventory, tc) """ # 1) Retrieve the list of ips for all nodes (Ansible) # 2) Build all the constraints (Python) # {source:src, target: ip_dest, device: if, rate:x, delay:y} # 3) Enforce those constraints (Ansible) # TODO(msimonin) # - allow finer grained filtering based on network roles and/or nic name # 1. getting ips/devices information logger.debug('Getting the ips of all nodes') tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME) _check_tmpdir(tmpdir) utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') ips_file = os.path.join(tmpdir, 'ips.txt') options = {'enos_action': 'tc_ips', 'ips_file': ips_file} run_ansible([utils_playbook], inventory, extra_vars=options) # 2.a building the group constraints logger.debug('Building all the constraints') constraints = _build_grp_constraints(roles, network_constraints) # 2.b Building the ip/device level constaints with open(ips_file) as f: ips = yaml.safe_load(f) # will hold every single constraint ips_with_constraints = _build_ip_constraints(roles, ips, constraints) # dumping it for debugging purpose ips_with_constraints_file = os.path.join(tmpdir, 'ips_with_constraints.yml') with open(ips_with_constraints_file, 'w') as g: yaml.dump(ips_with_constraints, g) # 3. Enforcing those constraints logger.info('Enforcing the constraints') # enabling/disabling network constraints enable = network_constraints.setdefault('enable', True) utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') options = { 'enos_action': 'tc_apply', 'ips_with_constraints': ips_with_constraints, 'tc_enable': enable, } run_ansible([utils_playbook], inventory, extra_vars=options)
def discover_networks(roles, networks, fake_interfaces=None, fake_networks=None): """Checks the network interfaces on the nodes. This enables to auto-discover the mapping interface name <-> network role. Beware, this has a side effect on each Host in roles. Args: roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` networks (list): network list as returned by :py:meth:`enoslib.infra.provider.Provider.init` fake_interfaces (list): names of optionnal dummy interfaces to create fake_networks (list): names of the roles to associate with the fake interfaces. Like reguilar network interfaces, the mapping will be added to the host vars. Internally this will be zipped with the fake_interfaces to produce the mapping. If the command is successful each host will be added some variables. Assuming that one network whose role is `mynetwork` has been declared, the following variables will be available through the ansible hostvars: - ``mynetwork=eth1``, `eth1` has been discovered has the interface in the network `mynetwork`. - ``mynetwork_dev=eth1``, same as above with a different accessor names - ``mynetwork_ip=192.168.42.42``, this indicates the ip in the network `mynetwork` for this node All of this variable can then be accessed by the other nodes through the hostvars: ``hostvars[remote_node]["mynetwork_ip"]`` """ def get_devices(facts): """Extract the network devices information from the facts.""" devices = [] for interface in facts['ansible_interfaces']: ansible_interface = 'ansible_' + interface # filter here (active/ name...) if 'ansible_' + interface in facts: interface = facts[ansible_interface] devices.append(interface) return devices wait_ssh(roles) tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME) _check_tmpdir(tmpdir) fake_interfaces = fake_interfaces or [] fake_networks = fake_networks or [] utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') facts_file = os.path.join(tmpdir, 'facts.json') options = { 'enos_action': 'check_network', 'facts_file': facts_file, 'fake_interfaces': fake_interfaces } run_ansible([utils_playbook], roles=roles, extra_vars=options, on_error_continue=False) # Read the file # Match provider networks to interface names for each host with open(facts_file) as f: facts = json.load(f) for _, host_facts in facts.items(): host_nets = _map_device_on_host_networks(networks, get_devices(host_facts)) # Add the mapping : networks <-> nic name host_facts['networks'] = host_nets # Finally update the env with this information # generate the extra_mapping for the fake interfaces extra_mapping = dict(zip(fake_networks, fake_interfaces)) _update_hosts(roles, facts, extra_mapping=extra_mapping)
def discover_networks(roles, networks, fake_interfaces=None, fake_networks=None): """Checks the network interfaces on the nodes. This enables to auto-discover the mapping interface name <-> network role. Beware, this has a side effect on each Host in roles. Args: roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` networks (list): network list as returned by :py:meth:`enoslib.infra.provider.Provider.init` fake_interfaces (list): names of optionnal dummy interfaces to create fake_networks (list): names of the roles to associate with the fake interfaces. Like reguilar network interfaces, the mapping will be added to the host vars. Internally this will be zipped with the fake_interfaces to produce the mapping. """ def get_devices(facts): """Extract the network devices information from the facts.""" devices = [] for interface in facts['ansible_interfaces']: ansible_interface = 'ansible_' + interface # filter here (active/ name...) if 'ansible_' + interface in facts: interface = facts[ansible_interface] devices.append(interface) return devices wait_ssh(roles) tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME) _check_tmpdir(tmpdir) fake_interfaces = fake_interfaces or [] fake_networks = fake_networks or [] utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml') facts_file = os.path.join(tmpdir, 'facts.json') options = { 'enos_action': 'check_network', 'facts_file': facts_file, 'fake_interfaces': fake_interfaces } run_ansible([utils_playbook], roles=roles, extra_vars=options, on_error_continue=False) # Read the file # Match provider networks to interface names for each host with open(facts_file) as f: facts = json.load(f) for _, host_facts in facts.items(): host_nets = _map_device_on_host_networks(networks, get_devices(host_facts)) # Add the mapping : networks <-> nic name host_facts['networks'] = host_nets # Finally update the env with this information # generate the extra_mapping for the fake interfaces extra_mapping = dict(zip(fake_networks, fake_interfaces)) _update_hosts(roles, facts, extra_mapping=extra_mapping)
def deploy(self): """Emulate network links. Read ``network_constraints`` and apply ``tc`` rules on all the nodes. Constraints are applied between groups of machines. Theses groups are described in the ``network_constraints`` variable and must be found in the inventory file. The newtwork constraints support ``delay``, ``rate`` and ``loss``. Args: network_constraints (dict): network constraints to apply roles (dict): role->hosts mapping as returned by :py:meth:`enoslib.infra.provider.Provider.init` inventory_path(string): path to an inventory extra_vars (dict): extra_vars to pass to ansible Examples: * Using defaults The following will apply the network constraints between every groups. For instance the constraints will be applied for the communication between "n1" and "n3" but not between "n1" and "n2". Note that using default leads to symetric constraints. .. code-block:: python roles = { "grp1": ["n1", "n2"], "grp2": ["n3", "n4"], "grp3": ["n3", "n4"], } tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", } netem = Netem(tc, roles=roles) netem.deploy() If you want to control more precisely which groups need to be taken into account, you can use ``except`` or ``groups`` key .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "except": "grp3" } netem = Netem(tc, roles=roles) netem.deploy() If you want to control more precisely which groups need to be taken into account: .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "groups": ["grp1", "grp2"] } netem = Netem(tc, roles=roles) netem.deploy() * Using ``src`` and ``dst`` The following will enforce a symetric constraint between ``grp1`` and ``grp2``. .. code-block:: python tc = { "enable": True, "default_delay": "20ms", "default_rate": "1gbit", "constraints": [{ "src": "grp1" "dst": "grp2" "delay": "10ms" "symetric": True }] } netem = Netem(tc, roles=roles) netem.deploy() Examples: .. literalinclude:: examples/netem.py :language: python :linenos: """ # 1) Retrieve the list of ips for all nodes (Ansible) # 2) Build all the constraints (Python) # {source:src, target: ip_dest, device: if, rate:x, delay:y} # 3) Enforce those constraints (Ansible) # 1. getting ips/devices information logger.debug("Getting the ips of all nodes") tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME) _check_tmpdir(tmpdir) _playbook = os.path.join(SERVICE_PATH, "netem.yml") ips_file = os.path.join(tmpdir, "ips.txt") options = self._build_options({ "enos_action": "tc_ips", "ips_file": ips_file }) run_ansible([_playbook], roles=self.roles, extra_vars=options) # 2.a building the group constraints logger.debug("Building all the constraints") constraints = _build_grp_constraints(self.roles, self.network_constraints) # 2.b Building the ip/device level constaints with open(ips_file) as f: ips = yaml.safe_load(f) # will hold every single constraint ips_with_constraints = _build_ip_constraints( self.roles, ips, constraints) # dumping it for debugging purpose ips_with_constraints_file = os.path.join( tmpdir, "ips_with_constraints.yml") with open(ips_with_constraints_file, "w") as g: yaml.dump(ips_with_constraints, g) # 3. Enforcing those constraints logger.info("Enforcing the constraints") # enabling/disabling network constraints enable = self.network_constraints.setdefault("enable", True) options = self._build_options({ "enos_action": "tc_apply", "ips_with_constraints": ips_with_constraints, "tc_enable": enable, }) run_ansible([_playbook], roles=self.roles, extra_vars=options)