def validate(self, *, output_dir=None):
"""Validate the network parameters (latency, bandwidth ...)
Performs flent, ping tests to validate the constraints set by
:py:meth:`enoslib.service.netem.Netem.deploy`.
Reports are available in the tmp directory
used by enos.
Args:
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
inventory_path (str): path to an inventory
output_dir (str): directory where validation files will be stored.
Default to :py:const:`enoslib.constants.TMP_DIRNAME`.
"""
logger.debug("Checking the constraints")
if not output_dir:
output_dir = os.path.join(os.getcwd(), TMP_DIRNAME)
output_dir = os.path.abspath(output_dir)
_check_tmpdir(output_dir)
_playbook = os.path.join(SERVICE_PATH, "netem.yml")
options = self._build_options({
"enos_action": "tc_validate",
"tc_output_dir": output_dir
})
run_ansible([_playbook], roles=self.roles, extra_vars=options)
def validate_network(roles=None,
inventory_path=None,
output_dir=None,
extra_vars=None):
"""Validate the network parameters (latency, bandwidth ...)
Performs flent, ping tests to validate the constraints set by
:py:func:`emulate_network`. Reports are available in the tmp directory used
by enos.
Args:
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
inventory_path (str): path to an inventory
output_dir (str): directory where validation files will be stored.
Default to :py:const:`enoslib.constants.TMP_DIRNAME`.
"""
logger.debug('Checking the constraints')
if not output_dir:
output_dir = os.path.join(os.getcwd(), TMP_DIRNAME)
if not extra_vars:
extra_vars = {}
output_dir = os.path.abspath(output_dir)
_check_tmpdir(output_dir)
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
options = {'enos_action': 'tc_validate', 'tc_output_dir': output_dir}
options.update(extra_vars)
run_ansible([utils_playbook],
roles=roles,
inventory_path=inventory_path,
extra_vars=options)
def _check_networks(roles,
networks,
inventory,
fake_interfaces=None,
fake_networks=None):
"""Checks the network interfaces on the nodes
Beware, this has a side effect on each Host in env['rsc'].
"""
def get_devices(facts):
"""Extract the network devices information from the facts."""
devices = []
for interface in facts['ansible_interfaces']:
ansible_interface = 'ansible_' + interface
# filter here (active/ name...)
if 'ansible_' + interface in facts:
interface = facts[ansible_interface]
devices.append(interface)
return devices
wait_ssh(inventory)
tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME)
_check_tmpdir(tmpdir)
fake_interfaces = fake_interfaces or []
fake_networks = fake_networks or []
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
facts_file = os.path.join(tmpdir, 'facts.json')
options = {
'enos_action': 'check_network',
'facts_file': facts_file,
'fake_interfaces': fake_interfaces
}
run_ansible([utils_playbook],
inventory,
extra_vars=options,
on_error_continue=False)
# Read the file
# Match provider networks to interface names for each host
with open(facts_file) as f:
facts = json.load(f)
for _, host_facts in facts.items():
host_nets = _map_device_on_host_networks(networks,
get_devices(host_facts))
# Add the mapping : networks <-> nic name
host_facts['networks'] = host_nets
# Finally update the env with this information
# generate the extra_mapping for the fake interfaces
extra_mapping = dict(zip(fake_networks, fake_interfaces))
_update_hosts(roles, facts, extra_mapping=extra_mapping)
def reset_network(roles, inventory):
"""Reset the network constraints (latency, bandwidth ...)
Remove any filter that have been applied to shape the traffic.
Args:
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
inventory (str): path to the inventory
"""
logger.debug('Reset the constraints')
tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME)
_check_tmpdir(tmpdir)
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
options = {'enos_action': 'tc_reset', 'tc_output_dir': tmpdir}
run_ansible([utils_playbook], inventory, extra_vars=options)
def destroy(self):
"""Reset the network constraints (latency, bandwidth ...)
Remove any filter that have been applied to shape the traffic.
"""
logger.debug("Reset the constraints")
tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME)
_check_tmpdir(tmpdir)
_playbook = os.path.join(SERVICE_PATH, "netem.yml")
options = self._build_options({
"enos_action": "tc_reset",
"tc_output_dir": tmpdir
})
run_ansible([_playbook], roles=self.roles, extra_vars=options)
def validate_network(roles, inventory):
"""Validate the network parameters (latency, bandwidth ...)
Performs flent, ping tests to validate the constraints set by
:py:func:`emulate_network`. Reports are available in the tmp directory used
by enos.
Args:
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
inventory (str): path to the inventory
"""
logging.debug('Checking the constraints')
tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME)
_check_tmpdir(tmpdir)
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
options = {'enos_action': 'tc_validate', 'tc_output_dir': tmpdir}
run_ansible([utils_playbook], inventory, extra_vars=options)
def emulate_network(roles, inventory, network_constraints):
"""Emulate network links.
Read ``network_constraints`` and apply ``tc`` rules on all the nodes.
Constraints are applied between groups of machines. Theses groups are
described in the ``network_constraints`` variable and must be found in the
inventory file. The newtwork constraints support ``delay``, ``rate`` and
``loss``.
Args:
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
inventory (str): path to the inventory
network_constraints (dict): network constraints to apply
Examples:
* Using defaults
The following will apply the network constraints between every groups.
For instance the constraints will be applied for the communication
between "n1" and "n3" but not between "n1" and "n2". Note that using
default leads to symetric constraints.
.. code-block:: python
roles = {
"grp1": ["n1", "n2"],
"grp2": ["n3", "n4"],
"grp3": ["n3", "n4"],
}
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
}
emulate_network(roles, inventory, tc)
If you want to control more precisely which groups need to be taken
into account, you can use ``except`` or ``groups`` key
.. code-block:: python
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
"except": "grp3"
}
emulate_network(roles, inventory, tc)
is equivalent to
.. code-block:: python
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
"groups": ["grp1", "grp2"]
}
emulate_network(roles, inventory, tc)
* Using ``src`` and ``dst``
The following will enforce a symetric constraint between ``grp1`` and
``grp2``.
.. code-block:: python
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
"constraints": [{
"src": "grp1"
"dst": "grp2"
"delay": "10ms"
"symetric": True
}]
}
emulate_network(roles, inventory, tc)
"""
# 1) Retrieve the list of ips for all nodes (Ansible)
# 2) Build all the constraints (Python)
# {source:src, target: ip_dest, device: if, rate:x, delay:y}
# 3) Enforce those constraints (Ansible)
# TODO(msimonin)
# - allow finer grained filtering based on network roles and/or nic name
# 1. getting ips/devices information
logger.debug('Getting the ips of all nodes')
tmpdir = os.path.join(os.path.dirname(inventory), TMP_DIRNAME)
_check_tmpdir(tmpdir)
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
ips_file = os.path.join(tmpdir, 'ips.txt')
options = {'enos_action': 'tc_ips', 'ips_file': ips_file}
run_ansible([utils_playbook], inventory, extra_vars=options)
# 2.a building the group constraints
logger.debug('Building all the constraints')
constraints = _build_grp_constraints(roles, network_constraints)
# 2.b Building the ip/device level constaints
with open(ips_file) as f:
ips = yaml.safe_load(f)
# will hold every single constraint
ips_with_constraints = _build_ip_constraints(roles, ips, constraints)
# dumping it for debugging purpose
ips_with_constraints_file = os.path.join(tmpdir,
'ips_with_constraints.yml')
with open(ips_with_constraints_file, 'w') as g:
yaml.dump(ips_with_constraints, g)
# 3. Enforcing those constraints
logger.info('Enforcing the constraints')
# enabling/disabling network constraints
enable = network_constraints.setdefault('enable', True)
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
options = {
'enos_action': 'tc_apply',
'ips_with_constraints': ips_with_constraints,
'tc_enable': enable,
}
run_ansible([utils_playbook], inventory, extra_vars=options)
def discover_networks(roles,
networks,
fake_interfaces=None,
fake_networks=None):
"""Checks the network interfaces on the nodes.
This enables to auto-discover the mapping interface name <-> network role.
Beware, this has a side effect on each Host in roles.
Args:
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
networks (list): network list as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
fake_interfaces (list): names of optionnal dummy interfaces to create
fake_networks (list): names of the roles to associate with the fake
interfaces. Like reguilar network interfaces, the mapping will be
added to the host vars. Internally this will be zipped with the
fake_interfaces to produce the mapping.
If the command is successful each host will be added some variables.
Assuming that one network whose role is `mynetwork` has been declared, the
following variables will be available through the ansible hostvars:
- ``mynetwork=eth1``, `eth1` has been discovered has the interface in the
network `mynetwork`.
- ``mynetwork_dev=eth1``, same as above with a different accessor names
- ``mynetwork_ip=192.168.42.42``, this indicates the ip in the network
`mynetwork` for this node
All of this variable can then be accessed by the other nodes through the
hostvars: ``hostvars[remote_node]["mynetwork_ip"]``
"""
def get_devices(facts):
"""Extract the network devices information from the facts."""
devices = []
for interface in facts['ansible_interfaces']:
ansible_interface = 'ansible_' + interface
# filter here (active/ name...)
if 'ansible_' + interface in facts:
interface = facts[ansible_interface]
devices.append(interface)
return devices
wait_ssh(roles)
tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME)
_check_tmpdir(tmpdir)
fake_interfaces = fake_interfaces or []
fake_networks = fake_networks or []
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
facts_file = os.path.join(tmpdir, 'facts.json')
options = {
'enos_action': 'check_network',
'facts_file': facts_file,
'fake_interfaces': fake_interfaces
}
run_ansible([utils_playbook],
roles=roles,
extra_vars=options,
on_error_continue=False)
# Read the file
# Match provider networks to interface names for each host
with open(facts_file) as f:
facts = json.load(f)
for _, host_facts in facts.items():
host_nets = _map_device_on_host_networks(networks,
get_devices(host_facts))
# Add the mapping : networks <-> nic name
host_facts['networks'] = host_nets
# Finally update the env with this information
# generate the extra_mapping for the fake interfaces
extra_mapping = dict(zip(fake_networks, fake_interfaces))
_update_hosts(roles, facts, extra_mapping=extra_mapping)
def discover_networks(roles,
networks,
fake_interfaces=None,
fake_networks=None):
"""Checks the network interfaces on the nodes. This enables to
auto-discover the mapping interface name <-> network role.
Beware, this has a side effect on each Host in roles.
Args:
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
networks (list): network list as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
fake_interfaces (list): names of optionnal dummy interfaces to create
fake_networks (list): names of the roles to associate with the fake
interfaces. Like reguilar network interfaces, the mapping will be
added to the host vars. Internally this will be zipped with the
fake_interfaces to produce the mapping.
"""
def get_devices(facts):
"""Extract the network devices information from the facts."""
devices = []
for interface in facts['ansible_interfaces']:
ansible_interface = 'ansible_' + interface
# filter here (active/ name...)
if 'ansible_' + interface in facts:
interface = facts[ansible_interface]
devices.append(interface)
return devices
wait_ssh(roles)
tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME)
_check_tmpdir(tmpdir)
fake_interfaces = fake_interfaces or []
fake_networks = fake_networks or []
utils_playbook = os.path.join(ANSIBLE_DIR, 'utils.yml')
facts_file = os.path.join(tmpdir, 'facts.json')
options = {
'enos_action': 'check_network',
'facts_file': facts_file,
'fake_interfaces': fake_interfaces
}
run_ansible([utils_playbook],
roles=roles,
extra_vars=options,
on_error_continue=False)
# Read the file
# Match provider networks to interface names for each host
with open(facts_file) as f:
facts = json.load(f)
for _, host_facts in facts.items():
host_nets = _map_device_on_host_networks(networks,
get_devices(host_facts))
# Add the mapping : networks <-> nic name
host_facts['networks'] = host_nets
# Finally update the env with this information
# generate the extra_mapping for the fake interfaces
extra_mapping = dict(zip(fake_networks, fake_interfaces))
_update_hosts(roles, facts, extra_mapping=extra_mapping)
def deploy(self):
"""Emulate network links.
Read ``network_constraints`` and apply ``tc`` rules on all the nodes.
Constraints are applied between groups of machines. Theses groups are
described in the ``network_constraints`` variable and must be found in
the inventory file. The newtwork constraints support ``delay``,
``rate`` and ``loss``.
Args:
network_constraints (dict): network constraints to apply
roles (dict): role->hosts mapping as returned by
:py:meth:`enoslib.infra.provider.Provider.init`
inventory_path(string): path to an inventory
extra_vars (dict): extra_vars to pass to ansible
Examples:
* Using defaults
The following will apply the network constraints between every
groups. For instance the constraints will be applied for the
communication between "n1" and "n3" but not between "n1" and "n2".
Note that using default leads to symetric constraints.
.. code-block:: python
roles = {
"grp1": ["n1", "n2"],
"grp2": ["n3", "n4"],
"grp3": ["n3", "n4"],
}
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
}
netem = Netem(tc, roles=roles)
netem.deploy()
If you want to control more precisely which groups need to be taken
into account, you can use ``except`` or ``groups`` key
.. code-block:: python
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
"except": "grp3"
}
netem = Netem(tc, roles=roles)
netem.deploy()
If you want to control more precisely which groups need to be taken
into account:
.. code-block:: python
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
"groups": ["grp1", "grp2"]
}
netem = Netem(tc, roles=roles)
netem.deploy()
* Using ``src`` and ``dst``
The following will enforce a symetric constraint between ``grp1``
and ``grp2``.
.. code-block:: python
tc = {
"enable": True,
"default_delay": "20ms",
"default_rate": "1gbit",
"constraints": [{
"src": "grp1"
"dst": "grp2"
"delay": "10ms"
"symetric": True
}]
}
netem = Netem(tc, roles=roles)
netem.deploy()
Examples:
.. literalinclude:: examples/netem.py
:language: python
:linenos:
"""
# 1) Retrieve the list of ips for all nodes (Ansible)
# 2) Build all the constraints (Python)
# {source:src, target: ip_dest, device: if, rate:x, delay:y}
# 3) Enforce those constraints (Ansible)
# 1. getting ips/devices information
logger.debug("Getting the ips of all nodes")
tmpdir = os.path.join(os.getcwd(), TMP_DIRNAME)
_check_tmpdir(tmpdir)
_playbook = os.path.join(SERVICE_PATH, "netem.yml")
ips_file = os.path.join(tmpdir, "ips.txt")
options = self._build_options({
"enos_action": "tc_ips",
"ips_file": ips_file
})
run_ansible([_playbook], roles=self.roles, extra_vars=options)
# 2.a building the group constraints
logger.debug("Building all the constraints")
constraints = _build_grp_constraints(self.roles,
self.network_constraints)
# 2.b Building the ip/device level constaints
with open(ips_file) as f:
ips = yaml.safe_load(f)
# will hold every single constraint
ips_with_constraints = _build_ip_constraints(
self.roles, ips, constraints)
# dumping it for debugging purpose
ips_with_constraints_file = os.path.join(
tmpdir, "ips_with_constraints.yml")
with open(ips_with_constraints_file, "w") as g:
yaml.dump(ips_with_constraints, g)
# 3. Enforcing those constraints
logger.info("Enforcing the constraints")
# enabling/disabling network constraints
enable = self.network_constraints.setdefault("enable", True)
options = self._build_options({
"enos_action": "tc_apply",
"ips_with_constraints": ips_with_constraints,
"tc_enable": enable,
})
run_ansible([_playbook], roles=self.roles, extra_vars=options)
