def setUp(self):
asys = AS(LocalHost(), True)
asys.border_routers = [
BorderRouter(1),
BorderRouter(2)
]
asys.border_routers[0].links[IfId(2)] = Link(LinkEp("1-ff00:0:110-br1#2"), LinkEp("2-ff00:0:210#1"), LinkType.CORE)
asys.border_routers[0].links[IfId(4)] = Link(LinkEp("1-ff00:0:110-br1#4"), LinkEp("2-ff00:0:211#1"), LinkType.CORE)
asys.border_routers[1].links[IfId(5)] = Link(LinkEp("1-ff00:0:110-br2#5"), LinkEp("1-ff00:0:111#1"), LinkType.CHILD)
self.asys = asys
def _restart_container(self, cntr, isd_as: ISD_AS, asys: AS) -> bool:
"""Try to restart an AS container currently not running.
:param cntr: Container to restart.
:param isd_as: AS the container belongs to.
:param asys: AS the container belongs to.
:returns: True, if container is now running. False, if the restart failed.
"""
cntr.start() # try to start the container
cntr.reload() # get the new status
if cntr.status == "running":
log.info("Restarted container %s [%s] (%s).", cntr.name,
asys.host.name, asys.container_id)
# Delete the socket used by the supervisor so scion.sh knows it has to be restarted.
# See supervisor/supervisor.sh in the SCION source code.
run_cmd_in_cntr(cntr, const.SCION_USER, "rm /tmp/supervisor.sock")
# Network bridges not created by Docker don't reconnect automatically.
disconnect_bridges(isd_as, asys, non_docker_only=True)
connect_bridges(isd_as, asys, non_docker_only=True)
# Restart the SSH server in managed ASes.
if self.coordinator is not None:
if asys.is_attachment_point or self.coordinator.ssh_management:
dc = asys.host.docker_client
self._start_sshd(dc.container.get(asys.container_id))
return True # container is now running
else:
log.warning("Restarting container %s [%s] (%s) failed.", cntr.name,
asys.host.name, asys.container_id)
asys.container_id = None
return False
def _stop_container(self, isd_as: ISD_AS, asys: AS) -> None:
"""Stop the Docker container hosting the given AS."""
if asys.container_id:
dc = asys.host.docker_client
try:
cntr = dc.containers.get(asys.container_id)
except docker.errors.NotFound:
asys.container_id = None
else:
# Disconnect bridges not created by Docker here, to avoid leaving
# unused interfaces behind when stopping the container.
disconnect_bridges(isd_as, asys, non_docker_only=True)
cntr.remove(force=True)
log.info("Stopped container %s [%s] (%s).", cntr.name,
asys.host.name, asys.container_id)
asys.container_id = None
def test_addr_assignment(self):
"""Test correct assignment of Docker host addresses to border router interfaces."""
net = HostNetwork("host_network", ipaddress.ip_network("10.0.0.0/24"))
hosts = [LocalHost(), LocalHost()]
asys = [AS(host, False) for host in hosts]
net.set_host_ip(hosts[0], ipaddress.ip_address("10.0.0.10"))
net.set_host_ip(hosts[1], ipaddress.ip_address("10.0.0.11"))
with self.assertRaises(errors.NotAvailable):
net.assign_br_address(ISD_AS("1-ff00:0:000"),
asys[0],
IfId(1),
pref_ip=ipaddress.ip_address("10.0.0.2"))
ip, port = net.assign_br_address(ISD_AS("1-ff00:0:000"), asys[0],
IfId(1))
self.assertEqual(ip, ipaddress.ip_address("10.0.0.10"))
self.assertEqual(port, L4Port(50000))
ip, port = net.assign_br_address(ISD_AS("1-ff00:0:001"), asys[1],
IfId(1))
self.assertEqual(ip, ipaddress.ip_address("10.0.0.11"))
self.assertEqual(port, L4Port(50000))
ip, port = net.assign_br_address(ISD_AS("1-ff00:0:001"), asys[1],
IfId(2))
self.assertEqual(ip, ipaddress.ip_address("10.0.0.11"))
self.assertEqual(port, L4Port(50001))
def test_ip_addr_assignment(self):
br = DockerBridge("test", LocalHost(),
ipaddress.IPv4Network("10.0.0.0/29"))
asys = AS(LocalHost(), False)
# Assign coordinator IP
for _ in range(2): # Multiple calls must return the same address
ip = br.assign_ip_address("coordinator")
self.assertEqual(ip, ipaddress.IPv4Address(0x0A000002))
# Assign AS IPs
for asys in range(1, 5):
ip = br.assign_ip_address(ISD_AS(asys))
self.assertEqual(ip, ipaddress.IPv4Address(0x0A000000 + asys + 2))
with self.assertRaises(errors.OutOfResources):
br.assign_ip_address(ISD_AS(8))
# Retrive assigned addresses
self.assertEqual(br.get_ip_address("coordinator"),
ipaddress.IPv4Address(0x0A000002))
for asys in range(1, 5):
ip = br.get_ip_address(ISD_AS(asys))
self.assertEqual(ip, ipaddress.IPv4Address(0x0A000000 + asys + 2))
# Free and reassign an addresses
ip = br.get_ip_address(ISD_AS(2))
self.assertEqual(br.free_ip_address(ISD_AS(2)), 0)
self.assertIsNone(br.get_ip_address(ISD_AS(2)))
self.assertEqual(br.assign_ip_address(ISD_AS(6)), ip)
def _get_container(self, isd_as: ISD_AS, asys: AS):
"""Get the container hosting the given AS.
:returns: Docker container object.
:raises docker.errors.NotFound: The container associated with the AS does not exist anymore.
This error has been logged and the stale container ID deleted.
"""
dc = asys.host.docker_client
try:
return dc.containers.get(asys.container_id)
except docker.errors.NotFound:
cntr_name = self.get_cntr_name(isd_as)
log.error("Container %s [%s] (%s) not found.", cntr_name,
asys.host.name, asys.container_id)
asys.container_id = None
raise
def _format_and_run(
topo: Topology, isd_as: ISD_AS, asys: AS, cmd_template: str,
user: str, detach:bool, dry_run: bool):
"""Format the given command template and run the command in the given AS."""
if not asys.container_id:
print("No container for AS{}.".format(isd_as))
else:
try:
cntr = asys.get_container()
except docker.errors.NotFound:
cntr_name = topo.get_cntr_name(isd_as)
log.warning("Container {} ({}) not found.".format(cntr_name, asys.container_id))
else:
cmd = cmd_template.format(isd_as=str(isd_as), file_fmt=isd_as.file_fmt())
if dry_run:
print("Would run '{}' in {}.".format(cmd, cntr.name))
else:
_run_in_cntr(cntr, cmd, user, detach)
def test_br_addr_assignment(self):
"""Test assignment of (IP, port) tuples to border router interfaces."""
br = DockerBridge("test", LocalHost(),
ipaddress.IPv4Network("10.0.0.0/29"))
asys = AS(LocalHost(), False)
# Assign BR interface addresses
for _ in range(2): # Multiple calls must return the same addresses
for ifid in range(1, 3):
for as_id in range(1, 6):
ip, port = br.assign_br_address(ISD_AS(as_id), asys,
IfId(ifid))
self.assertEqual(
ip, ipaddress.IPv4Address(0x0A000000 + as_id + 1))
self.assertEqual(port, 50000 + ifid - 1)
with self.assertRaises(errors.OutOfResources):
br.assign_br_address(ISD_AS(8), asys, IfId(1))
# AS IP assignment
for as_id in range(1, 6):
isd_as = ISD_AS(as_id)
self.assertEqual(br.get_ip_address(isd_as),
br.assign_ip_address(isd_as))
# Retrieve BR interface underlay addresses
for ifid in range(1, 3):
for asys in range(1, 6):
ip, port = unwrap(br.get_br_address(ISD_AS(asys), IfId(ifid)))
self.assertEqual(ip,
ipaddress.IPv4Address(0x0A000000 + asys + 1))
self.assertEqual(port, 50000 + ifid - 1)
# Free BR interfaces
for asys in range(1, 6):
for ifid in range(1, 3):
self.assertEqual(br.free_br_address(ISD_AS(asys), IfId(ifid)),
3 - ifid)
# Check whether IPs are still bound
for asys in range(1, 6):
ip = br.get_ip_address(ISD_AS(asys))
self.assertEqual(ip, ipaddress.IPv4Address(0x0A000000 + asys + 1))
def _get_ap_links(topo, user_as: AS) -> List[AttachmentLink]:
"""Get all links connecting a user AS to attachment points."""
links = []
for user_ifid, link in user_as.links():
if link.is_dummy():
continue
elif topo.ases[link.ep_a].is_attachment_point:
ap, user = link.ep_a, link.ep_b
ap_underlay_addr, user_underlay_addr = link.ep_a_underlay, link.ep_b_underlay
elif topo.ases[link.ep_b].is_attachment_point:
ap, user = link.ep_b, link.ep_a
ap_underlay_addr, user_underlay_addr = link.ep_b_underlay, link.ep_a_underlay
else:
continue # not an AP link
links.append(
AttachmentLink(
user_ifid, unwrap(user_underlay_addr),
link.bridge.get_br_bind_address(user, topo.ases[user],
user_ifid), ap,
unwrap(ap_underlay_addr),
link.bridge.get_br_bind_address(ap, topo.ases[ap], ap.ifid)))
return links
def _start_container(self, isd_as: ISD_AS, asys: AS, workdir: Path,
sc: Path) -> None:
"""Start the Docker container hosting the given AS and connect it to the necessary bridges.
"""
dc = asys.host.docker_client
# Check if container is already running
if asys.container_id:
try:
cntr = dc.containers.get(asys.container_id)
except docker.errors.NotFound:
# container has been removed
asys.container_id = None
else:
if cntr.status == "running":
return # container is already running
elif cntr.status == 'paused':
cntr.unpause()
log.info("Unpaused container %s [%s] (%s).", cntr.name,
asys.host.name, cntr.id)
return
else:
if self._restart_container(cntr, isd_as, asys):
return # restart successful
# Create and start a new container
cntr_name = self.get_cntr_name(isd_as)
ports = get_published_ports(isd_as, asys)
for cntr_port, (host_ip, host_port) in ports.items():
log.info("Exposing port %s of %s on %s:%s [%s].", cntr_port,
cntr_name, host_ip, host_port, asys.host.name)
cntr = None
if asys.host.is_local:
mount_dir = workdir.joinpath(isd_as.file_fmt()).resolve()
if self.coordinator is not None:
# Starting a new instance of the coordinator generates new configuration files,
# certificates, etc. If there are configuration or cache files from a previous run,
# we remove them here.
shutil.rmtree(mount_dir.joinpath("gen"), ignore_errors=True)
shutil.rmtree(mount_dir.joinpath("gen-cache"),
ignore_errors=True)
kwargs = {}
if not asys.cpu_affinity.is_unrestricted():
kwargs['cpuset_cpus'] = str(asys.cpu_affinity)
cntr = start_scion_cntr(dc,
const.AS_IMG_NAME,
cntr_name=cntr_name,
mount_dir=mount_dir,
ports=ports,
extra_args=kwargs)
asys.container_id = cntr.id
else: # Start container on a remote host
kwargs = {}
if not asys.cpu_affinity.is_unrestricted():
kwargs['cpuset_cpus'] = str(asys.cpu_affinity)
cntr = dc.containers.run(
const.AS_IMG_NAME,
name=cntr_name,
tty=True, # keep the container running
detach=True,
ports=ports,
**kwargs)
asys.container_id = cntr.id
log.info("Started container %s [%s] with ID %s.", cntr_name,
asys.host.name, asys.container_id)
if self.coordinator is None:
# If the coordinator creates the gen folder, 'gen-certs.sh' is invoked by
# 'scionlab-config-user'.
# If the topology is generated by 'scion.sh topology', we create the certificates
# now.
run_cmd_in_cntr(cntr,
const.SCION_USER,
"./gen-certs.sh",
check=True)
else:
# Connect the new container to the coordinator.
self.coordinator.bridge.connect(isd_as, asys)
if asys.is_attachment_point or self.coordinator.ssh_management:
# Allow the coordinator to access the container via SSH.
self._authorize_coord_ssh_key(cntr, workdir)
self._start_sshd(cntr)
# Connect bridges SCION links.
connect_bridges(isd_as, asys)
def connect(self, isd_as: ISD_AS, asys: AS) -> None:
ip = unwrap(self.get_ip_address(isd_as))
self._connect(asys.get_container(), ip, asys.host)
def extract_topo_info(topo_file: MutableMapping[str, Any],
name: Optional[str] = None) -> Topology:
"""Initialize a Topology object with information read from a topology definition.
Interface identifiers not specified in the input file are automatically assigned and added to
the returned Topology object and to `topo_file`.
:param topo_file: The input topology file parsed into a dictionary. When the function returns,
the IXP testbed specific entries have been removed.
:param name: An optional name for the topology. This name is added to all containers, network
bridges, etc. to distinguish them from other testbed instances.
:returns: Extracted topology information.
:raises InvalidTopo: The topology file is invalid.
"""
topo = Topology(name)
networks = NetworkFactory()
brs = BrFactory()
ifids = IfIdMapping(topo_file)
# Subnet for automatically generated local docker bridges
if 'link_subnet' in topo_file.get('defaults', {}):
topo.default_link_subnet = ipaddress.ip_network(
topo_file['defaults'].pop("link_subnet"))
topo.ipv6_enabled |= (topo.default_link_subnet.version == 6)
else:
topo.default_link_subnet = None
# Hosts (first pass: create host objects)
localhost = topo.hosts['localhost'] = LocalHost() # always exists
for host_name, host_def in topo_file.get('hosts', {}).items():
if host_name != 'localhost':
if host_name in topo.hosts:
log.error("Multiple hosts with name '%s'.", host_name)
raise errors.InvalidTopo()
if not 'coordinator' in topo_file:
log.error(
"Running a topology spanning multiple hosts requires a coordinator."
)
raise errors.InvalidTopo()
topo.hosts[host_name] = RemoteHost(
host_name,
_get_ip(host_def, 'ssh_host', host_name),
_get_value(host_def, 'username', host_name),
identity_file=host_def.get("identity_file"),
ssh_port=L4Port(int(host_def.get('ssh_port', 22))))
# Networks
if 'networks' in topo_file:
net_defs = topo_file.pop('networks') # remove networks section
for net_name, net_def in net_defs.items():
type = _get_value(net_def, 'type', net_name)
subnet = _get_value(net_def, 'subnet', net_name)
host = topo.hosts[net_def.get('host', 'localhost')]
networks.create(net_name, topo.get_name_prefix(), type, host,
subnet, net_def)
# Hosts (second pass: parse network addresses for host networks)
for host_name, host_def in topo_file.get('hosts', {}).items():
for net, addr in host_def.get('addresses', {}).items():
networks.set_host_ip(net, topo.hosts[host_name], addr)
topo_file.pop('hosts', None) # remove host section
# Coordinator
if 'coordinator' in topo_file:
coord_def = topo_file.pop('coordinator') # remove coordinator section
host = topo.hosts[coord_def.get('host', 'localhost')]
def_name = lambda: topo.get_name_prefix() + const.COORD_NET_NAME
bridge = networks.get(_get_value(coord_def, 'network', 'coordinator'),
def_name, localhost)
cpu_affinity = CpuSet(coord_def.get('cpu_affinity'))
ssh_management = coord_def.get('ssh_management', False)
debug = coord_def.get('debug', True)
compose_path = None
if debug:
if ssh_management:
log.warning(
"Coordinator in debug mode, 'ssh_management' has no effect."
)
else:
compose_path = Path(
_get_value(coord_def, 'compose_path', 'coordinator'))
if 'expose' not in coord_def:
log.warning(
"No interface to publish the coordinator on given. The coordinator will"
" be exposed at http://127.0.0.1:8000.")
coord = Coordinator(topo.get_coord_name(), host, bridge, cpu_affinity,
ssh_management, debug, compose_path)
coord.exposed_at = _get_external_address(coord_def)
for name, data in coord_def['users'].items():
if name is None:
log.error("User name missing.")
raise errors.InvalidTopo()
coord.users[name] = User(data['email'], data['password'],
data.get('superuser', False))
topo.coordinator = coord
# Prometheus
if 'prometheus' in topo_file:
prom_def = topo_file.pop('prometheus') # remove prometheus section
host = topo.hosts[prom_def.get('host', 'localhost')]
def_name = lambda: topo.gen_bridge_name()
bridge = networks.get(_get_value(prom_def, 'network', 'coordinator'),
def_name, localhost)
if not bridge.is_docker_managed:
log.error("Invalid network type for Prometheus.")
raise InvalidTopo()
prom = Prometheus(
host,
cast(DockerNetwork, bridge),
cpu_affinity=CpuSet(prom_def.get('cpu_affinity')),
scrape_interval=prom_def.get('scrape_interval', "30s"),
storage_dir=_get_optional_path(prom_def, 'storage_dir'),
targets=[ISD_AS(target) for target in prom_def['targets']])
prom.exposed_at = _get_external_address(prom_def)
topo.additional_services.append(prom)
# IXP definitions
for ixp_name, ixp_def in topo_file.pop('IXPs',
{}).items(): # remove IXP section
if ixp_name in topo.ixps:
log.error("IXP %s is defined multiple times.", name)
raise errors.InvalidTopo()
net_name = _get_value(ixp_def, 'network', ixp_name)
def_name = lambda: topo.get_name_prefix() + ixp_name
bridge = networks.get(net_name, def_name, localhost)
topo.ixps[ixp_name] = Ixp(bridge)
# ASes
for as_name, as_def in topo_file['ASes'].items():
isd_as = ISD_AS(as_name)
host_name = as_def.get('host', 'localhost')
host = None
try:
host = topo.hosts[host_name]
except KeyError:
log.error("Invalid host: '%s'.", as_def[host_name])
raise
cpu_affinity = CpuSet(as_def.get('cpu_affinity'))
asys = AS(host, as_def.get('core', False), cpu_affinity)
asys.is_attachment_point = as_def.pop('attachment_point', False)
asys.owner = as_def.pop('owner', None)
topo.ases[isd_as] = asys
if topo.coordinator:
for ixp_name in as_def.pop('ixps', []):
if asys.owner is None:
log.error("Infrastructure AS %s has an IXP list.", isd_as)
raise errors.InvalidTopo()
ixp = topo.ixps[ixp_name]
ixp.ases[isd_as] = asys
# Add dummy link to IXP to make sure there is a network connection.
# Actual links will be configured by the coordinator.
# The border router of the link endpoint is labeled here to avoid creating a new
# border router for every IXP link.
end_point = LinkEp(isd_as,
ifid=ifids.assign_ifid(isd_as),
br_label='peer')
link = Link(end_point, LinkEp(), LinkType.UNSET)
link.bridge = ixp.bridge
topo.links.append(link)
brs.add_link_ep(end_point, link)
# Link definitions
for link in topo_file['links']:
a, b = LinkEp(link['a']), LinkEp(link['b'])
# Assing IfIds if not given in the original topo file.
# Setting the IDs of all interfaces in the processed topology file ensures we can identify
# the interfaces in the configuration files generated by scion.sh.
for ep, name in [(a, 'a'), (b, 'b')]:
if ep.ifid is None:
ep.ifid = ifids.assign_ifid(ep)
link[name] = "{}#{}".format(link[name], ep.ifid)
topo.links.append(Link(a, b, link['linkAtoB']))
# Keep track of border routers that will be created for the links.
brs.add_link_ep(a, topo.links[-1])
brs.add_link_ep(b, topo.links[-1])
# Assign to a network if an IXP name or an explicit IP network is given.
if "network" in link:
net = link.pop('network')
if net in topo.ixps: # use the IXPs network
ixp = topo.ixps[net]
topo.links[-1].bridge = ixp.bridge
ixp.ases[a] = topo.ases[a]
ixp.ases[b] = topo.ases[b]
else:
def_name = lambda: topo.gen_bridge_name()
topo.links[-1].bridge = networks.get(net, def_name, localhost)
else:
if topo.ases[a].host != topo.ases[b].host:
log.error(
"Links between ASes on different hosts must specify the network to use."
)
raise errors.InvalidTopo()
# Enable IPv6 support if needed.
topo.ipv6_enabled = networks.is_ipv6_required()
# Store bridges in topology.
topo.bridges = networks.get_bridges()
# Store border router info in corresponsing AS.
for isd_as, asys in topo.ases.items():
asys.border_routers = brs.get_brs(isd_as)
return topo