def _gen_create_link(link: Link, out) -> None:
"""Generate code creating a link between two already existing BR interfaces.
:param out: Stream the generated code is written to.
"""
_gen_get_iface('a', link.ep_a, unwrap(link.ep_a.ifid), out)
_gen_get_iface('b', link.ep_b, unwrap(link.ep_b.ifid), out)
if link.type != LinkType.PARENT:
out.write("Link.objects.create(%s, a, b)\n" %
_get_link_type_constant(link.type))
else:
# The coordinator knows only 'child' ('PROVIDER') links, no 'parent' links.
out.write("Link.objects.create(%s, b, a)\n" %
_get_link_type_constant(LinkType.CHILD))
def connect(self, isd_as: ISD_AS, asys: AS) -> None:
ip = unwrap(self.get_ip_address(isd_as))
interface_address = _make_interface_address(ip, self.ip_network.prefixlen).with_prefixlen
command = [
# interface name in the container = bridge name
"sudo", "ovs-docker", "add-port", self.name, self.name, asys.container_id,
"--ipaddress={}".format(interface_address)
]
try:
result = asys.host.run_cmd(command, check=True, capture_output=True)
log.info("Connected %s (%s) to %s.", isd_as, interface_address, self.name)
if len(unwrap(result.output)) > 0:
log.info("ovs-docker returned:\n%s", result.output)
except errors.ProcessError as e:
log.error("Connecting %s (%s) to OVS bridge %s failed: %s",
isd_as, interface_address, self.name, e.output)
raise
def disconnect(self, isd_as: ISD_AS, asys: AS) -> None:
try:
command = [
"sudo", "ovs-docker", "del-port", self.name, self.name, asys.container_id,
]
result = asys.host.run_cmd(command, check=True, capture_output=True)
log.info("Disconnected %s from %s.", isd_as, self.name)
if len(unwrap(result.output)) > 0:
log.info("ovs-docker returned:\n{}".format(result.output))
except errors.ProcessError as e:
log.error("Disconnecting %s from OVS bridge %s failed: %s", isd_as, self.name, e.output)
def _get_ap_links(topo, user_as: AS) -> List[AttachmentLink]:
"""Get all links connecting a user AS to attachment points."""
links = []
for user_ifid, link in user_as.links():
if link.is_dummy():
continue
elif topo.ases[link.ep_a].is_attachment_point:
ap, user = link.ep_a, link.ep_b
ap_underlay_addr, user_underlay_addr = link.ep_a_underlay, link.ep_b_underlay
elif topo.ases[link.ep_b].is_attachment_point:
ap, user = link.ep_b, link.ep_a
ap_underlay_addr, user_underlay_addr = link.ep_b_underlay, link.ep_a_underlay
else:
continue # not an AP link
links.append(
AttachmentLink(
user_ifid, unwrap(user_underlay_addr),
link.bridge.get_br_bind_address(user, topo.ases[user],
user_ifid), ap,
unwrap(ap_underlay_addr),
link.bridge.get_br_bind_address(ap, topo.ases[ap], ap.ifid)))
return links
def test_br_addr_assignment(self):
"""Test assignment of (IP, port) tuples to border router interfaces."""
br = DockerBridge("test", LocalHost(),
ipaddress.IPv4Network("10.0.0.0/29"))
asys = AS(LocalHost(), False)
# Assign BR interface addresses
for _ in range(2): # Multiple calls must return the same addresses
for ifid in range(1, 3):
for as_id in range(1, 6):
ip, port = br.assign_br_address(ISD_AS(as_id), asys,
IfId(ifid))
self.assertEqual(
ip, ipaddress.IPv4Address(0x0A000000 + as_id + 1))
self.assertEqual(port, 50000 + ifid - 1)
with self.assertRaises(errors.OutOfResources):
br.assign_br_address(ISD_AS(8), asys, IfId(1))
# AS IP assignment
for as_id in range(1, 6):
isd_as = ISD_AS(as_id)
self.assertEqual(br.get_ip_address(isd_as),
br.assign_ip_address(isd_as))
# Retrieve BR interface underlay addresses
for ifid in range(1, 3):
for asys in range(1, 6):
ip, port = unwrap(br.get_br_address(ISD_AS(asys), IfId(ifid)))
self.assertEqual(ip,
ipaddress.IPv4Address(0x0A000000 + asys + 1))
self.assertEqual(port, 50000 + ifid - 1)
# Free BR interfaces
for asys in range(1, 6):
for ifid in range(1, 3):
self.assertEqual(br.free_br_address(ISD_AS(asys), IfId(ifid)),
3 - ifid)
# Check whether IPs are still bound
for asys in range(1, 6):
ip = br.get_ip_address(ISD_AS(asys))
self.assertEqual(ip, ipaddress.IPv4Address(0x0A000000 + asys + 1))
def start(self, *, host: Host, bridge: Bridge, internal_url: str,
publish_at: Optional[UnderlayAddress], cpu_affinity: CpuSet,
**args) -> None:
dc = host.docker_client
# Check wheather the coordinator is already running
if self._cntr_id:
try:
cntr = dc.containers.get(self._cntr_id)
except docker.errors.NotFound:
self._cntr_id = None
else:
if cntr.status == 'running':
return # coordinator is already running
else:
# Remove old container
cntr.stop()
cntr.remove()
# Expose coordinator on host interface
ports = {}
if publish_at is not None:
external_ip, external_port = publish_at
ports['%d/tcp' % const.COORD_PORT] = (str(external_ip),
int(external_port))
log.info("Exposing coordinator at http://%s",
publish_at.format_url())
# Create and run the container
kwargs = {}
if not cpu_affinity.is_unrestricted():
kwargs['cpuset_cpus'] = str(cpu_affinity)
cntr = dc.containers.run(const.COORD_IMG_NAME,
name=self._cntr_name,
ports=ports,
environment={"SCIONLAB_SITE": internal_url},
detach=True,
**kwargs)
self._cntr_id = cntr.id
log.info("Started coordinator %s [%s] (%s).", self._cntr_name,
host.name, self._cntr_id)
ip = unwrap(bridge.get_ip_address(self._COORD_DEBUG_SERVER))
bridge.connect_container(cntr, ip, host)
def __init__(self,
coord_name: str,
host: Host,
bridge: Bridge,
cpu_affinity: CpuSet = CpuSet(),
ssh_management: bool = False,
debug: bool = True,
compose_path: Optional[Path] = None):
"""
:param coord_name: Name of the coordinator instance. Used as a name prefix for images,
containers, networks, and volumes created for the coordinator.
:param host: Host running the coordinator.
:param cpu_affinity: The CPUs on `host` the coordinator is allowed to run on.
:param ssh_management: Controls whether the coordinator has SSH access to all ASes, instead
of just attachment points. Automatic deployment of AS configurations
is only available for APs and currently only works if `debug=false`.
:param debug: Whether to run the coordinator in debug mode. If the coordinator is run in
debug mode, it uses an SQLite database and runs in a single container.
If debug mode is disabled, multiple container are started by invoking
docker-compose on `host`. In this mode a PosgreSQL DB stores the coordinator's
data. Running docker-compose requires the coordinator's source to be present
on `host`. Use `compose_path` to specify the location of the directory
containing the compose file on `host`.
:param compose_path: Path to the coordinator's docker-compose file on `host`. Only necessary
when `debug` is false.
"""
self.host = host
self.cpu_affinity = cpu_affinity
self.bridge = bridge
self.exposed_at: Optional[UnderlayAddress] = None
self.users: Dict[str, User] = {}
self.api_credentials: Dict[ISD_AS, ApiCredentials]
self.ssh_management = ssh_management
if debug:
self._containers = _DebugCoord(coord_name)
else:
self._containers = _ProductionCoord(coord_name,
unwrap(compose_path))
self._initialized = False
def get_cntr_ip(self, network: str) -> IpAddress:
"""Get the IP address of the AS container in the given network. Requirers the AS container
to exist.
This methods retrieves the IP addresses actually in use by Docker right now, not the
assignments from the Bridge instances of the ASes links. Therefore it can get IP addresses
from networks which were not explicitly configured by this script, like the default Docker
bridge.
"""
template = "'{{with index .NetworkSettings.Networks \"%s\"}}{{.IPAddress}}{{end}}'" % network
try:
_, output = self.host.run_cmd([
"docker", "inspect", "-f", template,
unwrap(self.container_id)
],
check=True,
capture_output=True)
return ipaddress.ip_address(output.strip("'\n"))
except Exception:
log.error(
"Could not retrieve IP address of container '%s' in network '%s'.",
self.container_id, network)
raise
def getgid(self) -> int:
"""Get the group id on the remote host."""
res = self.run_cmd(["id", "-g"], check=True, capture_output=True)
return int(unwrap(res.output))
def assign_underlay_addresses(topo: Topology) -> None:
"""Assign underlay addresses to the border router interfaces in the given topology.
:raises OutOfResources: Not enough underlay addresses availabile.
"""
link_subnets = None
if topo.default_link_subnet:
def_subnet = topo.default_link_subnet
prefixlen_diff = def_subnet.max_prefixlen - def_subnet.prefixlen - LINK_SUBNET_HOST_LEN
if prefixlen_diff >= 0:
link_subnets = topo.default_link_subnet.subnets(prefixlen_diff)
# Wrapper around IP network host iterator.
class HostAddrGenerator:
def __init__(self, bridge: Bridge):
self._iter = bridge.valid_ip_iter()
self.current = next(self._iter)
def next(self):
self.current = next(self._iter)
# Mapping from IP subnet to generator producing addresses from said subnet.
addr_gens: Dict[IpNetwork, HostAddrGenerator] = {}
for link in topo.links:
if link.bridge is None: # assign a subnet of the default link network
# DockerBridge cannot span multiple hosts.
assert topo.ases[link.ep_a].host == topo.ases[link.ep_b].host
if not link_subnets:
log.error("No default link network specified.")
raise errors.OutOfResources()
try:
ip_net = next(link_subnets)
link.bridge = DockerBridge(
topo.gen_bridge_name(), topo.ases[link.ep_a].host, ip_net)
topo.bridges.append(link.bridge)
except StopIteration:
log.error("Not enough IP addresses for all links.")
raise errors.OutOfResources()
# Assign IP addresses to link endpoints
addr_gen = _lazy_setdefault(addr_gens, link.bridge.ip_network,
lambda: HostAddrGenerator(unwrap(link.bridge)))
try:
if not link.ep_a.is_zero():
link.ep_a_underlay = link.bridge.assign_br_address(
link.ep_a, topo.ases[link.ep_a], link.ep_a.ifid,
pref_ip=None if isinstance(link.bridge, HostNetwork) else addr_gen.current)
if link.ep_a_underlay.ip == addr_gen.current:
addr_gen.next()
if not link.ep_b.is_zero():
link.ep_b_underlay = link.bridge.assign_br_address(
link.ep_b, topo.ases[link.ep_b], link.ep_b.ifid,
pref_ip=None if isinstance(link.bridge, HostNetwork) else addr_gen.current)
if link.ep_b_underlay.ip == addr_gen.current:
addr_gen.next()
except (errors.OutOfResources, StopIteration):
log.error("Not enough IP addresses in subnet '%s'.", link.bridge.ip_network)
raise errors.OutOfResources()
def connect(self, isd_as: ISD_AS, asys: AS) -> None:
ip = unwrap(self.get_ip_address(isd_as))
self._connect(asys.get_container(), ip, asys.host)
def start(self, *, host: Host, bridge: Bridge, internal_url: str,
publish_at: Optional[UnderlayAddress], cpu_affinity: CpuSet,
**args) -> None:
dc = host.docker_client
# Check wheather the coordinator is already running
restart_existing = False
if self._django_cntr_id:
result = host.run_cmd(self._compose_cmd(["ps"]),
check=True,
capture_output=True)
lines = result.output.splitlines()
if len(lines) < 3:
# coordinator is down
restart_existing = False
else:
for line in lines:
if line.startswith(
self._project_name) and not "Up" in line:
# some containers are not running
restart_existing = True
break
else:
return # coordinator is already running
# Invoke docker-compose
if restart_existing:
result = host.run_cmd(self._compose_cmd(["restart"]),
check=True,
capture_output=True)
log.info("Restarting coordinator containers:\n" + result.output)
else:
env = {
"SCIONLAB_SITE": internal_url,
"COORD_CPUSET": str(cpu_affinity),
"COORD_ADDR": str(publish_at) if publish_at is not None else ""
}
if publish_at is not None:
log.info("Exposing coordinator at http://%s",
publish_at.format_url())
result = host.run_cmd(self._compose_cmd(["up", "--detach"]),
env=env,
check=True,
capture_output=True)
log.info("Starting coordinator:\n%s", result.output)
# Get the django and the caddy container
django_cntr = dc.containers.get(self._project_name + "_django_1")
self._django_cntr_id = django_cntr.id
caddy_cntr = dc.containers.get(self._project_name + "_caddy_1")
self._caddy_cntr_id = caddy_cntr.id
huey_cntr = dc.containers.get(self._project_name + "_huey_1")
self._huey_cntr_id = huey_cntr.id
# Connect caddy to the coordinator network to publish the web interface
bridge.connect_container(
caddy_cntr, unwrap(bridge.get_ip_address(self._COORD_CADDY)), host)
# Connect huey to the coordinator network to enable AS configuration over SSH
bridge.connect_container(
huey_cntr, unwrap(bridge.get_ip_address(self._COORD_HUEY)), host)
def get_http_interface(self, bridge: Bridge) -> UnderlayAddress:
return UnderlayAddress(
unwrap(bridge.get_ip_address(self._COORD_CADDY)),
L4Port(const.COORD_PORT))