def __init__(self, net, clash):
self.net = net
self.clash = clash
UvnException.__init__(
self,
f"remote site clash detected: {self.net.handle} x-x {self.clash.handle}"
)
def _load(self):
(self._vpn_cls,
self._vpn_extra) = self._get_vpn_class()
(self._router_cls,
self._router_extra) = self._get_router_class()
(self._participant_cls,
self._participant_extra,) = self._get_participant_class()
self._connection_test_peers = self._get_connection_test_peers()
self.vpn = self._create_vpn(self._vpn_cls, self._vpn_extra)
self.router = self._create_router(self._router_cls, self._router_extra)
self.participant = self._create_participant(
self._participant_cls, self._participant_extra)
self.connection_test = self._create_connection_test(
self._connection_test_peers)
# register agent as the nameserver's listener
self.registry.nameserver.listener = self
# Determine default gateway
self._default_gw = ip.ipv4_default_gateway()
# Read network interfaces to determine the list of networks to which
# the agent is attached
self._local_sites = self._list_local_sites()
# Determine list of private addresses on which to listen
self._private_ports = {n["address"] for n in self._local_sites}
self._loaded = True
self._ts_loaded = Timestamp.now()
logger.info("[loaded] UVN agent: {}", self.agent_id())
if self.registry.packaged:
if not self.registry.bootstrapped:
logger.warning("no deployment loaded")
elif not self.registry.deployed_cell_config:
raise UvnException(f"configuration not found: {self.registry.deployed_cell.id.name}@{self.registry.deployment_id}")
def _find_cell_peer(self, cell_name, noexcept=False):
for p in self.peers:
if p.cell.id.name == cell_name:
return self._find_peer_connection(cell_name, noexcept=noexcept)
if not noexcept:
raise UvnException("failed to find peer for {}".format(cell_name))
return None, None
def _enable_nat(self):
if self._nat_wgs:
raise UvnException("[NAT] already enabled")
try:
self._nat_nets = []
self._nat_wgs = []
wg_nics = list(self.list_wg_interfaces())
local_nets = list(self.list_local_networks())
logger.activity("[NAT][enable] wireguard interfaces: {}", wg_nics)
for nic in wg_nics:
ip.ipv4_enable_forward(nic)
ip.ipv4_enable_output_nat(nic)
self._nat_wgs.append(nic)
logger.activity("[NAT][enable] local LANs: {}",
list(map(lambda n: n["nic"], local_nets)))
for net in local_nets:
ip.ipv4_enable_output_nat(net["nic"])
self._nat_nets.append(net)
except Exception as e:
logger.exception(e)
logger.error("failed to enable NAT for local networks")
# Try to disable NAT on already enabled nics
self._disable_nat()
raise e
def register_cell(
self,
name,
address=None,
admin=None,
admin_name=None,
location=None,
peer_ports=None,
# Catch all other extra arguments, to allow users to
# store custom entries in the input YAML file
**kwargs):
if name in self.cells:
raise UvnException(f"cell name already in use : '{name}'")
if address is None:
address = "{}.{}".format(name,
self.identity_db.registry_id.address)
if admin is None:
admin = self.identity_db.registry_id.admin
if admin_name is None:
admin_name = UvnDefaults["registry"]["admin_name"]
if location is None:
location = UvnDefaults["cell"]["location"]
if peer_ports is None:
peer_ports = list(UvnDefaults["cell"]["peer_ports"])
self.identity_db.register_cell(name=name,
address=address,
admin=admin,
admin_name=admin_name,
generate=True)
keymat = CellKeyMaterial()
cell_id = CellIdentity(name=name,
n=len(self.cells) + 1,
address=address,
keymat=keymat,
location=location,
admin=admin,
admin_name=admin_name)
cell = Cell(cell_id=cell_id,
psk=wg.genkeypreshared(),
peer_ports=peer_ports)
self.cells[cell.id.name] = cell
logger.debug("registered cell: {} ({})", cell.id.name, cell.id.address)
self._generate_vpn_config()
self._generate_router_ports()
self._generate_particles()
self.dirty = True
return cell
def _find_peer_connection(self, peer_name, noexcept=False):
for bbone_connection in self.backbone:
for pc in bbone_connection.peers:
if pc.name == peer_name:
return bbone_connection.interface, pc
if not noexcept:
raise UvnException(
"failed to find peer connection for {}".format(peer_name))
return None, None
def _random_port_number(self):
max_tries = 1e12
i = 0
n = None
while n is None and i < max_tries:
n = random.randint(self.min, self.max)
if n in self.in_use or n in self.reserved:
n = 0
i += 1
if not n:
raise UvnException(f"failed to pick a port after {i} tries")
return n
def _install_handler(k, handler):
s = {
"SIGINT": signal.SIGINT,
"SIGUSR1": signal.SIGUSR1,
"SIGUSR2": signal.SIGUSR2
}.get(k)
if not s:
raise UvnException(f"unsupported signal: {k}")
def _handler(sig, frame):
if logger:
logger.warning("received {}", k)
handler()
signal.signal(s, _handler)
return k
def load(registry_dir, keep=False, roaming=False, daemon=False, interfaces=[]):
from .agent_cell import CellAgent
from .agent_root import RootAgent
registry_dir = pathlib.Path(registry_dir)
identity_db = UvnIdentityDatabase.load(basedir=registry_dir)
registry = UvnRegistry.load(identity_db)
if registry.packaged:
return CellAgent(registry, keep=keep, roaming=roaming, daemon=daemon, interfaces=interfaces)
else:
if roaming:
raise UvnException("roaming mode not supported for root agent")
return RootAgent(registry, keep=keep, daemon=daemon, interfaces=interfaces)
def __init__(self,
commands=[],
command_mappings={},
builtin_mappings=_builtin_mappings,
nobuiltin=False,
args=None,
daemon=False,
cli=False):
self.daemon = daemon
self.cli = cli
if (self.daemon and self.cli) or (not self.daemon and not self.cli):
raise ValueError(self.daemon, self.cli)
commands = list(commands)
command_mappings = dict(command_mappings)
if not nobuiltin:
commands.extend(_builtin_commands)
command_mappings.update(builtin_mappings)
self.commands = cmd_create_all(self, commands, command_mappings)
if len(self.commands) == 0:
raise UvnException("no uvn command enabled")
(self.parser, self._subparsers, self._ssubparsers,
self._scmdparsers) = Uvn.define_parser(self.commands, daemon=daemon)
if not args:
# If uvn was spawned from cli, use sys.argv as default arguments
if cli:
args = sys.argv[1:]
else:
args = []
if self.daemon:
# Daemon always runs the "agent" command
parser_args = ["A"]
parser_args.extend(args)
else:
parser_args = args
self.args = self.parser.parse_args(args=parser_args)
# Handle global options (must be applied before commands)
self._handle_global_args()
# Load paths object based on selected target directory
self.paths = UvnPaths(basedir=self.args.directory)
def _load_deployment(self, deployment_id, store=True, deployment_dir=None):
if (deployment_dir is None):
deployment_dir = self.paths.dir_deployment(deployment_id)
deployment_manifest_file = deployment_dir / UvnDefaults["registry"][
"deployment_file"]
db_args = UvnIdentityDatabase.get_load_args(
identity_db=self.identity_db)
deployment = yml_obj(UvnDeployment,
deployment_manifest_file,
from_file=True,
registry=self,
**db_args)
if (deployment.id != deployment_id):
raise UvnException(
f"Invalid deployment loaded: {deployment.id}, expected {deployment_id}"
)
if (store):
self.deployments.append(deployment)
return deployment
def deployments(agent):
latest_deployment = agent.registry.latest_deployment
if latest_deployment is None:
logger.warning("no deployment generated")
return
logger.activity("publishing latest deployment: {}", latest_deployment.id)
installers = agent.registry._list_deployment_installers(
deployment=latest_deployment)
installers = {
str(pathlib.Path(i).stem).split("-")[-1]: i
for i in installers
}
missing_cells = [
c for c in agent.registry.cells.keys() if c not in installers
]
if missing_cells:
raise UvnException(f"missing deployment installers: {missing_cells}")
return list(
map((lambda i: deployment(agent, latest_deployment.id, *i)),
installers.items()))
def register_particle(self, name, contact=None):
if name in self.particles:
raise UvnException(f"particle name already in use : '{name}'")
if contact is None:
contact = "{}@{}".format(name, self.address)
particle = Particle(name=name,
n=len(self.particles) + 1,
contact=contact)
self.particles[particle.name] = particle
logger.debug("registered particle: {} ({})", particle.name,
particle.contact)
self._generate_particles()
self.dirty = True
return particle
def repr_py(self, yml_repr, **kwargs):
identity_db = kwargs["identity_db"]
nameserver = kwargs["nameserver"]
deployments = kwargs.get("deployments")
cells = {
c["id"]["name"]: repr_py(Cell, c, **kwargs)
for c in yml_repr["cells"]
}
particles = {
p["name"]: repr_py(Particle, p, **kwargs)
for p in yml_repr["particles"]
}
keymat = repr_py(WireGuardKeyPair, yml_repr["keymat"], **kwargs)
if "vpn_config" in yml_repr:
vpn_config = repr_py(UvnRegistry.RegistryVpn,
yml_repr["vpn_config"], **kwargs)
else:
vpn_config = None
if "router_ports" in yml_repr:
router_ports = repr_py(RegistryRouterPorts,
yml_repr["router_ports"], **kwargs)
else:
router_ports = None
py_repr = UvnRegistry(identity_db=identity_db,
cells=cells,
particles=particles,
keymat=keymat,
ports=yml_repr["ports"],
loaded=True,
pkg_cell=yml_repr.get("pkg_cell", None),
deployment_id=yml_repr.get(
"deployment_id", None),
vpn_config=vpn_config,
nameserver=nameserver,
router_ports=router_ports)
# Register cells with identity_db
for c in py_repr.cells.values():
with_secret = (py_repr.packaged
and c.id.name == py_repr.pkg_cell)
py_repr.identity_db.register_cell(name=c.id.name,
address=c.id.address,
admin=c.id.admin,
admin_name=c.id.admin_name,
with_secret=with_secret)
deployment_loaded = False
if ("deployments" in yml_repr):
for d in yml_repr["deployments"]:
if (deployments is not None and d in deployments):
py_repr.deployments.append(deployments[d])
continue
try:
py_repr._load_deployment(deployment_id=d)
if d == py_repr.deployment_id:
deployment_loaded = True
except Exception as e:
# traceback.print_exc()
logger.exception(e)
logger.warning("failed to load deployment {}: {}", d,
e)
if not py_repr.packaged:
# Generate particle configurations
py_repr._generate_particles()
else:
if (py_repr.deployment_id !=
UvnDefaults["registry"]["deployment_bootstrap"]
and not deployment_loaded):
raise UvnException(
f"required deployment not loaded: {py_repr.deployment_id}"
)
try:
file_path = py_repr.paths.basedir / UvnDefaults[
"registry"]["cell_file"]
db_args = UvnIdentityDatabase.get_load_args(
identity_db=identity_db)
cell = yml_obj(Cell,
file_path,
from_file=True,
identity_db=identity_db,
**db_args)
if cell.id.name != py_repr.pkg_cell:
raise UvnException(
f"invalid UVN package: expected={py_repr.pkg_cell}, found={cell.id.name}"
)
py_repr.cells[cell.id.name] = cell
# Generate particle server
py_repr._register_particles(py_repr.deployed_cell)
# logger.activity("[loaded] UVN package for {} [{}]",
# cell.id.name, py_repr.deployment_id)
except Exception as e:
raise UvnException(
f"failed to load UVN package for {py_repr.pkg_cell}: {e}"
)
logger.activity(
"[{}]{} loaded UVN: {}",
py_repr.pkg_cell if py_repr.packaged else "root",
f"[{py_repr.deployment_id}]" if py_repr.packaged else "",
py_repr.identity_db.registry_id.address)
return py_repr
def _export_cell_package(self,
pkg_dir,
cell=None,
cell_cfg=None,
deployment=None,
keep=False):
# Check if we are generating a "bootstrap" or a "deployment" package
bootstrap = cell_cfg is None or deployment is None
if cell_cfg is not None and deployment is None:
raise UvnException("no deployment with cell_cfg: {}",
cell_cfg.cell.id.name)
if not bootstrap:
cell = cell_cfg.cell
deployment_id = deployment.id
archive_out_dir = self.paths.dir_deployment(
deployment_id) / UvnDefaults["registry"]["deployment_packages"]
else:
deployment_id = UvnDefaults["registry"]["deployment_bootstrap"]
archive_out_dir = self.paths.dir_cell_bootstrap(
) / UvnDefaults["registry"]["deployment_packages"]
logger.debug("Generating package for cell {} in {}", cell.id.name,
pkg_dir)
pkg_dir.mkdir(parents=True, exist_ok=True)
db_args = self.identity_db.get_export_args()
# Serialize cell manifest
cell_manifest = pkg_dir / UvnDefaults["registry"]["cell_file"]
yml(cell, to_file=cell_manifest, **db_args)
# Serialize registry manifest
registry_manifest = pkg_dir / UvnDefaults["registry"]["persist_file"]
yml(self,
to_file=registry_manifest,
target_cell=cell,
deployed_cell=cell_cfg,
deployment_id=deployment_id,
**db_args)
# Serialize nameserver database
ns_db = pkg_dir / UvnDefaults["nameserver"]["persist_file"]
yml(self.nameserver, to_file=ns_db, **db_args)
db_export_args = {"tgt_cell": cell}
# Serialize identity database
(db_dir, db_manifest,
cell_secret) = self.identity_db.export_cell(self,
pkg_dir=pkg_dir,
**db_export_args)
if not bootstrap:
# Serialize deployment
deployment_manifest = self.paths.dir_deployment(
deployment_id=deployment.id,
basedir=pkg_dir) / UvnDefaults["registry"]["deployment_file"]
yml(deployment,
to_file=deployment_manifest,
tgt_cell_cfg=cell_cfg,
deployment_id=deployment.id,
**db_args)
# Create package archive
archive_path = self._zip_cell_pkg(deployment_id=deployment_id,
cell_name=cell.id.name,
cell_pkg_dir=pkg_dir,
archive_out_dir=archive_out_dir)
# Encrypt archive
encrypt_result = self._encrypt_file_for_cell(cell.id.name,
archive_path)
if not keep:
# Delete unencrypted archive
archive_path.unlink()
# Delete staging directory
try:
# For some reason, this call succeeds on x86_64, but fails
# on RPi with error: "No such file or directory: 'S.gpg-agent.ssh'"
shutil.rmtree(str(pkg_dir))
except Exception as e:
logger.exception(e)
logger.warning("failed to remove build directory: {}", pkg_dir)
else:
logger.warning("[tmp] not deleted: {}", archive_path)
logger.warning("[tmp] not deleted: {}", pkg_dir)
cell_record = self.identity_db.get_cell_record(cell.id.name)
if not cell_record:
raise UvnException(
f"cell record not found in db: {cell.id.address}")
installer = UvnCellInstaller(
uvn_admin=self.identity_db.registry_id.admin,
uvn_address=self.identity_db.registry_id.address,
uvn_deployment=deployment_id,
cell_name=cell.id.name,
cell_address=cell.id.address,
cell_admin=cell.id.admin,
uvn_public_key=self.identity_db.registry_id.key.public,
cell_public_key=cell_record.key.public,
cell_private_key=cell_record.key.private,
cell_secret=self.identity_db._secret_cell(cell.id.name,
cell.id.admin),
cell_pkg=str(encrypt_result["output"]),
cell_sig=str(encrypt_result["signature"]))
basedir = self.paths.basedir / UvnDefaults["registry"]["installers_dir"]
installer.export(basedir=basedir, keep=keep)
def bootstrap(package, install_prefix, keep=False):
package = pathlib.Path(package)
install_prefix = pathlib.Path(install_prefix).resolve()
logger.activity("installing cell package: {}", package.name)
# Create a temporary directory to extract the installer and bootstrap
# the gpg database
tmp_dir = tempfile.mkdtemp(prefix="{}-".format(package.stem))
tmp_dir = pathlib.Path(tmp_dir)
try:
logger.debug("extracting {} to {}", package, tmp_dir)
shutil.unpack_archive(str(package), extract_dir=str(tmp_dir),
format=UvnDefaults["cell"]["pkg"]["clear_format"])
# Load installer manifest
manifest = UvnCellInstaller._manifest_file(tmp_dir)
installer = yml_obj(UvnCellInstaller, manifest, from_file=True)
logger.debug("loaded installer for cell {} of UVN {} [{}]",
installer.cell_name, installer.uvn_address, installer.uvn_deployment)
installer_files = UvnCellInstaller._installer_files(
tmp_dir, bootstrap=installer._bootstrap)
# Check that all files are there as expected
missing_files = [str(f) for f in installer_files.values()
if not f.exists()]
if missing_files:
raise UvnException("missing uvn installer files: [{}]".format(
",".join(missing_files)))
installer_dir = tmp_dir / UvnDefaults["cell"]["pkg"]["export_name"]
if installer._bootstrap:
logger.activity("bootstrap: {} -> {}", package.stem, install_prefix)
bootstrap_dir = installer_dir
registry = None
else:
# extract deployment package into target cell's dir
logger.activity("deployment: {} -> {}", package.stem, install_prefix)
bootstrap_dir = install_prefix
identity_db = UvnIdentityDatabase.load(basedir=bootstrap_dir)
from libuno.reg import UvnRegistry
registry = UvnRegistry.load(identity_db)
# Decrypt cell package and extract it
UvnIdentityDatabase.bootstrap_cell(
bootstrap_dir=bootstrap_dir,
registry=registry,
uvn_address=installer.uvn_address,
uvn_admin=installer.uvn_admin,
cell_name=installer.cell_name,
cell_admin=installer.cell_admin,
cell_pkg=installer_files["cell_pkg"],
cell_sig=installer_files["cell_sig"],
uvn_public_key=installer_files.get("uvn_public_key"),
cell_public_key=installer_files.get("cell_public_key"),
cell_private_key=installer_files.get("cell_private_key"),
cell_secret=installer_files.get("cell_secret"),
keep=keep)
if installer._bootstrap:
shutil.copytree(str(installer_dir), str(install_prefix))
finally:
if not keep:
# Delete temporary directory
shutil.rmtree(str(tmp_dir))
else:
logger.warning("[tmp] not deleted: {}", tmp_dir)
logger.activity("installed package: {} -> {}", package.name, install_prefix)
def __init__(self, registry):
if registry.packaged:
raise UvnException("gateway must be run on root registry")
self.registry = registry
def _create_entity(participant, ep_type, ep_key, ep_name, load_fn):
logger.debug("dds {} [{}]: {}", ep_type, ep_key, ep_name)
ep = load_fn(participant, ep_name)
if ep is None:
raise UvnException(f"unknown {ep_type}: {ep_name}")
return ep