def handle_net_cfg(deque):
"""
Handle the initial net_cfg for a (new) member node. Required format
derived from async wrapper funcs. Context is netstate runner and
bootstrap_mbr_node.
:param deque: netobj queue
:return: tuple of formatted cfg fragments
"""
with deque.transact():
ipnet = deque.popleft()
deque.append(ipnet)
netcfg = ipnet_get_netcfg(ipnet)
gw_ip = find_ipv4_iface(netcfg.gateway[0])
src_ip = find_ipv4_iface(netcfg.host[0])
ip_range = [{
'ipRangeStart': '{}'.format(gw_ip),
'ipRangeEnd': '{}'.format(src_ip)
}]
src_addr = {'ipAssignments': netcfg.host, 'authorized': True}
gw_addr = {'ipAssignments': netcfg.gateway, 'authorized': True}
net_defs = {'routes': netcfg.net_routes, 'ipAssignmentPools': ip_range}
mbr_ip = AttrDict.from_nested_dict(src_addr)
gw_ip = AttrDict.from_nested_dict(gw_addr)
ip_net = AttrDict.from_nested_dict(net_defs)
return ip_net, mbr_ip, gw_ip
def parse_moon_data(data):
"""
Parse moon metadata returned from get_moon_data function so we can
use the ID and endpoint address during daemon startup.
"""
import ipaddress
from node_tools.helper_funcs import AttrDict
result = []
for item in data:
moon_data = AttrDict.from_nested_dict(item)
moon_id = moon_data.id.replace('000000', '', 1)
for root in moon_data.roots:
root_data = AttrDict.from_nested_dict(root)
for endpoint in root_data.stableEndpoints:
addr = endpoint.split('/')
try:
addr_obj = ipaddress.ip_address(addr[0])
except ValueError as exc:
logger.error('ipaddress exception: {}'.format(exc))
# filter out IPv6 addresses for now
if addr_obj.version == 4:
moon_addr = addr[0]
moon_port = addr[1]
else:
return result
result.append((moon_id, moon_addr, moon_port))
return result
def state_check(*args, **kwargs):
"""
Diskcache wrapper for checking nodeState before and after the
update_runner() tries to grab new data.
"""
from node_tools import state_data as st
get_state(cache)
prev_state = AttrDict.from_nested_dict(st.fpnState)
if not prev_state.online:
logger.warning('nodeState not initialized (node not online)')
else:
logger.info('Node online with id: {}'.format(prev_state.fpn_id))
result = func(*args, **kwargs)
get_state(cache)
next_state = AttrDict.from_nested_dict(st.fpnState)
if not next_state.online and not prev_state.online:
logger.warning('nodeState still not initialized (node not online)')
elif next_state.online and prev_state.online:
get_state_values(prev_state, next_state)
if st.changes:
logger.info('NETSTATE: diff is {}'.format(st.changes))
put_state_msg('CONFIG')
if next_state.fallback:
logger.error('NETSTATE: fallback mode is True (network suspect)')
return result
def setUp(self):
super(SetRolesTest, self).setUp()
from node_tools import state_data as st
self.saved_state = AttrDict.from_nested_dict(st.fpnState)
self.default_state = AttrDict.from_nested_dict(st.defState)
self.state = st.fpnState
self.state.update(fpn_id='deadbeef00')
self.role = NODE_SETTINGS['node_role']
def setUp(self):
super(HandleMoonDataTest, self).setUp()
from node_tools import state_data as st
self.saved_state = AttrDict.from_nested_dict(st.fpnState)
self.default_state = AttrDict.from_nested_dict(st.defState)
self.state = st.fpnState
self.client = client = mock_zt_api_client()
_, moon_data = client.get_data('moon')
self.data = parse_moon_data(moon_data)
self.moons = ['deadd738e6']
NODE_SETTINGS['moon_list'] = self.moons
def send_wedged_msg(addr=None):
"""
Send a special msg type if my routing is stuffed.
:param addr: moon address if known
"""
from node_tools.helper_funcs import AttrDict
from node_tools import state_data as st
state = AttrDict.from_nested_dict(st.fpnState)
node_id = state.fpn_id
reply = []
if not addr:
addr = state.moon_addr
if NODE_SETTINGS['use_localhost'] or not addr:
addr = '127.0.0.1'
try:
reply = send_req_msg(addr, 'wedged', node_id)
logger.warning('WEDGED: msg reply: {}'.format(reply))
except Exception as exc:
logger.error('Send error is {}'.format(exc))
return reply
def ipnet_get_netcfg(netobj):
"""
Process a (python) network object into config Attrdict.
:notes: Each item in the route list must be processed as a separate
config fragment.
:param netobj: python subnet object from the netobj queue
:return: `dict` Attrdict of JSON config fragments
"""
import ipaddress as ip
if isinstance(netobj, ip.IPv4Network):
net_cidr = str(netobj)
net_pfx = '/' + str(netobj.prefixlen)
gate_iface = ip.IPv4Interface(str(list(netobj.hosts())[0]) + net_pfx)
host_iface = ip.IPv4Interface(str(list(netobj.hosts())[1]) + net_pfx)
gate_addr = str(gate_iface.ip)
host_cidr = [str(host_iface)]
gate_cidr = [str(gate_iface)]
net_routes = [{
"target": "{}".format(net_cidr)
}, {
"target": "0.0.0.0/0",
"via": "{}".format(gate_addr)
}]
d = {"net_routes": net_routes, "host": host_cidr, "gateway": gate_cidr}
return AttrDict.from_nested_dict(d)
else:
raise ValueError('{} is not a valid IPv4Network object'.format(netobj))
def do_startup(nwid):
"""
Run network startup command once before the state runner is
available (if we have a network and interface up).
"""
import time
from node_tools.helper_funcs import AttrDict
from node_tools.network_funcs import do_net_cmd
from node_tools.network_funcs import get_net_cmds
from node_tools import state_data as st
run_ztcli_cmd(action='join', extra=nwid)
# time.sleep(1)
state = AttrDict.from_nested_dict(st.fpnState)
fpn_home = NODE_SETTINGS['home_dir']
nets = ['fpn_id0', 'fpn_id1']
ifaces = ['fpn0', 'fpn1']
for iface, net in zip(ifaces, nets):
if st.fpnState[iface] and net == nwid:
logger.debug('STARTUP: running setup on {}'.format(iface))
cmd = get_net_cmds(fpn_home, iface, True)
logger.debug('run_net_cmd using cmd: {}'.format(cmd))
schedule.every(1).seconds.do(run_net_cmd, cmd).tag('net-change')
def test_get_state_values():
from node_tools import state_data as stest
from node_tools.data_funcs import get_state_values
assert isinstance(stest.changes, list)
assert not stest.changes
get_state(cache)
prev_state = AttrDict.from_nested_dict(stest.fpnState)
assert prev_state.online
assert prev_state.fpn0
assert prev_state.fpn1
# induce a change
stest.fpnState.update(fpn1=False, fpn_id1=None)
next_state = AttrDict.from_nested_dict(stest.fpnState)
assert not next_state.fpn1
assert not stest.changes
# now we should see old/new values in the state diff
get_state_values(prev_state, next_state, True)
assert isinstance(stest.changes, tuple)
assert len(stest.changes) == 2
assert len(stest.changes[0]) == 2
get_state_values(prev_state, next_state)
assert len(stest.changes[0]) == 2
# reset shared state vars
stest.changes = []
stest.fpnState.update(fpn1=True, fpn_id1='b6079f73ca8129ad')
assert stest.fpnState == prev_state
# induce two changes
stest.fpnState.update(fpn0=False, fpn1=False, fpn_id0=None, fpn_id1=None)
next_state = AttrDict.from_nested_dict(stest.fpnState)
assert not next_state.fpn0
assert not next_state.fpn1
assert not stest.changes
# now we should see only new values for both changes in the state diff
get_state_values(prev_state, next_state)
assert isinstance(stest.changes, tuple)
assert len(stest.changes) == 4
assert len(stest.changes[0]) == 2
# reset shared state vars
stest.changes = []
def get_network_id(data):
"""
Get the network ID from the dict-ish client payload (ie, the content
of client.data) passed to/from `add_network_object`.
:param data: <dict> network attributres
:return: <str> network ID
"""
net_data = AttrDict.from_nested_dict(data)
return net_data.id
def unset_network_cfg():
"""
Create a config fragment to unset (remove) the IP address and
deauthorize the node.
:return: <dict> formatted cfg fragment for async payload
"""
src_addr = {'ipAssignments': [], 'authorized': False}
return AttrDict.from_nested_dict(src_addr)
def test_node_state_check():
from node_tools import state_data as stest
state = AttrDict.from_nested_dict(stest.fpnState)
assert state.cfg_ref is None
res = node_state_check()
assert res is None
state.update(cfg_ref='7f5c6fc9-28d9-45d6-b210-1a08b958e219')
res = node_state_check()
state.update(cfg_ref=None)
def set_network_cfg(cfg_addr):
"""
Take the netcfg for mbr and wrap it so it can be applied during mbr
bootstrap (ie, wrap the address with authorized=True, etc).
:param cfg_addr: <list> gw or host portion of netcfg object
:return: <dict> formatted cfg fragment for async payload
"""
src_addr = {'ipAssignments': cfg_addr, 'authorized': True}
return AttrDict.from_nested_dict(src_addr)
def do_cleanup(path=None, addr=None):
"""
Run network cleanup commands via daemon cleanup hook.
:param path: path to scripts dir
:param addr: moon address if known
"""
from node_tools.helper_funcs import AttrDict
from node_tools.network_funcs import do_net_cmd
from node_tools.network_funcs import get_net_cmds
from node_tools.helper_funcs import put_state_msg
from node_tools.network_funcs import send_req_msg
from node_tools import state_data as st
if NODE_SETTINGS['node_role'] == 'moon':
for script in ['msg_responder.py', 'msg_subscriber.py']:
res = control_daemon('stop', script)
logger.info('CLEANUP: shutting down {}'.format(script))
elif NODE_SETTINGS['node_role'] == 'controller':
for script in ['msg_subscriber.py']:
res = control_daemon('stop', script)
logger.info('CLEANUP: shutting down {}'.format(script))
else:
state = AttrDict.from_nested_dict(st.fpnState)
moon_addr = addr
moon_id = state.moon_id0
node_id = state.fpn_id
if not addr:
moon_addr = state.moon_addr
if not NODE_SETTINGS['use_localhost'] and not addr:
addr = moon_addr
if not path:
path = NODE_SETTINGS['home_dir']
nets = ['fpn_id0', 'fpn_id1']
ifaces = ['fpn0', 'fpn1']
put_state_msg('NONE')
for iface, net in zip(ifaces, nets):
if state[iface]:
logger.info('CLEANUP: shutting down {}'.format(iface))
cmd = get_net_cmds(path, iface)
res = do_net_cmd(cmd)
logger.debug('CLEANUP: {} shutdown returned {}'.format(
iface, res))
# logger.info('CLEANUP: leaving network ID: {}'.format(state[net]))
res = run_ztcli_cmd(action='leave', extra=state[net])
logger.debug('CLEANUP: action leave returned: {}'.format(res))
if moon_id is not None:
run_moon_cmd(moon_id, action='deorbit')
reply = send_req_msg(addr, 'offline', node_id)
logger.debug('CLEANUP: offline reply: {}'.format(reply))
def test_do_cleanup():
from node_tools import state_data as stest
get_state(cache)
nodeState = AttrDict.from_nested_dict(stest.fpnState)
# print(nodeState)
with pytest.raises(FileNotFoundError):
do_cleanup('./bin', '127.0.0.1')
with pytest.raises(FileNotFoundError):
do_cleanup(path='./bin')
def test_send_wedged_no_responder(self):
nodeState = AttrDict.from_nested_dict(self.state)
fpn_id = nodeState.fpn_id
# expected command result is a list so the return
# result for echo_client() is actually None
mock_job = make_mock_job()
tj = every().second.do(mock_job)
send_wedged_msg()
schedule.run_all()
result = send_wedged_msg(self.addr)
self.assertEqual([], result)
def test_send_wedged_no_responder(self):
nodeState = AttrDict.from_nested_dict(self.state)
fpn_id = nodeState.fpn_id
mock_job = make_mock_job()
tj = every().second.do(mock_job)
send_wedged_msg()
schedule.run_all()
# expected command result is a list
result = send_wedged_msg(self.addr)
# print(result)
self.assertEqual([], result)
def test_run_event_handler():
"""
This stuff really needs better tests.
"""
from node_tools import state_data as st
from node_tools.data_funcs import get_state_values
home, pid_file, log_file, debug, msg, mode, role = do_setup()
nets = ['b6079f73c63cea29', 'b6079f73ca8129ad']
net_q.clear()
# for nwid in nets:
# net_q.append(nwid)
net_id_handler('fpn_id0', 'b6079f73c63cea29')
net_id_handler('fpn_id1', 'b6079f73ca8129ad')
for nwid in nets:
net_id_handler(None, nwid, old=True)
prev_state = AttrDict.from_nested_dict(st.defState)
next_state = AttrDict.from_nested_dict(st.defState)
assert not st.changes
get_state_values(prev_state, next_state)
run_event_handlers(st.changes)
assert len(st.changes) == 0
next_state.update(fpn0=True,
fpn1=True,
fpn_id0='b6079f73c63cea29',
fpn_id1='b6079f73ca8129ad')
get_state_values(prev_state, next_state)
run_event_handlers(st.changes)
log_fpn_state(st.changes)
assert len(st.changes) == 4
network_cruft_cleaner()
def test_get_ztnwid():
from node_tools import state_data as stest
nodeState = AttrDict.from_nested_dict(stest.fpnState)
res = get_ztnwid('fpn0', 'fpn_id0', nodeState)
assert res == 'b6079f73c63cea29'
res = get_ztnwid('fpn1', 'fpn_id1', nodeState)
assert res == 'b6079f73ca8129ad'
nodeState['fpn1'] = None
nodeState['fpn_id1'] = None
res = get_ztnwid('fpn1', 'fpn_id1', nodeState)
assert res is None
def test_get_state():
from node_tools import state_data as stest
get_state(cache)
nodeState = AttrDict.from_nested_dict(stest.fpnState)
assert isinstance(nodeState, dict)
assert nodeState.online
assert nodeState.fpn_id == 'beefea68e6'
assert not nodeState.fallback
assert nodeState.fpn0
assert nodeState.fpn1
assert nodeState.moon_id0 == 'deadd738e6'
assert nodeState['fpn_id0'] == 'b6079f73c63cea29'
assert nodeState['fpn_id1'] == 'b6079f73ca8129ad'
def create_cache_entry(cache, data, key_str):
"""
Load new cache entry by key type.
:param cache: Index <cache> object
:param data: payload data in a dictionary
:param key_str: desired 'key_str', one of
['node'|'peer'|'net'|'mbr'|'moon'] or
['nstate'|'mstate'|'istate']
"""
new_data = AttrDict.from_nested_dict(data)
logger.debug('Pushing entry for: {}'.format(key_str))
with cache.transact():
key = cache.push(new_data, prefix=key_str)
logger.debug('New key created for: {}'.format(key))
def test_send_echo_no_responder(self):
nodeState = AttrDict.from_nested_dict(self.state)
fpn_id = nodeState.fpn_id
# expected command result is a list so the return
# result for echo_client() is actually None
mock_job = make_mock_job()
tj = every().second.do(mock_job)
send_announce_msg(fpn_id, None)
schedule.run_all()
with self.assertWarns(RuntimeWarning) as err:
result = echo_client(fpn_id, self.addr)
# print(result)
self.assertIs(result, None)
def test_send_cfg_no_responder(self):
nodeState = AttrDict.from_nested_dict(self.state)
fpn_id = nodeState.fpn_id
# expected command result is a list so the return
# result for echo_client() is actually None
mock_job = make_mock_job()
tj = every().second.do(mock_job)
send_announce_msg(fpn_id, None, send_cfg=True)
schedule.run_all()
with self.assertWarns(RuntimeWarning) as err:
result = echo_client(fpn_id, self.addr, send_cfg=True)
# print(err.warning)
self.assertEqual('Connection timed out', '{}'.format(err.warning))
def node_state_check(deorbit=False):
"""
Post-startup state check for moon data and msg_ref so we can deorbit.
:return: None or cmd result
"""
from node_tools import state_data as st
from node_tools.helper_funcs import AttrDict
result = None
state = AttrDict.from_nested_dict(st.fpnState)
if deorbit:
if state.moon_id0 and state.msg_ref:
result = run_moon_cmd(state.moon_id0, action='deorbit')
return result
def update_cache_entry(cache, data, key):
"""
Update single cache entry by key.
:param cache: <cache> object
:param data: payload data in a dictionary
:param key_str: desired 'key_str', one of
['node'|'peer'|'net'|'mbr'|'moon'] or
['nstate'|'mstate'|'istate']
"""
new_data = AttrDict.from_nested_dict(data)
if 'state' in key.rstrip('-0123456789'):
tgt = 'identity'
elif key.rstrip('-0123456789') in ('net', 'moon'):
tgt = 'id'
else:
tgt = 'address'
data_id = new_data[tgt]
logger.debug('New data has id: {}'.format(data_id))
logger.debug('Updating cache entry for key: {}'.format(key))
with cache.transact():
cache[key] = new_data
async def main():
"""State cache updater to retrieve data from a local ZeroTier node."""
async with aiohttp.ClientSession() as session:
ZT_API = get_token()
client = ZeroTier(ZT_API, loop, session)
nsState = AttrDict.from_nested_dict(st.fpnState)
net_wait = st.wait_cache
try:
# get status details of the local node and update state
await client.get_data('status')
node_id = handle_node_status(client.data, cache)
if NODE_SETTINGS['mode'] == 'peer':
# get status details of the node peers
await client.get_data('peer')
peer_data = client.data
logger.info('Found {} peers'.format(len(peer_data)))
peer_keys = find_keys(cache, 'peer')
logger.debug('Returned peer keys: {}'.format(peer_keys))
load_cache_by_type(cache, peer_data, 'peer')
# check for moon data (only exists for moons we orbit)
if not nsState.moon_id0:
moon_data = run_ztcli_cmd(action='listmoons')
if moon_data:
load_cache_by_type(cache, moon_data, 'moon')
moonStatus = []
fpn_moons = NODE_SETTINGS['moon_list']
peerStatus = get_peer_status(cache)
for peer in peerStatus:
if peer['role'] == 'MOON' and peer['identity'] in fpn_moons:
moonStatus.append(peer)
break
logger.debug('Got moon state: {}'.format(moonStatus))
load_cache_by_type(cache, moonStatus, 'mstate')
# get all available network data
await client.get_data('network')
net_data = client.data
logger.info('Found {} networks'.format(len(net_data)))
if NODE_SETTINGS['mode'] == 'peer':
wait_for_nets = net_wait.get('offline_wait')
if len(net_data) == 0 and not nsState.cfg_ref:
send_cfg_handler()
put_state_msg('WAITING')
elif len(net_data) == 0 and nsState.cfg_ref and not wait_for_nets:
put_state_msg('ERROR')
net_keys = find_keys(cache, 'net')
logger.debug('Returned network keys: {}'.format(net_keys))
load_cache_by_type(cache, net_data, 'net')
netStatus = get_net_status(cache)
logger.debug('Got net state: {}'.format(netStatus))
load_cache_by_type(cache, netStatus, 'istate')
if NODE_SETTINGS['mode'] == 'peer':
# check for reconfiguration events
for net in netStatus:
if net['status'] == 'NOT_FOUND' or net['status'] == 'ACCESS_DENIED':
# if net['ztaddress'] != net['gateway']:
# do_net_cmd(get_net_cmds(NODE_SETTINGS['home_dir'], 'fpn0'))
run_ztcli_cmd(action='leave', extra=net['identity'])
net_id_handler(None, net['identity'], old=True)
st.fpnState['cfg_ref'] = None
net_wait.set('offline_wait', True, 75)
if len(net_data) < 2 and not nsState.cfg_ref:
send_cfg_handler()
put_state_msg('WAITING')
# check the state of exit network/route
exit_id = get_ztnwid('fpn0', 'fpn_id0', nsState)
if exit_id is not None:
for net in netStatus:
if net['identity'] == exit_id:
ztaddr = net['ztaddress']
break
exit_state, _, _ = do_peer_check(ztaddr)
logger.debug('HEALTH: peer state is {}'.format(exit_state))
wait_for_nets = net_wait.get('offline_wait')
logger.debug('HEALTH: network route state is {}'.format(nsState.route))
if nsState.route is False:
if not st.fpnState['wdg_ref'] and not wait_for_nets:
# logger.error('HEALTH: net_health state is {}'.format(nsState.route))
reply = send_wedged_msg()
if 'result' in reply[0]:
st.fpnState['wdg_ref'] = True
logger.error('HEALTH: network is unreachable!!')
put_state_msg('ERROR')
else:
logger.debug('HEALTH: wait_for_nets is {}'.format(wait_for_nets))
elif NODE_SETTINGS['mode'] == 'adhoc':
if not NODE_SETTINGS['nwid']:
logger.warning('ADHOC: network ID not set {}'.format(NODE_SETTINGS['nwid']))
else:
logger.debug('ADHOC: found network ID {}'.format(NODE_SETTINGS['nwid']))
if netStatus != []:
nwid = netStatus[0]['identity']
addr = netStatus[0]['ztaddress']
nwstat = netStatus[0]['status']
logger.debug('ADHOC: found network with ID {}'.format(nwid))
logger.debug('ADHOC: network status is {}'.format(nwstat))
if addr:
res = do_peer_check(addr)
# elif NODE_SETTINGS['nwid']:
# run_ztcli_cmd(action='join', extra=NODE_SETTINGS['nwid'])
except Exception as exc:
logger.error('nodestate exception was: {}'.format(exc))
raise exc
def setUp(self):
super(StateChangeTest, self).setUp()
from node_tools import state_data as st
self.saved_state = AttrDict.from_nested_dict(st.defState)
self.state = self.saved_state
