def state_check(*args, **kwargs):
"""
Diskcache wrapper for checking nodeState before and after the
update_runner() tries to grab new data.
"""
from node_tools import state_data as st
get_state(cache)
prev_state = AttrDict.from_nested_dict(st.fpnState)
if not prev_state.online:
logger.warning('nodeState not initialized (node not online)')
else:
logger.info('Node online with id: {}'.format(prev_state.fpn_id))
result = func(*args, **kwargs)
get_state(cache)
next_state = AttrDict.from_nested_dict(st.fpnState)
if not next_state.online and not prev_state.online:
logger.warning('nodeState still not initialized (node not online)')
elif next_state.online and prev_state.online:
get_state_values(prev_state, next_state)
if st.changes:
logger.info('NETSTATE: diff is {}'.format(st.changes))
put_state_msg('CONFIG')
if next_state.fallback:
logger.error('NETSTATE: fallback mode is True (network suspect)')
return result
def test_put_state_msg_save():
"""
Same as test_put_state_msg but without cleaning
"""
state_path = Path(get_runtimedir(user_dirs=True)).joinpath('fpnd.state')
msgs = [
'STARTING', 'CONNECTED', 'CONFIG', 'ERROR', 'UPGRADE', 'WAITING',
'NONE'
]
for msg in msgs:
put_state_msg(msg, clean=False)
status_queue = get_status(str(state_path))
assert len(list(status_queue)) == 1
data = status_queue.pop().strip()
assert data == msg
assert len(list(status_queue)) == 0
for msg in msgs:
put_state_msg(msg, clean=False)
status_queue = get_status(str(state_path), 7)
assert len(list(status_queue)) == 7
data = status_queue.pop().strip()
assert data == 'NONE'
assert len(list(status_queue)) == 6
state_path.unlink()
def test_put_state_msg():
"""
Use get_status() from freepn-gtk3-indicator (replicated above)
"""
state_path = Path(get_runtimedir(user_dirs=True)).joinpath('fpnd.state')
msgs = [
'STARTING', 'CONNECTED', 'CONFIG', 'ERROR', 'UPGRADE', 'WAITING',
'NONE'
]
for msg in msgs:
put_state_msg(msg)
status_queue = get_status(str(state_path))
assert len(list(status_queue)) == 1
data = status_queue.pop().strip()
assert data == msg
assert len(list(status_queue)) == 0
for msg in msgs:
put_state_msg(msg)
status_queue = get_status(str(state_path), 7)
assert len(list(status_queue)) == 1
data = status_queue.pop().strip()
assert data == 'NONE'
assert len(list(status_queue)) == 0
state_path.unlink()
def echo_client(fpn_id, addr, send_cfg=False):
import json
from node_tools import state_data as st
from node_tools.msg_queues import make_version_msg
from node_tools.node_funcs import do_shutdown
from node_tools.node_funcs import node_state_check
from node_tools.node_funcs import run_ztcli_cmd
if NODE_SETTINGS['use_localhost'] or not addr:
addr = '127.0.0.1'
cfg = st.cfg_msgs
compatible = True
node_data = st.fpnState
reply_list = []
reciept = False
try:
if send_cfg:
reply_list = send_req_msg(addr, 'node_cfg', fpn_id)
logger.debug('CFG: send_cfg reply is {}'.format(reply_list))
if 'result' not in reply_list[0]:
logger.warning('CFG: malformed reply {}'.format(reply_list))
else:
node_data['cfg_ref'] = reply_list[0]['ref']
cfg = json.loads(reply_list[0]['result'])
logger.debug('CFG: state has payload {}'.format(cfg))
for net in cfg['networks']:
res = run_ztcli_cmd(action='join', extra=net)
logger.debug('run_ztcli_cmd join result: {}'.format(res))
else:
ver_msg = make_version_msg(fpn_id)
reply_list = send_req_msg(addr, 'echo', ver_msg)
logger.debug('ECHO: ver_msg reply is {}'.format(reply_list))
if 'result' not in reply_list[0]:
logger.warning('ECHO: malformed reply {}'.format(reply_list))
else:
node_data['msg_ref'] = reply_list[0]['ref']
msg = json.loads(reply_list[0]['result'])
logger.debug('ECHO: got msg reply {}'.format(msg))
if 'UPGRADE' in msg['version']:
put_state_msg('UPGRADE')
compatible = False
reciept = True
logger.debug('Send result is {}'.format(reply_list))
if not compatible:
logger.error(
'Shutting down due to incompatible version: {}'.format(
ver_msg))
do_shutdown()
if not send_cfg and not node_data['cfg_ref']:
res = node_state_check(deorbit=True)
logger.debug('node_state_check returned {}'.format(res))
except Exception as exc:
# success wrapper needs a warning to catch
logger.warning('Send error is {}'.format(exc))
raise exc
return reply_list, reciept
def do_net_check(path=None):
"""
Try and get the geoip location using fpn route.
:param path: path to script dir
"""
import os
if not path:
path = NODE_SETTINGS['home_dir']
cmd_file = os.path.join(path, 'show-geoip.sh')
cmd = [cmd_file]
doh_host = NODE_SETTINGS['doh_host']
max_wait = NODE_SETTINGS['max_timeout']
if doh_host is not None:
cmd = [cmd_file, doh_host]
logger.debug('ENV: geoip script using doh_host: {}'.format(doh_host))
result = do_net_cmd(cmd)
state, res, retcode = result
fpn_data = st.fpnState
net_wait = st.wait_cache
if not state:
host_state, _, _ = do_host_check()
if net_wait.get('fpn0_UP'):
fpn_data['route'] = None
elif fpn_data['fpn0'] and fpn_data['fpn1'] and retcode == 4:
if fpn_data['route'] is True:
fpn_data['route'] = None
net_wait.set('failed_once', True, max_wait)
else:
if host_state:
fpn_data['route'] = False
elif not net_wait.get('failed_once') and not fpn_data['route']:
fpn_data['route'] = False
logger.error('HEALTH: network route state is {}'.format(
fpn_data['route']))
logger.error('HEALTH: host route state is {}'.format(host_state))
logger.debug('HEALTH: net_wait is {}'.format(
net_wait.get('failed_once')))
else:
logger.error('do_net_check {} returned: {}'.format(cmd, result))
else:
if fpn_data['fpn0'] and fpn_data['fpn1']:
if retcode == 0:
fpn_data['route'] = True
fpn_data['wdg_ref'] = None
put_state_msg('CONNECTED')
logger.info('HEALTH: network route state is {}'.format(
fpn_data['route']))
if fpn_data['route'] is None:
logger.info('HEALTH: no state yet (state is {})'.format(
fpn_data['route']))
return result
def do_cleanup(path=None, addr=None):
"""
Run network cleanup commands via daemon cleanup hook.
:param path: path to scripts dir
:param addr: moon address if known
"""
from node_tools.helper_funcs import AttrDict
from node_tools.network_funcs import do_net_cmd
from node_tools.network_funcs import get_net_cmds
from node_tools.helper_funcs import put_state_msg
from node_tools.network_funcs import send_req_msg
from node_tools import state_data as st
if NODE_SETTINGS['node_role'] == 'moon':
for script in ['msg_responder.py', 'msg_subscriber.py']:
res = control_daemon('stop', script)
logger.info('CLEANUP: shutting down {}'.format(script))
elif NODE_SETTINGS['node_role'] == 'controller':
for script in ['msg_subscriber.py']:
res = control_daemon('stop', script)
logger.info('CLEANUP: shutting down {}'.format(script))
else:
state = AttrDict.from_nested_dict(st.fpnState)
moon_addr = addr
moon_id = state.moon_id0
node_id = state.fpn_id
if not addr:
moon_addr = state.moon_addr
if not NODE_SETTINGS['use_localhost'] and not addr:
addr = moon_addr
if not path:
path = NODE_SETTINGS['home_dir']
nets = ['fpn_id0', 'fpn_id1']
ifaces = ['fpn0', 'fpn1']
put_state_msg('NONE')
for iface, net in zip(ifaces, nets):
if state[iface]:
logger.info('CLEANUP: shutting down {}'.format(iface))
cmd = get_net_cmds(path, iface)
res = do_net_cmd(cmd)
logger.debug('CLEANUP: {} shutdown returned {}'.format(
iface, res))
# logger.info('CLEANUP: leaving network ID: {}'.format(state[net]))
res = run_ztcli_cmd(action='leave', extra=state[net])
logger.debug('CLEANUP: action leave returned: {}'.format(res))
if moon_id is not None:
run_moon_cmd(moon_id, action='deorbit')
reply = send_req_msg(addr, 'offline', node_id)
logger.debug('CLEANUP: offline reply: {}'.format(reply))
async def main():
"""State cache updater to retrieve data from a local ZeroTier node."""
async with aiohttp.ClientSession() as session:
ZT_API = get_token()
client = ZeroTier(ZT_API, loop, session)
nsState = AttrDict.from_nested_dict(st.fpnState)
net_wait = st.wait_cache
try:
# get status details of the local node and update state
await client.get_data('status')
node_id = handle_node_status(client.data, cache)
if NODE_SETTINGS['mode'] == 'peer':
# get status details of the node peers
await client.get_data('peer')
peer_data = client.data
logger.info('Found {} peers'.format(len(peer_data)))
peer_keys = find_keys(cache, 'peer')
logger.debug('Returned peer keys: {}'.format(peer_keys))
load_cache_by_type(cache, peer_data, 'peer')
# check for moon data (only exists for moons we orbit)
if not nsState.moon_id0:
moon_data = run_ztcli_cmd(action='listmoons')
if moon_data:
load_cache_by_type(cache, moon_data, 'moon')
moonStatus = []
fpn_moons = NODE_SETTINGS['moon_list']
peerStatus = get_peer_status(cache)
for peer in peerStatus:
if peer['role'] == 'MOON' and peer['identity'] in fpn_moons:
moonStatus.append(peer)
break
logger.debug('Got moon state: {}'.format(moonStatus))
load_cache_by_type(cache, moonStatus, 'mstate')
# get all available network data
await client.get_data('network')
net_data = client.data
logger.info('Found {} networks'.format(len(net_data)))
if NODE_SETTINGS['mode'] == 'peer':
wait_for_nets = net_wait.get('offline_wait')
if len(net_data) == 0 and not nsState.cfg_ref:
send_cfg_handler()
put_state_msg('WAITING')
elif len(net_data) == 0 and nsState.cfg_ref and not wait_for_nets:
put_state_msg('ERROR')
net_keys = find_keys(cache, 'net')
logger.debug('Returned network keys: {}'.format(net_keys))
load_cache_by_type(cache, net_data, 'net')
netStatus = get_net_status(cache)
logger.debug('Got net state: {}'.format(netStatus))
load_cache_by_type(cache, netStatus, 'istate')
if NODE_SETTINGS['mode'] == 'peer':
# check for reconfiguration events
for net in netStatus:
if net['status'] == 'NOT_FOUND' or net['status'] == 'ACCESS_DENIED':
# if net['ztaddress'] != net['gateway']:
# do_net_cmd(get_net_cmds(NODE_SETTINGS['home_dir'], 'fpn0'))
run_ztcli_cmd(action='leave', extra=net['identity'])
net_id_handler(None, net['identity'], old=True)
st.fpnState['cfg_ref'] = None
net_wait.set('offline_wait', True, 75)
if len(net_data) < 2 and not nsState.cfg_ref:
send_cfg_handler()
put_state_msg('WAITING')
# check the state of exit network/route
exit_id = get_ztnwid('fpn0', 'fpn_id0', nsState)
if exit_id is not None:
for net in netStatus:
if net['identity'] == exit_id:
ztaddr = net['ztaddress']
break
exit_state, _, _ = do_peer_check(ztaddr)
logger.debug('HEALTH: peer state is {}'.format(exit_state))
wait_for_nets = net_wait.get('offline_wait')
logger.debug('HEALTH: network route state is {}'.format(nsState.route))
if nsState.route is False:
if not st.fpnState['wdg_ref'] and not wait_for_nets:
# logger.error('HEALTH: net_health state is {}'.format(nsState.route))
reply = send_wedged_msg()
if 'result' in reply[0]:
st.fpnState['wdg_ref'] = True
logger.error('HEALTH: network is unreachable!!')
put_state_msg('ERROR')
else:
logger.debug('HEALTH: wait_for_nets is {}'.format(wait_for_nets))
elif NODE_SETTINGS['mode'] == 'adhoc':
if not NODE_SETTINGS['nwid']:
logger.warning('ADHOC: network ID not set {}'.format(NODE_SETTINGS['nwid']))
else:
logger.debug('ADHOC: found network ID {}'.format(NODE_SETTINGS['nwid']))
if netStatus != []:
nwid = netStatus[0]['identity']
addr = netStatus[0]['ztaddress']
nwstat = netStatus[0]['status']
logger.debug('ADHOC: found network with ID {}'.format(nwid))
logger.debug('ADHOC: network status is {}'.format(nwstat))
if addr:
res = do_peer_check(addr)
# elif NODE_SETTINGS['nwid']:
# run_ztcli_cmd(action='join', extra=NODE_SETTINGS['nwid'])
except Exception as exc:
logger.error('nodestate exception was: {}'.format(exc))
raise exc
def do_scheduling():
set_initial_role()
network_cruft_cleaner()
schedule.run_all(1, 'base-tasks')
validate_role()
node_role = NODE_SETTINGS['node_role']
mode = NODE_SETTINGS['mode']
if node_role is None and mode == 'peer':
NODE_SETTINGS['use_localhost'] = True
if mode == 'peer':
if node_role is None:
check_time = 33
baseCheckJob = schedule.every(check_time).seconds
baseCheckJob.do(run_net_check).tag('base-tasks', 'route-status')
try:
data = wait_for_moon(timeout=45)
except Exception as exc:
logger.error('ENODATA exception {}'.format(exc))
put_state_msg('ERROR')
try:
handle_moon_data(data)
put_state_msg('STARTING')
except MemberNodeError as exc:
logger.error('ENODATA exception {}'.format(exc))
put_state_msg('ERROR')
str_level = logging.getLevelName(logger.getEffectiveLevel())
logger.debug('Current log level is: {}'.format(str_level))
startup_handlers()
else:
if node_role == 'controller':
netobj_q = dc.Deque(directory=get_cachedir('netobj_queue'))
gen_netobj_queue(netobj_q)
cache = dc.Index(get_cachedir())
for key_str in ['peer', 'moon', 'mstate']:
delete_cache_entry(cache, key_str)
elif node_role == 'moon':
cln_q = dc.Deque(directory=get_cachedir('clean_queue'))
pub_q = dc.Deque(directory=get_cachedir('pub_queue'))
schedule.every(37).seconds.do(run_cleanup_check, cln_q,
pub_q).tag(
'chk-tasks', 'cleanup')
schedule.every(15).minutes.do(check_daemon_status).tag(
'chk-tasks', 'responder')
schedule.every(15).minutes.do(check_daemon_status,
script='msg_subscriber.py').tag(
'chk-tasks', 'subscriber')
schedule.run_all(1, 'chk-tasks')
elif mode == 'adhoc':
logger.debug('Running in adhoc mode...')
if NODE_SETTINGS['nwid']:
logger.debug('ADHOC: found network {}'.format(
NODE_SETTINGS['nwid']))
do_startup(NODE_SETTINGS['nwid'])
else:
logger.error('No network ID found in NODE_SETTINGS!!')
logger.error('Have you created a network yet?')
logger.debug('MODE: startup mode is {} and role is {}'.format(
mode, node_role))
logger.info(
'You are running fpnd/node_tools version {}'.format(fpnd_version))
while True:
schedule.run_pending()
time.sleep(1)