def state_check(*args, **kwargs): """ Diskcache wrapper for checking nodeState before and after the update_runner() tries to grab new data. """ from node_tools import state_data as st get_state(cache) prev_state = AttrDict.from_nested_dict(st.fpnState) if not prev_state.online: logger.warning('nodeState not initialized (node not online)') else: logger.info('Node online with id: {}'.format(prev_state.fpn_id)) result = func(*args, **kwargs) get_state(cache) next_state = AttrDict.from_nested_dict(st.fpnState) if not next_state.online and not prev_state.online: logger.warning('nodeState still not initialized (node not online)') elif next_state.online and prev_state.online: get_state_values(prev_state, next_state) if st.changes: logger.info('NETSTATE: diff is {}'.format(st.changes)) put_state_msg('CONFIG') if next_state.fallback: logger.error('NETSTATE: fallback mode is True (network suspect)') return result
def test_put_state_msg_save(): """ Same as test_put_state_msg but without cleaning """ state_path = Path(get_runtimedir(user_dirs=True)).joinpath('fpnd.state') msgs = [ 'STARTING', 'CONNECTED', 'CONFIG', 'ERROR', 'UPGRADE', 'WAITING', 'NONE' ] for msg in msgs: put_state_msg(msg, clean=False) status_queue = get_status(str(state_path)) assert len(list(status_queue)) == 1 data = status_queue.pop().strip() assert data == msg assert len(list(status_queue)) == 0 for msg in msgs: put_state_msg(msg, clean=False) status_queue = get_status(str(state_path), 7) assert len(list(status_queue)) == 7 data = status_queue.pop().strip() assert data == 'NONE' assert len(list(status_queue)) == 6 state_path.unlink()
def test_put_state_msg(): """ Use get_status() from freepn-gtk3-indicator (replicated above) """ state_path = Path(get_runtimedir(user_dirs=True)).joinpath('fpnd.state') msgs = [ 'STARTING', 'CONNECTED', 'CONFIG', 'ERROR', 'UPGRADE', 'WAITING', 'NONE' ] for msg in msgs: put_state_msg(msg) status_queue = get_status(str(state_path)) assert len(list(status_queue)) == 1 data = status_queue.pop().strip() assert data == msg assert len(list(status_queue)) == 0 for msg in msgs: put_state_msg(msg) status_queue = get_status(str(state_path), 7) assert len(list(status_queue)) == 1 data = status_queue.pop().strip() assert data == 'NONE' assert len(list(status_queue)) == 0 state_path.unlink()
def echo_client(fpn_id, addr, send_cfg=False): import json from node_tools import state_data as st from node_tools.msg_queues import make_version_msg from node_tools.node_funcs import do_shutdown from node_tools.node_funcs import node_state_check from node_tools.node_funcs import run_ztcli_cmd if NODE_SETTINGS['use_localhost'] or not addr: addr = '127.0.0.1' cfg = st.cfg_msgs compatible = True node_data = st.fpnState reply_list = [] reciept = False try: if send_cfg: reply_list = send_req_msg(addr, 'node_cfg', fpn_id) logger.debug('CFG: send_cfg reply is {}'.format(reply_list)) if 'result' not in reply_list[0]: logger.warning('CFG: malformed reply {}'.format(reply_list)) else: node_data['cfg_ref'] = reply_list[0]['ref'] cfg = json.loads(reply_list[0]['result']) logger.debug('CFG: state has payload {}'.format(cfg)) for net in cfg['networks']: res = run_ztcli_cmd(action='join', extra=net) logger.debug('run_ztcli_cmd join result: {}'.format(res)) else: ver_msg = make_version_msg(fpn_id) reply_list = send_req_msg(addr, 'echo', ver_msg) logger.debug('ECHO: ver_msg reply is {}'.format(reply_list)) if 'result' not in reply_list[0]: logger.warning('ECHO: malformed reply {}'.format(reply_list)) else: node_data['msg_ref'] = reply_list[0]['ref'] msg = json.loads(reply_list[0]['result']) logger.debug('ECHO: got msg reply {}'.format(msg)) if 'UPGRADE' in msg['version']: put_state_msg('UPGRADE') compatible = False reciept = True logger.debug('Send result is {}'.format(reply_list)) if not compatible: logger.error( 'Shutting down due to incompatible version: {}'.format( ver_msg)) do_shutdown() if not send_cfg and not node_data['cfg_ref']: res = node_state_check(deorbit=True) logger.debug('node_state_check returned {}'.format(res)) except Exception as exc: # success wrapper needs a warning to catch logger.warning('Send error is {}'.format(exc)) raise exc return reply_list, reciept
def do_net_check(path=None): """ Try and get the geoip location using fpn route. :param path: path to script dir """ import os if not path: path = NODE_SETTINGS['home_dir'] cmd_file = os.path.join(path, 'show-geoip.sh') cmd = [cmd_file] doh_host = NODE_SETTINGS['doh_host'] max_wait = NODE_SETTINGS['max_timeout'] if doh_host is not None: cmd = [cmd_file, doh_host] logger.debug('ENV: geoip script using doh_host: {}'.format(doh_host)) result = do_net_cmd(cmd) state, res, retcode = result fpn_data = st.fpnState net_wait = st.wait_cache if not state: host_state, _, _ = do_host_check() if net_wait.get('fpn0_UP'): fpn_data['route'] = None elif fpn_data['fpn0'] and fpn_data['fpn1'] and retcode == 4: if fpn_data['route'] is True: fpn_data['route'] = None net_wait.set('failed_once', True, max_wait) else: if host_state: fpn_data['route'] = False elif not net_wait.get('failed_once') and not fpn_data['route']: fpn_data['route'] = False logger.error('HEALTH: network route state is {}'.format( fpn_data['route'])) logger.error('HEALTH: host route state is {}'.format(host_state)) logger.debug('HEALTH: net_wait is {}'.format( net_wait.get('failed_once'))) else: logger.error('do_net_check {} returned: {}'.format(cmd, result)) else: if fpn_data['fpn0'] and fpn_data['fpn1']: if retcode == 0: fpn_data['route'] = True fpn_data['wdg_ref'] = None put_state_msg('CONNECTED') logger.info('HEALTH: network route state is {}'.format( fpn_data['route'])) if fpn_data['route'] is None: logger.info('HEALTH: no state yet (state is {})'.format( fpn_data['route'])) return result
def do_cleanup(path=None, addr=None): """ Run network cleanup commands via daemon cleanup hook. :param path: path to scripts dir :param addr: moon address if known """ from node_tools.helper_funcs import AttrDict from node_tools.network_funcs import do_net_cmd from node_tools.network_funcs import get_net_cmds from node_tools.helper_funcs import put_state_msg from node_tools.network_funcs import send_req_msg from node_tools import state_data as st if NODE_SETTINGS['node_role'] == 'moon': for script in ['msg_responder.py', 'msg_subscriber.py']: res = control_daemon('stop', script) logger.info('CLEANUP: shutting down {}'.format(script)) elif NODE_SETTINGS['node_role'] == 'controller': for script in ['msg_subscriber.py']: res = control_daemon('stop', script) logger.info('CLEANUP: shutting down {}'.format(script)) else: state = AttrDict.from_nested_dict(st.fpnState) moon_addr = addr moon_id = state.moon_id0 node_id = state.fpn_id if not addr: moon_addr = state.moon_addr if not NODE_SETTINGS['use_localhost'] and not addr: addr = moon_addr if not path: path = NODE_SETTINGS['home_dir'] nets = ['fpn_id0', 'fpn_id1'] ifaces = ['fpn0', 'fpn1'] put_state_msg('NONE') for iface, net in zip(ifaces, nets): if state[iface]: logger.info('CLEANUP: shutting down {}'.format(iface)) cmd = get_net_cmds(path, iface) res = do_net_cmd(cmd) logger.debug('CLEANUP: {} shutdown returned {}'.format( iface, res)) # logger.info('CLEANUP: leaving network ID: {}'.format(state[net])) res = run_ztcli_cmd(action='leave', extra=state[net]) logger.debug('CLEANUP: action leave returned: {}'.format(res)) if moon_id is not None: run_moon_cmd(moon_id, action='deorbit') reply = send_req_msg(addr, 'offline', node_id) logger.debug('CLEANUP: offline reply: {}'.format(reply))
async def main(): """State cache updater to retrieve data from a local ZeroTier node.""" async with aiohttp.ClientSession() as session: ZT_API = get_token() client = ZeroTier(ZT_API, loop, session) nsState = AttrDict.from_nested_dict(st.fpnState) net_wait = st.wait_cache try: # get status details of the local node and update state await client.get_data('status') node_id = handle_node_status(client.data, cache) if NODE_SETTINGS['mode'] == 'peer': # get status details of the node peers await client.get_data('peer') peer_data = client.data logger.info('Found {} peers'.format(len(peer_data))) peer_keys = find_keys(cache, 'peer') logger.debug('Returned peer keys: {}'.format(peer_keys)) load_cache_by_type(cache, peer_data, 'peer') # check for moon data (only exists for moons we orbit) if not nsState.moon_id0: moon_data = run_ztcli_cmd(action='listmoons') if moon_data: load_cache_by_type(cache, moon_data, 'moon') moonStatus = [] fpn_moons = NODE_SETTINGS['moon_list'] peerStatus = get_peer_status(cache) for peer in peerStatus: if peer['role'] == 'MOON' and peer['identity'] in fpn_moons: moonStatus.append(peer) break logger.debug('Got moon state: {}'.format(moonStatus)) load_cache_by_type(cache, moonStatus, 'mstate') # get all available network data await client.get_data('network') net_data = client.data logger.info('Found {} networks'.format(len(net_data))) if NODE_SETTINGS['mode'] == 'peer': wait_for_nets = net_wait.get('offline_wait') if len(net_data) == 0 and not nsState.cfg_ref: send_cfg_handler() put_state_msg('WAITING') elif len(net_data) == 0 and nsState.cfg_ref and not wait_for_nets: put_state_msg('ERROR') net_keys = find_keys(cache, 'net') logger.debug('Returned network keys: {}'.format(net_keys)) load_cache_by_type(cache, net_data, 'net') netStatus = get_net_status(cache) logger.debug('Got net state: {}'.format(netStatus)) load_cache_by_type(cache, netStatus, 'istate') if NODE_SETTINGS['mode'] == 'peer': # check for reconfiguration events for net in netStatus: if net['status'] == 'NOT_FOUND' or net['status'] == 'ACCESS_DENIED': # if net['ztaddress'] != net['gateway']: # do_net_cmd(get_net_cmds(NODE_SETTINGS['home_dir'], 'fpn0')) run_ztcli_cmd(action='leave', extra=net['identity']) net_id_handler(None, net['identity'], old=True) st.fpnState['cfg_ref'] = None net_wait.set('offline_wait', True, 75) if len(net_data) < 2 and not nsState.cfg_ref: send_cfg_handler() put_state_msg('WAITING') # check the state of exit network/route exit_id = get_ztnwid('fpn0', 'fpn_id0', nsState) if exit_id is not None: for net in netStatus: if net['identity'] == exit_id: ztaddr = net['ztaddress'] break exit_state, _, _ = do_peer_check(ztaddr) logger.debug('HEALTH: peer state is {}'.format(exit_state)) wait_for_nets = net_wait.get('offline_wait') logger.debug('HEALTH: network route state is {}'.format(nsState.route)) if nsState.route is False: if not st.fpnState['wdg_ref'] and not wait_for_nets: # logger.error('HEALTH: net_health state is {}'.format(nsState.route)) reply = send_wedged_msg() if 'result' in reply[0]: st.fpnState['wdg_ref'] = True logger.error('HEALTH: network is unreachable!!') put_state_msg('ERROR') else: logger.debug('HEALTH: wait_for_nets is {}'.format(wait_for_nets)) elif NODE_SETTINGS['mode'] == 'adhoc': if not NODE_SETTINGS['nwid']: logger.warning('ADHOC: network ID not set {}'.format(NODE_SETTINGS['nwid'])) else: logger.debug('ADHOC: found network ID {}'.format(NODE_SETTINGS['nwid'])) if netStatus != []: nwid = netStatus[0]['identity'] addr = netStatus[0]['ztaddress'] nwstat = netStatus[0]['status'] logger.debug('ADHOC: found network with ID {}'.format(nwid)) logger.debug('ADHOC: network status is {}'.format(nwstat)) if addr: res = do_peer_check(addr) # elif NODE_SETTINGS['nwid']: # run_ztcli_cmd(action='join', extra=NODE_SETTINGS['nwid']) except Exception as exc: logger.error('nodestate exception was: {}'.format(exc)) raise exc
def do_scheduling(): set_initial_role() network_cruft_cleaner() schedule.run_all(1, 'base-tasks') validate_role() node_role = NODE_SETTINGS['node_role'] mode = NODE_SETTINGS['mode'] if node_role is None and mode == 'peer': NODE_SETTINGS['use_localhost'] = True if mode == 'peer': if node_role is None: check_time = 33 baseCheckJob = schedule.every(check_time).seconds baseCheckJob.do(run_net_check).tag('base-tasks', 'route-status') try: data = wait_for_moon(timeout=45) except Exception as exc: logger.error('ENODATA exception {}'.format(exc)) put_state_msg('ERROR') try: handle_moon_data(data) put_state_msg('STARTING') except MemberNodeError as exc: logger.error('ENODATA exception {}'.format(exc)) put_state_msg('ERROR') str_level = logging.getLevelName(logger.getEffectiveLevel()) logger.debug('Current log level is: {}'.format(str_level)) startup_handlers() else: if node_role == 'controller': netobj_q = dc.Deque(directory=get_cachedir('netobj_queue')) gen_netobj_queue(netobj_q) cache = dc.Index(get_cachedir()) for key_str in ['peer', 'moon', 'mstate']: delete_cache_entry(cache, key_str) elif node_role == 'moon': cln_q = dc.Deque(directory=get_cachedir('clean_queue')) pub_q = dc.Deque(directory=get_cachedir('pub_queue')) schedule.every(37).seconds.do(run_cleanup_check, cln_q, pub_q).tag( 'chk-tasks', 'cleanup') schedule.every(15).minutes.do(check_daemon_status).tag( 'chk-tasks', 'responder') schedule.every(15).minutes.do(check_daemon_status, script='msg_subscriber.py').tag( 'chk-tasks', 'subscriber') schedule.run_all(1, 'chk-tasks') elif mode == 'adhoc': logger.debug('Running in adhoc mode...') if NODE_SETTINGS['nwid']: logger.debug('ADHOC: found network {}'.format( NODE_SETTINGS['nwid'])) do_startup(NODE_SETTINGS['nwid']) else: logger.error('No network ID found in NODE_SETTINGS!!') logger.error('Have you created a network yet?') logger.debug('MODE: startup mode is {} and role is {}'.format( mode, node_role)) logger.info( 'You are running fpnd/node_tools version {}'.format(fpnd_version)) while True: schedule.run_pending() time.sleep(1)