def network_scan(self, this_ip_addr: str = None, netmask_bits: int = None, nmap_scan_opts: list = DEFAULT_NMAP_SCAN_OPTS, save_to_kvs: bool = True) -> dict: if not this_ip_addr: this_ip_addr = self.default_ip if not netmask_bits: netmask_bits = self.default_netmask_bits net_to_scan = this_ip_addr + '/' + str(netmask_bits) nmap_args = [net_to_scan] + nmap_scan_opts logger.info(f"Running nmap scan on {' '.join(nmap_args)}") scan_res = self.run_nmap(nmap_args) logger.debug(f"nmap scan result: {scan_res}") if scan_res is None: return hosts = [] for h in scan_res['nmaprun'].get('host', []): if h['status']['state'] == 'up': this_host = {} for a in h['address']: if a.get('addrtype') == 'ipv4': this_host['ipv4'] = a.get('addr') elif a.get('addrtype') == 'mac': this_host['mac'] = a.get('addr') if 'vendor' in a: this_host['vendor'] = a['vendor'] if h['hostnames']: this_host['hostname'] = h['hostnames']['hostname'][0].get( 'name') if 'ports' in h: this_host['ports'] = [] for p in h['ports'].get('port', []): if p['state']['state'] == 'open': this_host['ports'].append(p['portid']) hosts.append(this_host) # The default behavior is to save the results to the key-value store, # for potential use by other processes if save_to_kvs: try: kvs = KVStore() for h in hosts: if h.get('mac'): # Skip any hosts without MAC addresses this_mac = h['mac'].lower() kvs.set(f"env:net:mac:{this_mac}", h) except Exception as e: logger.error( f"Cannot save scan results to key-value store: {e}") return hosts
def main() -> None: kvs = KVStore() try: wifi_ap = WifiAPSnapCtl() kvs.set(KVS_AVAILABLE_KEY, True) except Exception as e: logger.warn( f"Exception while setting up Wifi access point control: {type(e).__name__}: {e}" ) kvs.set(KVS_AVAILABLE_KEY, False) sys.exit(1) initialize(wifi_ap, kvs) monitor_and_update(wifi_ap, kvs)
class Node(object): def __init__(self) -> None: self.kvs = KVStore() try: # Load base config from YAML file with open(os.path.join(os.getenv('SNAP', './'), 'remote.yaml'), 'r') as remote_yaml: remote = yaml.safe_load(remote_yaml) self.remote_api = remote['api'] self.data_endpoints = remote.get('data-endpoints', []) for dep in self.data_endpoints: if dep.get('isdefault') and dep.get('type') == 'mqtt': self._remote_mqtt_broker_config = dep['config'] else: logger.warning( f"No remote MQTT Broker found for endpoint: [{dep.get('name')}]" ) continue except Exception: logger.exception( 'Base configuration file remote.yaml cannot be loaded. Quitting' ) sys.exit( 'Base configuration file remote.yaml cannot be loaded. Quitting' ) # If additional provisioning remote.yaml is available, load it also try: with open( os.path.join(os.getenv('SNAP', './'), 'provisioning', 'remote.yaml'), 'r') as p_remote_yaml: remote = yaml.safe_load(p_remote_yaml) if isinstance(remote.get('data-endpoints'), list): self.data_endpoints.extend(remote['data-endpoints']) logger.info( f"Added {len(remote['data-endpoints'])} data endpoints from provisioning remote.yaml" ) else: logger.info( "No valid data-endpoints definition found in provisioning remote.yaml" ) except FileNotFoundError: logger.info("No provisioning remote.yaml found") except Exception: logger.exception( 'Exception while trying to process provisioning remote.yaml') try: self._dbconfig = NodeConfig.get() if self._dbconfig.node_id == 'd43639139e08': raise ValueError( 'Node ID indicates Moxa with hardcoded non-unique MAC. Needs re-initialization' ) except NodeConfig.DoesNotExist: logger.info( 'No node configuration found in internal database. Attempting node initialization' ) self.__initialize() except ValueError: logger.warning('ValueError in config.', exc_info=True) self.__initialize() self.node_id = self._dbconfig.node_id self.access_key = self._dbconfig.access_key logger.info('Node ID: %s', self.node_id) self.api = EdgeAPI() logger.info("Instantiated API") try: self.mqtt_client = MQTTPublisher( node_id=self.node_id, access_key=self.access_key, config=self._remote_mqtt_broker_config) logger.info("Instantiated MQTT") except AttributeError: logger.warning( "No MQTT Connection initialized. Missing MQTT endpoint in remote.yaml" ) self.events = NodeEvents() config_watch = ConfigWatch(self) config_watch.start() command_watch = CommandWatch(self) command_watch.start() self.config = None if self._dbconfig.config: # Configuration is available in DB; use this logger.info('Using stored configuration from database') self.config = self._dbconfig.config else: # Check for a provisioning configuration try: with open( os.path.join(os.getenv('SNAP', './'), 'provisioning', 'config.json'), 'r') as config_json: config = json.load(config_json) logger.info("Using configuration from provisioning file") self.config = config except FileNotFoundError: logger.info("No provisioning config.json file found") except Exception: logger.exception( "Exception while trying to process provisioning config.json" ) # Even if we loaded a stored config, check for a new one self.events.check_new_config.set() # If we still have not got a config, wait for one to be provided if self.config is None: logger.info('No stored configuration available') with self.events.getting_config: self.events.getting_config.wait_for( lambda: self.config is not None) # Load drivers from files, and also add any from the config self.drivers = self.__get_drivers() self.update_drv_from_config() # Updates data endpoints if present in the remote config self.update_endpoints_from_config() @property def node_id(self) -> str: return self._node_id @node_id.setter def node_id(self, value) -> None: self._node_id = value self.kvs.set('node:node_id', value) @property def config(self) -> dict: return self._config @config.setter def config(self, value) -> None: self._config = value if value is not None: self.kvs.set('node:config', value) @property def access_key(self) -> str: return self._access_key @access_key.setter def access_key(self, value) -> None: self._access_key = value self.kvs.set('node:access_key', value) @property def remote_api(self) -> dict: return self._remote_api @remote_api.setter def remote_api(self, value) -> None: self._remote_api = value self.kvs.set('node:remote_api', value) @property def drivers(self) -> dict: return self._drivers @drivers.setter def drivers(self, value) -> None: self._drivers = value def __initialize(self) -> None: node_id = self.__generate_node_id() logger.info('Generated node ID %s' % node_id) access_key = None while not access_key: access_key = self.__do_node_activation(node_id) if not access_key: logger.error( 'Unable to obtain access key. Retrying in %d seconds...' % ACTIVATE_RETRY_DELAY) time.sleep(ACTIVATE_RETRY_DELAY) # If there are existing saved configs (potentially for a different node_id, wipe them) try: q = NodeConfig.delete() n_deleted = q.execute() if n_deleted: logger.info('Deleted %d existing config(s)' % n_deleted) except Exception: logger.warning('Could not clean existing config database') # Save node_id and access_key in database self._dbconfig = NodeConfig.create(node_id=node_id, access_key=access_key) self._dbconfig.save() logger.debug('Saved new config for node ID %s' % node_id) def __generate_node_id(self) -> str: # Get ID (ideally hardware MAC address) that is used to identify logger when pushing data def get_hw_addr(ifname: str) -> str: import socket from fcntl import ioctl import struct with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: info = ioctl(s.fileno(), 0x8927, struct.pack('256s', bytes(ifname, 'utf-8')[:15])) return info[18:24].hex() node_id = None # First try to get the address of the primary Ethernet adapter ifn_wanted = [ 'eth0', 'en0', 'eth1', 'en1', 'em0', 'em1', 'wlan0', 'wlan1' ] for ifn in ifn_wanted: try: node_id = get_hw_addr(ifn) except Exception as e: logger.warn( f"Could not get MAC address of interface {ifn}. Exception {e}" ) if node_id: break if not node_id: logger.warn( 'Cannot find primary network interface MAC; trying UUID MAC') try: from uuid import getnode uuid_node = getnode() node_id = "{0:0{1}x}".format(uuid_node, 12) except Exception: logger.exception( 'Cannot get MAC via UUID method; generating random node ID starting with ff' ) # If that also doesn't work, generate a random 12-character hex string import random node_id = 'ff' + '%010x' % random.randrange(16**10) if node_id == 'd43639139e08': # This is a Moxa with a hardcoded MAC address. Need to generate something semi-random... logger.warning( 'Generating semi-random ID for Moxa with hardcoded MAC') import random node_id = 'd43639' + ('%06x' % random.randrange(16**6)) return node_id def __do_node_activation(self, node_id: str) -> str: # Initiate activation logger.info('Requesting activation for node %s' % node_id) try: r1 = requests.get( 'https://%s/api/%s/nodes/%s/activate' % (self.remote_api['host'], self.remote_api['apiver'], node_id)) rtn = json.loads(r1.text) if r1.status_code == 200: access_key = rtn['access_key'] logger.info('Obtained API key') if rtn: logger.debug('API response: %s' % rtn) else: logger.error('Error %d requesting activation from API' % r1.status_code) if rtn: logger.debug('API response: %s' % rtn) return None except Exception: logger.exception( 'Exception raised while requesting activation from API') return None # Confirm activation logger.info('Confirming activation for node %s' % node_id) try: r2 = requests.post( 'https://%s/api/%s/nodes/%s/activate' % (self.remote_api['host'], self.remote_api['apiver'], node_id), headers={'Authorization': access_key}) rtn = json.loads(r2.text) if r2.status_code == 200: logger.info('Confirmed activation') if rtn: logger.debug('API response: %s' % rtn) else: logger.error('Error %d confirming activation with API' % r2.status_code) if rtn: logger.debug('API response: %s' % rtn) return None except Exception: logger.exception( 'Exception raised while confirming activation with API') return None return access_key def __get_drivers(self) -> dict: drivers = {} drvpath = os.path.join(os.getenv('SNAP', './'), 'drivers') driver_files = [ pos_json for pos_json in os.listdir(drvpath) if pos_json.endswith('.json') ] for drv in driver_files: try: with open(os.path.join(drvpath, drv)) as driver_file: drivers[os.path.splitext(drv)[0]] = json.load(driver_file) logger.info('Loaded driver %s' % drv) except Exception: logger.error('Could not load driver %s' % drv, exc_info=True) return drivers def save_config(self) -> None: """ This method saves the current config to the database. It is not only an internal method, as it needs to be called also by the config_watch thread, when a new config has been obtained from the API. """ try: self._dbconfig.config = self.config self._dbconfig.save() logger.debug('Saved active config to internal database') except Exception: logger.exception( 'Exception raised when attempting to commit configuration to database' ) def update_drv_from_config(self) -> None: """ Check whether there are custom drivers in the config definition, and if so add them to the driver definition. """ if 'drivers' in self.config: try: self.drivers.update(self.config['drivers']) except AttributeError: self.drivers = self.config['drivers'] def update_endpoints_from_config(self) -> None: """ Check whether there are custom endpoints in the config definition, and if so update the data_endpoints """ if 'data_endpoints' in self.config: try: self.data_endpoints = self.config['data_endpoints'] except Exception: logger.exception( 'Exception raised while updating data-endpoints from config' )