def getIdentifier(self): '''! Obtain the resolved identifier string based on the identifier data. @return In case of success: \n Identifier string \n In case of error: \n None ''' try: identifier_data = self.identifier_data if isinstance(identifier_data, dict) is False: if identifier_data == "hostname": return socket.gethostname() elif identifier_data == "hostname-fqdn": return socket.getfqdn() elif identifier_data == "serial-number": return sysEeprom.get_serial_number() elif identifier_data == "product-name": return sysEeprom.get_product_name() elif identifier_data == "sonic-version": return get_sonic_version() else: return identifier_data elif identifier_data.get('url') is not None: url_data = identifier_data.get('url') (fd, filename) = tempfile.mkstemp(prefix='identifier_', dir=getCfg('ztp-tmp')) os.close(fd) urlObj = URL(url_data, filename) updateActivity('Downloading identifier script from \'%s\'' % (urlObj.getSource())) rc, identifier_script = urlObj.download() if rc == 0 and os.path.isfile(identifier_script): updateActivity( 'Executing identifier script downloaded from \'%s\'' % (urlObj.getSource())) (rc, cmd_stdout, cmd_stderr) = runCommand([identifier_script]) if rc != 0: logger.error( 'Error encountered while executing identifier %s Exit code: (%d).' % (identifier_script, rc)) return None if len(cmd_stdout) == 0: return "" else: return cmd_stdout[0] except (TypeError, ValueError) as e: logger.error( 'Exception: [%s] encountered while processing identifier data in dynamic URL.' % str(e)) return None return None
def updateStatus(self, obj, status): '''! Update status of configuration section. Also update the timestamp indicating date/time when it is updated. The changes are also saved to the JSON file on disk which corresponds to the configuration section. @param status (str) Value to be stored as status ''' if isinstance(obj, dict) and isString(status): self.objJson.set(obj, 'status', status) self.objJson.set(obj, 'timestamp', getTimestamp(), True) else: logger.error('Invalid argument type.') raise TypeError('Invalid argument type')
def __removeZTPProfile(self): '''! If ZTP configuration profile is operational, remove ZTP configuration profile and load startup configuration file. If there is no startup configuration file, load factory default configuration. ''' # Do not attempt to remove ZTP configuration if working in unit test mode if self.test_mode: return # Remove ZTP configuration profile if loaded updateActivity('Verifying configuration') # Use a fallback default configuration if configured to _config_fallback = '' if (self.objztpJson is not None and (self.objztpJson['status'] == 'FAILED' or self.objztpJson['status'] == 'SUCCESS') \ and self.objztpJson['config-fallback']) or \ (self.objztpJson is None and getCfg('config-fallback') is True): _config_fallback = ' config-fallback' # Execute profile removal command with appropriate options rc = runCommand(getCfg('ztp-lib-dir') + '/ztp-profile.sh remove' + _config_fallback, capture_stdout=False) # Remove ZTP configuration startup-config if os.path.isfile(getCfg('config-db-json')) is True: try: config_db = None with open(getCfg('config-db-json')) as json_file: config_db = json.load(json_file) json_file.close() if config_db is not None and config_db.get('ZTP'): logger.info("Deleting ZTP configuration saved in '%s'." % (getCfg('config-db-json'))) del config_db['ZTP'] with open(getCfg('config-db-json'), 'w') as json_file: json.dump(config_db, json_file, indent=4) json_file.close() except Exception as e: logger.error( "Exception [%s] encountered while verifying '%s'." % (str(e), getCfg('config-db-json'))) self.__ztp_profile_loaded = False
def __cleanup(self): '''! Remove stale ZTP session data. ''' dir_list = ['ztp-tmp', 'ztp-tmp-persistent'] try: # Remove temporary files created by previous ZTP run for d in dir_list: if os.path.isdir(getCfg(d)): shutil.rmtree(getCfg(d), ignore_errors=True) # Create them again for d in dir_list: os.makedirs(getCfg(d)) except OSError as e: logger.error( 'Exception [%s] encountered while cleaning up temp directories.' % str(e))
def pluginArgs(self, section_name): '''! Resolve the plugin arguments used to be passed as command line arguments to the plugin used to process configuration section @param section_name (str) Configuration section name whose plugin arguments needs to be resolved. @return Concatenated string of all the argements defined \n If no arguments are defined or error encountered: \n None ''' if isString(section_name) is False: raise TypeError('Invalid argument used as section name') elif self.ztpDict.get(section_name) is None: logger.error('Configuration Section %s not found.' % section_name) return None # Obtain plugin data plugin_data = self.ztpDict.get(section_name).get('plugin') # Get the location of this configuration section's input data parsed from the input ZTP JSON file plugin_input = getCfg( 'ztp-tmp-persistent') + '/' + section_name + '/' + getCfg( 'section-input-file') plugin_args = '' ignore_section_data_arg = getField(plugin_data, 'ignore-section-data', bool, False) _plugin_json_args = getField(plugin_data, 'args', str, None) if ignore_section_data_arg is False: plugin_args = plugin_input if _plugin_json_args is not None: plugin_args = plugin_args + ' ' + _plugin_json_args logger.debug('Plugin arguments for %s evaluated to be %s.' % (section_name, plugin_args)) return plugin_args
def getUrl(self, url=None, dst_file=None, incl_http_headers=None, is_secure=True, timeout=None, retry=None, curl_args=None, encrypted=None, verbose=False): '''! Fetch a file using a given url. The content retrieved from the server is stored into a file. In case of a server erreur, the html content is stored into the file. This can be used to have a finer diagnostic of the issue. @param url (str, optional) url of the file you want to get. \n If the url is not given, the one specified in the constructor will be used. @param dst_file (str, optional) Filename for the data being stored. \n If not specified here and in the colnstructor, it will be derived from the url \n (last part of it, e.g. basename). @param incl_http_headers (bool, optional) Include or not the additional HTTP headers \n (product name, serial number, mac address) @param is_secure (bool, optional) Every SSL connection curl makes is verified or not to be secure. \n By default, the SSL connection is assumed secure. @param timeout (int, optional) Maximum number of seconds allowed for curl's connection to take. \n This only limits the connection phase. @param retry (int, optional) Number of times curl will retry in case of a transient error @param curl_args (str, optional) Options you want to pass to curl command line program. \n If no parameters are given here, the one given in the constructor will be used. @param encrypted (bool) Is the connection with the server being encrypted? @return Return a tuple: \n - In case of success: (0 destination_filename)\n - In case of error: (error_code None)\n See here to see the status codes returned:\n https://ec.haxx.se/usingcurl-returns.html \n Note that we return error 20 in case of an unknown error. ''' # Use arguments provided in the constructor if url is None and self.__url is not None: url = self.__url if dst_file is None and self.__dst_file is not None: dst_file = self.__dst_file if incl_http_headers is None and self.__incl_http_headers is not None: incl_http_headers = self.__incl_http_headers if is_secure is None and self.__is_secure is not None: is_secure = self.__is_secure if timeout is None and self.__timeout is not None: timeout = self.__timeout if retry is None and self.__retry is not None: retry = self.__retry if curl_args is None and self.__curl_args is not None: curl_args = self.__curl_args if encrypted is None and self.__encrypted is not None: encrypted = self.__encrypted # We can't run without a URL if url is None: return (-1, dst_file) # If no filename is provided, we use the last part of the url if dst_file is None: dst_file = os.path.basename(url) # If there is no path in the provided filename, we store the file under this default location try: if dst_file.find('/') == -1: dst_file = getCfg('ztp-tmp') + '/' + dst_file except (AttributeError) as e: logger.error("!Exception : %s" % (str(e))) return (20, None) # Create curl command cmd = '/usr/bin/curl -f -v -s -o ' + dst_file if self.__user_agent is not None: cmd += ' -A "' + self.__user_agent + '"' # --user-agent if is_secure is False: cmd += ' -k' # --insecure if timeout is not None and isinstance(timeout, int) is True: cmd += ' --connect-timeout ' + str(timeout) if retry is not None and isinstance(retry, int) is True: cmd += ' --retry ' + str(retry) if incl_http_headers is not None: for h in self.__http_headers: cmd += ' -H \"' + h + '"' # --header if curl_args is not None: cmd += ' ' + curl_args cmd += ' ' + url if verbose is True: logger.debug('%s' % (cmd)) # Execute curl command _retries = retry while True: _start_time = time.time() (rc, cmd_stdout, cmd_stderr) = runCommand(cmd) _current_time = time.time() if rc != 0 and rc in [ 5, 6, 7 ] and _retries != 0 and (_current_time - _start_time) < timeout: logger.debug( "!Error (%d) encountered while processing the command : %s" % (rc, cmd)) time.sleep(timeout - (_current_time - _start_time)) _retries = _retries - 1 continue if rc != 0: logger.error( "!Error (%d) encountered while processing the command : %s" % (rc, cmd)) for l in cmd_stdout: # pragma: no cover logger.error(str(l)) for l in cmd_stderr: logger.error(str(l)) if os.path.isfile(dst_file): os.remove(dst_file) return (20, None) else: break os.chmod(dst_file, stat.S_IRWXU) # Use curl result return (0, dst_file)
def executeLoop(self, test_mode=False): '''! ZTP service loop which peforms provisioning data discovery and initiates processing. ''' updateActivity('Initializing') # Set testing mode self.test_mode = test_mode # Check if ZTP is disabled administratively, bail out if disabled if getCfg('admin-mode') is False: logger.info('ZTP is administratively disabled.') self.__removeZTPProfile() return # Check if ZTP data restart flag is set if os.path.isfile(getCfg('ztp-restart-flag')): self.__ztp_restart = True os.remove(getCfg('ztp-restart-flag')) if self.test_mode: logger.warning( 'ZTP service started in test mode with restricted functionality.' ) else: logger.info('ZTP service started.') self.__ztp_engine_start_time = getTimestamp() _start_time = None self.ztp_mode = 'DISCOVERY' # Main provisioning data discovery loop while self.ztp_mode == 'DISCOVERY': updateActivity('Discovering provisioning data', overwrite=False) try: result = self.__discover() except Exception as e: logger.error( "Exception [%s] encountered while running the discovery logic." % (str(e))) _exc_type, _exc_value, _exc_traceback = sys.exc_info() __tb = traceback.extract_tb(_exc_traceback) for l in __tb: logger.debug(' File ' + l[0] + ', line ' + str(l[1]) + ', in ' + str(l[2])) logger.debug(' ' + str(l[3])) self.__forceRestartDiscovery( "Invalid provisioning data received") continue if result: if self.ztp_mode == 'MANUAL_CONFIG': logger.info( "Configuration file '%s' detected. Shutting down ZTP service." % (getCfg('config-db-json'))) break elif self.ztp_mode != 'DISCOVERY': (rv, msg) = self.__processZTPJson() if rv == "retry": self.ztp_mode = 'DISCOVERY' elif rv == "restart": self.__forceRestartDiscovery(msg) else: break # Initialize in-band interfaces to establish connectivity if not done already self.__loadZTPProfile("discovery") logger.debug('Provisioning data not found.') # Scan for inband interfaces to link up and restart interface connectivity if self.__link_scan(): updateActivity('Restarting network discovery after link scan') logger.info('Restarting network discovery after link scan.') runCommand('systemctl restart interfaces-config', capture_stdout=False) logger.info('Restarted network discovery after link scan.') _start_time = time.time() continue # Start keeping time of last time restart networking was done if _start_time is None: _start_time = time.time() # Check if we have to restart networking if (time.time() - _start_time > getCfg('restart-ztp-interval')): updateActivity('Restarting network discovery') if self.test_mode is False: # Remove existing leases to source new provisioning data self.__cleanup_dhcp_leases() logger.info('Restarting network discovery.') runCommand('systemctl restart interfaces-config', capture_stdout=False) logger.info('Restarted network discovery.') _start_time = time.time() continue # Try after sometime time.sleep(getCfg('discovery-interval')) # Cleanup installed ZTP configuration profile self.__removeZTPProfile() if self.reboot_on_completion and self.test_mode == False: updateActivity('System reboot requested') systemReboot() updateActivity('Exiting ZTP server')
def __downloadURL(self, url_file, dst_file, url_prefix=None): '''! Helper API to read url information from a file, download the file using the url and store contents as a dst_file. @param url_file (str) File containing URL to be downloaded @param dst_file (str) Destination file to be used @param url_prefix (str) Optional string to be prepended to url @return True - If url_file was successfully downloaded False - Failed to download url_file ''' logger.debug('Downloading provided URL %s and saving as %s.' % (url_file, dst_file)) try: # Read the url file and identify the URL to be downloaded f = open(url_file, 'r') url_str = f.readline().strip() f.close() res = urlparse(url_str) if res is None or res.scheme == '': # Use passed url_prefix to construct final URL if url_prefix is not None: url_str = url_prefix + url_str if urlparse(url_str) is None: logger.error( 'Failed to download provided URL %s, malformed url.' % (url_str)) return False else: logger.error( 'Failed to download provided URL %s, malformed url.' % (url_str)) return False # Create a downloader object using source and destination information updateActivity('Downloading provisioning data from %s to %s' % (url_str, dst_file)) logger.info('Downloading provisioning data from %s to %s' % (url_str, dst_file)) objDownloader = Downloader(url_str, dst_file) # Initiate download rc, fname = objDownloader.getUrl() # Check download result if rc == 0 and fname is not None and os.path.isfile(dst_file): # Get the interface on which ZTP data was received self.__read_ztp_interface() return True else: logger.error( 'Failed to download provided URL %s returncode=%d.' % (url_str, rc)) return False except (IOError, OSError) as e: logger.error( 'Exception [%s] encountered during download of provided URL %s.' % (str(e), url_str)) return False
def __processZTPJson(self): '''! Process ZTP JSON file downloaded using URL provided by DHCP Option 67, DHCPv6 Option 59 or local ZTP JSON file. ''' logger.debug('Starting to process ZTP JSON file %s.' % self.json_src) updateActivity('Processing ZTP JSON file %s' % self.json_src) try: # Read provided ZTP JSON file and load it self.objztpJson = ZTPJson(self.json_src, getCfg('ztp-json')) except ValueError as e: logger.error( 'Exception [%s] occured while processing ZTP JSON file %s.' % (str(e), self.json_src)) logger.error('ZTP JSON file %s processing failed.' % (self.json_src)) try: os.remove(getCfg('ztp-json')) if os.path.isfile(getCfg('ztp-json-shadow')): os.remove(getCfg('ztp-json-shadow')) except OSError as v: if v.errno != errno.ENOENT: logger.warning( 'Exception [%s] encountered while deleting ZTP JSON file %s.' % (str(v), getCfg('ztp-json'))) raise self.objztpJson = None # Restart networking after a wait time to discover new provisioning data if getCfg('restart-ztp-on-invalid-data'): return ("restart", "Invalid provisioning data processed") else: return ("stop", "Invalid provisioning data processed") if self.objztpJson['ztp-json-source'] is None: self.objztpJson['ztp-json-source'] = self.ztp_mode # Check if ZTP process has already completed. If not mark start of ZTP. if self.objztpJson['status'] == 'BOOT': self.objztpJson['status'] = 'IN-PROGRESS' if self.objztpJson['start-timestamp'] is None: self.objztpJson[ 'start-timestamp'] = self.__ztp_engine_start_time self.objztpJson.objJson.writeJson() elif self.objztpJson['status'] != 'IN-PROGRESS': # Re-start ZTP if requested if getCfg('monitor-startup-config') is True and self.__ztp_restart: self.__ztp_restart = False # Discover new ZTP data after deleting historic ZTP data logger.info( "ZTP restart requested. Deleting previous ZTP session JSON data." ) os.remove(getCfg('ztp-json')) if os.path.isfile(getCfg('ztp-json-shadow')): os.remove(getCfg('ztp-json-shadow')) self.objztpJson = None return ("retry", "ZTP restart requested") else: # ZTP was successfully completed in previous session. No need to proceed, return and exit service. logger.info( "ZTP already completed with result %s at %s." % (self.objztpJson['status'], self.objztpJson['timestamp'])) return ("stop", "ZTP completed") logger.info('Starting ZTP using JSON file %s at %s.' % (self.json_src, self.objztpJson['timestamp'])) # Initialize connectivity if not done already self.__loadZTPProfile("resume") # Process available configuration sections in ZTP JSON self.__processConfigSections() # Determine ZTP result self.__evalZTPResult() # Check restart ZTP condition # ZTP result is failed and restart-ztp-on-failure is set or _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \ self.objztpJson['restart-ztp-on-failure'] == True) # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \ self.objztpJson['restart-ztp-no-config'] == True and \ self.objztpJson['config-fallback'] == False and os.path.isfile(getCfg('config-db-json')) is False ) # Mark ZTP for restart if _restart_ztp_missing_config or _restart_ztp_on_failure: os.remove(getCfg('ztp-json')) if os.path.isfile(getCfg('ztp-json-shadow')): os.remove(getCfg('ztp-json-shadow')) self.objztpJson = None # Remove startup-config file to obtain a new one through ZTP if getCfg('monitor-startup-config') is True and os.path.isfile( getCfg('config-db-json')): os.remove(getCfg('config-db-json')) if _restart_ztp_missing_config: return ( "restart", "ZTP completed but startup configuration '%s' not found" % (getCfg('config-db-json'))) elif _restart_ztp_on_failure: return ("restart", "ZTP completed with FAILED status") return ("stop", "ZTP completed")
def __processConfigSections(self): '''! Process and execute individual configuration sections defined in ZTP JSON. Plugin for each configuration section is resolved and executed. Configuration section data is provided as command line argument to the plugin. Each and every section is processed before this function returns. ''' # Obtain a copy of the list of configuration sections section_names = list(self.objztpJson.section_names) # set temporary flags abort = False sort = True logger.debug('Processing configuration sections: %s' % ', '.join(section_names)) # Loop through each sections till all of them are processed while section_names and abort is False: # Take a fresh sorted list to begin with and if any changes happen to it while processing if sort: sorted_list = sorted(section_names) sort = False # Loop through configuration section in a sorted order for sec in sorted_list: # Retrieve configuration section data section = self.objztpJson.ztpDict.get(sec) try: # Retrieve individual section's progress sec_status = section.get('status') if sec_status == 'BOOT' or sec_status == 'SUSPEND': # Mark section status as in progress self.objztpJson.updateStatus(section, 'IN-PROGRESS') if section.get('start-timestamp') is None: section['start-timestamp'] = section['timestamp'] self.objztpJson.objJson.writeJson() logger.info( 'Processing configuration section %s at %s.' % (sec, section['timestamp'])) elif sec_status != 'IN-PROGRESS': # Skip completed sections logger.debug( 'Removing section %s from list. Status %s.' % (sec, sec_status)) section_names.remove(sec) # set flag to sort the configuration sections list again sort = True # Ignore disabled configuration sections if sec_status == 'DISABLED': logger.info( 'Configuration section %s skipped as its status is set to DISABLED.' % sec) continue updateActivity('Processing configuration section %s' % sec) # Get the appropriate plugin to be used for this configuration section plugin = self.objztpJson.plugin(sec) # Get the location of this configuration section's input data parsed from the input ZTP JSON file plugin_input = getCfg( 'ztp-tmp-persistent') + '/' + sec + '/' + getCfg( 'section-input-file') # Initialize result flag to FAILED finalResult = 'FAILED' rc = 1 # Check if plugin could not be resolved if plugin is None: logger.error( 'Unable to resolve plugin to be used for configuration section %s. Marking it as FAILED.' % sec) section[ 'error'] = 'Unable to find or download requested plugin' elif os.path.isfile(plugin) and os.path.isfile( plugin_input): plugin_args = self.objztpJson.pluginArgs(sec) plugin_data = section.get('plugin') # Determine if shell has to be used to execute plugin _shell = getField(plugin_data, 'shell', bool, False) # Construct the full plugin command string along with arguments plugin_cmd = plugin if plugin_args is not None: plugin_cmd = plugin_cmd + ' ' + plugin_args # A plugin has been resolved and its input configuration section data as well logger.debug('Executing plugin %s.' % (plugin_cmd)) # Execute identified plugin rc = runCommand(plugin_cmd, capture_stdout=False, use_shell=_shell) logger.debug('Plugin %s exit code = %d.' % (plugin_cmd, rc)) # Compare plugin exit code if rc == 0: finalResult = 'SUCCESS' elif section.get('suspend-exit-code' ) is not None and section.get( 'suspend-exit-code') == rc: finalResult = 'SUSPEND' else: finalResult = 'FAILED' except Exception as e: logger.debug( 'Exception [%s] encountered for configuration section %s.' % (str(e), sec)) logger.info( 'Exception encountered while processing configuration section %s. Marking it as FAILED.' % sec) section[ 'error'] = 'Exception [%s] encountered while executing the plugin' % ( str(e)) finalResult = 'FAILED' # Update this configuration section's result in ztp json file logger.info( 'Processed Configuration section %s with result %s, exit code (%d) at %s.' % (sec, finalResult, rc, section['timestamp'])) if finalResult == 'FAILED' and section.get('error') is None: section['error'] = 'Plugin failed' section['exit-code'] = rc self.objztpJson.updateStatus(section, finalResult) # Check if abort ZTP on failure flag is set if getField(section, 'halt-on-failure', bool, False) is True and finalResult == 'FAILED': logger.info( 'Halting ZTP as Configuration section %s FAILED and halt-on-failure flag is set.' % sec) abort = True break # Check reboot on result flags self.__rebootAction(section)
def plugin(self, section_name): '''! Resolve the plugin used to process a configuration section. If the plugin is specified as a url object, the plugin is downloaded. @param section_name (str) Configuration section name whose plugin needs to be resolved. @return If plugin is resolved using configuration section data: \n Expanded file path to plugin file used to process configuration section. \n If plugin is not found or error encountered: \n None ''' if isString(section_name) is False: raise TypeError('Invalid argument used as section name') elif self.ztpDict.get(section_name) is None: logger.error('Configuration Section %s not found.' % section_name) return None plugin_data = self.ztpDict.get(section_name).get('plugin') name = None if plugin_data is not None and isinstance(plugin_data, dict): logger.debug( 'User defined plugin detected for configuration section %s.' % section_name) plugin_file = getCfg( 'ztp-tmp-persistent') + '/' + section_name + '/' + 'plugin' try: # Re-use the plugin if already present if os.path.isfile(plugin_file) is True: return plugin_file if plugin_data.get('dynamic-url'): dyn_url_data = plugin_data.get('dynamic-url') if isinstance(dyn_url_data, dict) and dyn_url_data.get( 'destination') is not None: objDynUrl = DynamicURL(dyn_url_data) else: objDynUrl = DynamicURL(dyn_url_data, plugin_file) rc, plugin_file = objDynUrl.download() return plugin_file elif plugin_data.get('url'): url_data = plugin_data.get('url') if isinstance( url_data, dict) and url_data.get('destination') is not None: objUrl = URL(url_data) else: objUrl = URL(url_data, plugin_file) updateActivity( 'Downloading plugin \'%s\' for configuration section %s' % (objUrl.getSource(), section_name)) rc, plugin_file = objUrl.download() if rc != 0: logger.error( 'Failed to download plugin \'%s\' for configuration section %s.' % (objUrl.getSource(), section_name)) return plugin_file elif plugin_data.get('name') is not None: name = plugin_data.get('name') except (TypeError, ValueError, OSError, IOError) as e: logger.error( 'Exception [%s] encountered while determining plugin for configuration section %s.' % (str(e), section_name)) return None elif plugin_data is not None and isString(plugin_data): name = plugin_data elif plugin_data is not None: logger.error( 'Invalid plugin data type used for configuration section %s.' % section_name) return None # plugin name is not provided in section data, use section name as plugin name if name is None: res = re.split("^[0-9]+-", section_name, maxsplit=1) if len(res) > 1: name = res[1] else: name = res[0] logger.debug( 'ZTP provided plugin %s is being used for configuration section %s.' % (name, section_name)) if os.path.isfile(getCfg('plugins-dir') + '/' + name) is True: return getCfg('plugins-dir') + '/' + name return None