def signal_handler(signum, frame):
'''!
This signal handler is called on SIGTERM or SIGINT
'''
logger.warning('Received terminate signal. Shutting down.')
updateActivity('Received terminate signal. Shutting down.')
# Wait for some time
count = getCfg('sighandler-wait-interval')
while count > 0:
done = True
for pid in runcmd_pids:
if check_pid(pid):
done = False
try:
(wpid, status) = os.waitpid(pid, os.WNOHANG)
if wpid == pid:
print('Process pid %d returned with status %d.' %
(pid, status))
except OSError as v:
print('pid %d : %s' % (pid, str(v)))
if done:
break
time.sleep(1)
count -= 1
# Kill any process which might still be running
for pid in runcmd_pids:
if check_pid(pid):
print('Process %d still alive, send kill signal.' % (pid))
os.kill(pid, signal.SIGKILL)
sys.exit(0)
def getIdentifier(self):
'''!
Obtain the resolved identifier string based on the identifier data.
@return
In case of success: \n
Identifier string \n
In case of error: \n
None
'''
try:
identifier_data = self.identifier_data
if isinstance(identifier_data, dict) is False:
if identifier_data == "hostname":
return socket.gethostname()
elif identifier_data == "hostname-fqdn":
return socket.getfqdn()
elif identifier_data == "serial-number":
return sysEeprom.get_serial_number()
elif identifier_data == "product-name":
return sysEeprom.get_product_name()
elif identifier_data == "sonic-version":
return get_sonic_version()
else:
return identifier_data
elif identifier_data.get('url') is not None:
url_data = identifier_data.get('url')
(fd, filename) = tempfile.mkstemp(prefix='identifier_',
dir=getCfg('ztp-tmp'))
os.close(fd)
urlObj = URL(url_data, filename)
updateActivity('Downloading identifier script from \'%s\'' %
(urlObj.getSource()))
rc, identifier_script = urlObj.download()
if rc == 0 and os.path.isfile(identifier_script):
updateActivity(
'Executing identifier script downloaded from \'%s\'' %
(urlObj.getSource()))
(rc, cmd_stdout,
cmd_stderr) = runCommand([identifier_script])
if rc != 0:
logger.error(
'Error encountered while executing identifier %s Exit code: (%d).'
% (identifier_script, rc))
return None
if len(cmd_stdout) == 0:
return ""
else:
return cmd_stdout[0]
except (TypeError, ValueError) as e:
logger.error(
'Exception: [%s] encountered while processing identifier data in dynamic URL.'
% str(e))
return None
return None
def __forceRestartDiscovery(self, msg):
# Remove existing leases to source new provisioning data
self.__cleanup_dhcp_leases()
_msg = '%s. Waiting for %d seconds before restarting ZTP.' % (
msg, getCfg('restart-ztp-interval'))
logger.warning(_msg)
updateActivity(_msg)
time.sleep(getCfg('restart-ztp-interval'))
self.ztp_mode = 'DISCOVERY'
# Force install of ZTP configuration profile
self.__ztp_profile_loaded = False
# Restart link-scan
self.__intf_state = dict()
def __evalZTPResult(self):
'''!
Determines the final result of ZTP after processing all configuration sections and
their results. Als performs system reboot if reboot-on flag is set
ZTP result is determined as SUCCESS if - Configuration section(s) status is SUCCESS
or (configuration section(s) status is FAILED and
configuration section(s) ignore-result is True)
or ZTP ignore-result is True
Disabled Configuration sections are ignored.
'''
updateActivity('Evaluating ZTP result')
# Check if overall ZTP ignore-result flag is set
if self.objztpJson['ignore-result']:
self.objztpJson['status'] = 'SUCCESS'
logger.info(
'ZTP result is marked as SUCCESS at %s. ZTP ignore-result flag is set.'
% self.objztpJson['timestamp'])
else:
# Look through individual configuration sections
for sec in self.objztpJson.section_names:
# Retrieve section data
section = self.objztpJson.ztpDict.get(sec)
logger.info('Checking configuration section %s result: %s, ignore-result: %r.' % \
(sec, section.get('status'), section.get('ignore-result')))
# Check if configuration section has failed and ignore-result flag is not set
if section.get('status') == 'FAILED' and section.get(
'ignore-result') is False:
# Mark ZTP as failed and bail out
self.objztpJson['error'] = '%s FAILED' % sec
self.objztpJson['status'] = 'FAILED'
logger.info(
'ZTP failed at %s as configuration section %s FAILED.'
% (self.objztpJson['timestamp'], sec))
return
# Mark ZTP as SUCCESS
self.objztpJson['status'] = 'SUCCESS'
logger.info('ZTP successfully completed at %s.' %
self.objztpJson['timestamp'])
# Check reboot on result flags and take action
self.__rebootAction(self.objztpJson.ztpDict, delayed_reboot=True)
def __removeZTPProfile(self):
'''!
If ZTP configuration profile is operational, remove ZTP configuration profile and load
startup configuration file. If there is no startup configuration file,
load factory default configuration.
'''
# Do not attempt to remove ZTP configuration if working in unit test mode
if self.test_mode:
return
# Remove ZTP configuration profile if loaded
updateActivity('Verifying configuration')
# Use a fallback default configuration if configured to
_config_fallback = ''
if (self.objztpJson is not None and (self.objztpJson['status'] == 'FAILED' or self.objztpJson['status'] == 'SUCCESS') \
and self.objztpJson['config-fallback']) or \
(self.objztpJson is None and getCfg('config-fallback') is True):
_config_fallback = ' config-fallback'
# Execute profile removal command with appropriate options
rc = runCommand(getCfg('ztp-lib-dir') + '/ztp-profile.sh remove' +
_config_fallback,
capture_stdout=False)
# Remove ZTP configuration startup-config
if os.path.isfile(getCfg('config-db-json')) is True:
try:
config_db = None
with open(getCfg('config-db-json')) as json_file:
config_db = json.load(json_file)
json_file.close()
if config_db is not None and config_db.get('ZTP'):
logger.info("Deleting ZTP configuration saved in '%s'." %
(getCfg('config-db-json')))
del config_db['ZTP']
with open(getCfg('config-db-json'), 'w') as json_file:
json.dump(config_db, json_file, indent=4)
json_file.close()
except Exception as e:
logger.error(
"Exception [%s] encountered while verifying '%s'." %
(str(e), getCfg('config-db-json')))
self.__ztp_profile_loaded = False
def __rebootAction(self, section, delayed_reboot=False):
'''!
Perform system reboot if reboot-on-success or reboot-on-failure is defined in the
configuration section data.
@param section (dict) Configuration section data containing status and reboot-on flags
'''
# Obtain section status
status = section.get('status')
# Check if flag is set to reboot on SUCCESS and status is SUCCESS as well
if getField(section, 'reboot-on-success', bool,
False) is True and status == 'SUCCESS':
logger.warning(
'ZTP is rebooting the device as reboot-on-success flag is set.'
)
updateActivity('System reboot requested on success')
if self.test_mode and delayed_reboot == False:
sys.exit(0)
else:
if delayed_reboot:
self.reboot_on_completion = True
else:
systemReboot()
# Check if flag is set to reboot on FAIL and status is FAILED as well
if getField(section, 'reboot-on-failure', bool,
False) is True and status == 'FAILED':
logger.warning(
'ZTP is rebooting the device as reboot-on-failure flag is set.'
)
updateActivity('System reboot requested on failure')
if self.test_mode and delayed_reboot == False:
sys.exit(0)
else:
if delayed_reboot:
self.reboot_on_completion = True
else:
systemReboot()
def __loadZTPProfile(self, event):
'''!
Load ZTP configuration profile if there is no saved configuration file.
This establishes connectivity to all interfaces and starts DHCP discovery.
'''
# Do not attempt to install ZTP configuration if working in unit test mode
if self.test_mode:
return False
if self.__ztp_profile_loaded is False:
updateActivity('Checking running configuration')
logger.info(
'Checking running configuration to load ZTP configuration profile.'
)
cmd = getCfg('ztp-lib-dir') + '/ztp-profile.sh install ' + event
# When performing ZTP discovery, force load ZTP profile. When
# ZTP is resuming previous session, use configuration already loaded during
# config-setup
rc = runCommand(cmd, capture_stdout=False)
self.__ztp_profile_loaded = True
return True
return False
def executeLoop(self, test_mode=False):
'''!
ZTP service loop which peforms provisioning data discovery and initiates processing.
'''
updateActivity('Initializing')
# Set testing mode
self.test_mode = test_mode
# Check if ZTP is disabled administratively, bail out if disabled
if getCfg('admin-mode') is False:
logger.info('ZTP is administratively disabled.')
self.__removeZTPProfile()
return
# Check if ZTP data restart flag is set
if os.path.isfile(getCfg('ztp-restart-flag')):
self.__ztp_restart = True
os.remove(getCfg('ztp-restart-flag'))
if self.test_mode:
logger.warning(
'ZTP service started in test mode with restricted functionality.'
)
else:
logger.info('ZTP service started.')
self.__ztp_engine_start_time = getTimestamp()
_start_time = None
self.ztp_mode = 'DISCOVERY'
# Main provisioning data discovery loop
while self.ztp_mode == 'DISCOVERY':
updateActivity('Discovering provisioning data', overwrite=False)
try:
result = self.__discover()
except Exception as e:
logger.error(
"Exception [%s] encountered while running the discovery logic."
% (str(e)))
_exc_type, _exc_value, _exc_traceback = sys.exc_info()
__tb = traceback.extract_tb(_exc_traceback)
for l in __tb:
logger.debug(' File ' + l[0] + ', line ' + str(l[1]) +
', in ' + str(l[2]))
logger.debug(' ' + str(l[3]))
self.__forceRestartDiscovery(
"Invalid provisioning data received")
continue
if result:
if self.ztp_mode == 'MANUAL_CONFIG':
logger.info(
"Configuration file '%s' detected. Shutting down ZTP service."
% (getCfg('config-db-json')))
break
elif self.ztp_mode != 'DISCOVERY':
(rv, msg) = self.__processZTPJson()
if rv == "retry":
self.ztp_mode = 'DISCOVERY'
elif rv == "restart":
self.__forceRestartDiscovery(msg)
else:
break
# Initialize in-band interfaces to establish connectivity if not done already
self.__loadZTPProfile("discovery")
logger.debug('Provisioning data not found.')
# Scan for inband interfaces to link up and restart interface connectivity
if self.__link_scan():
updateActivity('Restarting network discovery after link scan')
logger.info('Restarting network discovery after link scan.')
runCommand('systemctl restart interfaces-config',
capture_stdout=False)
logger.info('Restarted network discovery after link scan.')
_start_time = time.time()
continue
# Start keeping time of last time restart networking was done
if _start_time is None:
_start_time = time.time()
# Check if we have to restart networking
if (time.time() - _start_time > getCfg('restart-ztp-interval')):
updateActivity('Restarting network discovery')
if self.test_mode is False:
# Remove existing leases to source new provisioning data
self.__cleanup_dhcp_leases()
logger.info('Restarting network discovery.')
runCommand('systemctl restart interfaces-config',
capture_stdout=False)
logger.info('Restarted network discovery.')
_start_time = time.time()
continue
# Try after sometime
time.sleep(getCfg('discovery-interval'))
# Cleanup installed ZTP configuration profile
self.__removeZTPProfile()
if self.reboot_on_completion and self.test_mode == False:
updateActivity('System reboot requested')
systemReboot()
updateActivity('Exiting ZTP server')
def __downloadURL(self, url_file, dst_file, url_prefix=None):
'''!
Helper API to read url information from a file, download the
file using the url and store contents as a dst_file.
@param url_file (str) File containing URL to be downloaded
@param dst_file (str) Destination file to be used
@param url_prefix (str) Optional string to be prepended to url
@return True - If url_file was successfully downloaded
False - Failed to download url_file
'''
logger.debug('Downloading provided URL %s and saving as %s.' %
(url_file, dst_file))
try:
# Read the url file and identify the URL to be downloaded
f = open(url_file, 'r')
url_str = f.readline().strip()
f.close()
res = urlparse(url_str)
if res is None or res.scheme == '':
# Use passed url_prefix to construct final URL
if url_prefix is not None:
url_str = url_prefix + url_str
if urlparse(url_str) is None:
logger.error(
'Failed to download provided URL %s, malformed url.'
% (url_str))
return False
else:
logger.error(
'Failed to download provided URL %s, malformed url.' %
(url_str))
return False
# Create a downloader object using source and destination information
updateActivity('Downloading provisioning data from %s to %s' %
(url_str, dst_file))
logger.info('Downloading provisioning data from %s to %s' %
(url_str, dst_file))
objDownloader = Downloader(url_str, dst_file)
# Initiate download
rc, fname = objDownloader.getUrl()
# Check download result
if rc == 0 and fname is not None and os.path.isfile(dst_file):
# Get the interface on which ZTP data was received
self.__read_ztp_interface()
return True
else:
logger.error(
'Failed to download provided URL %s returncode=%d.' %
(url_str, rc))
return False
except (IOError, OSError) as e:
logger.error(
'Exception [%s] encountered during download of provided URL %s.'
% (str(e), url_str))
return False
def __processZTPJson(self):
'''!
Process ZTP JSON file downloaded using URL provided by DHCP Option 67, DHCPv6 Option 59 or
local ZTP JSON file.
'''
logger.debug('Starting to process ZTP JSON file %s.' % self.json_src)
updateActivity('Processing ZTP JSON file %s' % self.json_src)
try:
# Read provided ZTP JSON file and load it
self.objztpJson = ZTPJson(self.json_src, getCfg('ztp-json'))
except ValueError as e:
logger.error(
'Exception [%s] occured while processing ZTP JSON file %s.' %
(str(e), self.json_src))
logger.error('ZTP JSON file %s processing failed.' %
(self.json_src))
try:
os.remove(getCfg('ztp-json'))
if os.path.isfile(getCfg('ztp-json-shadow')):
os.remove(getCfg('ztp-json-shadow'))
except OSError as v:
if v.errno != errno.ENOENT:
logger.warning(
'Exception [%s] encountered while deleting ZTP JSON file %s.'
% (str(v), getCfg('ztp-json')))
raise
self.objztpJson = None
# Restart networking after a wait time to discover new provisioning data
if getCfg('restart-ztp-on-invalid-data'):
return ("restart", "Invalid provisioning data processed")
else:
return ("stop", "Invalid provisioning data processed")
if self.objztpJson['ztp-json-source'] is None:
self.objztpJson['ztp-json-source'] = self.ztp_mode
# Check if ZTP process has already completed. If not mark start of ZTP.
if self.objztpJson['status'] == 'BOOT':
self.objztpJson['status'] = 'IN-PROGRESS'
if self.objztpJson['start-timestamp'] is None:
self.objztpJson[
'start-timestamp'] = self.__ztp_engine_start_time
self.objztpJson.objJson.writeJson()
elif self.objztpJson['status'] != 'IN-PROGRESS':
# Re-start ZTP if requested
if getCfg('monitor-startup-config') is True and self.__ztp_restart:
self.__ztp_restart = False
# Discover new ZTP data after deleting historic ZTP data
logger.info(
"ZTP restart requested. Deleting previous ZTP session JSON data."
)
os.remove(getCfg('ztp-json'))
if os.path.isfile(getCfg('ztp-json-shadow')):
os.remove(getCfg('ztp-json-shadow'))
self.objztpJson = None
return ("retry", "ZTP restart requested")
else:
# ZTP was successfully completed in previous session. No need to proceed, return and exit service.
logger.info(
"ZTP already completed with result %s at %s." %
(self.objztpJson['status'], self.objztpJson['timestamp']))
return ("stop", "ZTP completed")
logger.info('Starting ZTP using JSON file %s at %s.' %
(self.json_src, self.objztpJson['timestamp']))
# Initialize connectivity if not done already
self.__loadZTPProfile("resume")
# Process available configuration sections in ZTP JSON
self.__processConfigSections()
# Determine ZTP result
self.__evalZTPResult()
# Check restart ZTP condition
# ZTP result is failed and restart-ztp-on-failure is set or
_restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \
self.objztpJson['restart-ztp-on-failure'] == True)
# ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set
_restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \
self.objztpJson['restart-ztp-no-config'] == True and \
self.objztpJson['config-fallback'] == False and
os.path.isfile(getCfg('config-db-json')) is False )
# Mark ZTP for restart
if _restart_ztp_missing_config or _restart_ztp_on_failure:
os.remove(getCfg('ztp-json'))
if os.path.isfile(getCfg('ztp-json-shadow')):
os.remove(getCfg('ztp-json-shadow'))
self.objztpJson = None
# Remove startup-config file to obtain a new one through ZTP
if getCfg('monitor-startup-config') is True and os.path.isfile(
getCfg('config-db-json')):
os.remove(getCfg('config-db-json'))
if _restart_ztp_missing_config:
return (
"restart",
"ZTP completed but startup configuration '%s' not found" %
(getCfg('config-db-json')))
elif _restart_ztp_on_failure:
return ("restart", "ZTP completed with FAILED status")
return ("stop", "ZTP completed")
def __processConfigSections(self):
'''!
Process and execute individual configuration sections defined in ZTP JSON. Plugin for each
configuration section is resolved and executed. Configuration section data is provided as
command line argument to the plugin. Each and every section is processed before this function
returns.
'''
# Obtain a copy of the list of configuration sections
section_names = list(self.objztpJson.section_names)
# set temporary flags
abort = False
sort = True
logger.debug('Processing configuration sections: %s' %
', '.join(section_names))
# Loop through each sections till all of them are processed
while section_names and abort is False:
# Take a fresh sorted list to begin with and if any changes happen to it while processing
if sort:
sorted_list = sorted(section_names)
sort = False
# Loop through configuration section in a sorted order
for sec in sorted_list:
# Retrieve configuration section data
section = self.objztpJson.ztpDict.get(sec)
try:
# Retrieve individual section's progress
sec_status = section.get('status')
if sec_status == 'BOOT' or sec_status == 'SUSPEND':
# Mark section status as in progress
self.objztpJson.updateStatus(section, 'IN-PROGRESS')
if section.get('start-timestamp') is None:
section['start-timestamp'] = section['timestamp']
self.objztpJson.objJson.writeJson()
logger.info(
'Processing configuration section %s at %s.' %
(sec, section['timestamp']))
elif sec_status != 'IN-PROGRESS':
# Skip completed sections
logger.debug(
'Removing section %s from list. Status %s.' %
(sec, sec_status))
section_names.remove(sec)
# set flag to sort the configuration sections list again
sort = True
# Ignore disabled configuration sections
if sec_status == 'DISABLED':
logger.info(
'Configuration section %s skipped as its status is set to DISABLED.'
% sec)
continue
updateActivity('Processing configuration section %s' % sec)
# Get the appropriate plugin to be used for this configuration section
plugin = self.objztpJson.plugin(sec)
# Get the location of this configuration section's input data parsed from the input ZTP JSON file
plugin_input = getCfg(
'ztp-tmp-persistent') + '/' + sec + '/' + getCfg(
'section-input-file')
# Initialize result flag to FAILED
finalResult = 'FAILED'
rc = 1
# Check if plugin could not be resolved
if plugin is None:
logger.error(
'Unable to resolve plugin to be used for configuration section %s. Marking it as FAILED.'
% sec)
section[
'error'] = 'Unable to find or download requested plugin'
elif os.path.isfile(plugin) and os.path.isfile(
plugin_input):
plugin_args = self.objztpJson.pluginArgs(sec)
plugin_data = section.get('plugin')
# Determine if shell has to be used to execute plugin
_shell = getField(plugin_data, 'shell', bool, False)
# Construct the full plugin command string along with arguments
plugin_cmd = plugin
if plugin_args is not None:
plugin_cmd = plugin_cmd + ' ' + plugin_args
# A plugin has been resolved and its input configuration section data as well
logger.debug('Executing plugin %s.' % (plugin_cmd))
# Execute identified plugin
rc = runCommand(plugin_cmd,
capture_stdout=False,
use_shell=_shell)
logger.debug('Plugin %s exit code = %d.' %
(plugin_cmd, rc))
# Compare plugin exit code
if rc == 0:
finalResult = 'SUCCESS'
elif section.get('suspend-exit-code'
) is not None and section.get(
'suspend-exit-code') == rc:
finalResult = 'SUSPEND'
else:
finalResult = 'FAILED'
except Exception as e:
logger.debug(
'Exception [%s] encountered for configuration section %s.'
% (str(e), sec))
logger.info(
'Exception encountered while processing configuration section %s. Marking it as FAILED.'
% sec)
section[
'error'] = 'Exception [%s] encountered while executing the plugin' % (
str(e))
finalResult = 'FAILED'
# Update this configuration section's result in ztp json file
logger.info(
'Processed Configuration section %s with result %s, exit code (%d) at %s.'
% (sec, finalResult, rc, section['timestamp']))
if finalResult == 'FAILED' and section.get('error') is None:
section['error'] = 'Plugin failed'
section['exit-code'] = rc
self.objztpJson.updateStatus(section, finalResult)
# Check if abort ZTP on failure flag is set
if getField(section, 'halt-on-failure', bool,
False) is True and finalResult == 'FAILED':
logger.info(
'Halting ZTP as Configuration section %s FAILED and halt-on-failure flag is set.'
% sec)
abort = True
break
# Check reboot on result flags
self.__rebootAction(section)
def plugin(self, section_name):
'''!
Resolve the plugin used to process a configuration section. If the plugin is specified
as a url object, the plugin is downloaded.
@param section_name (str) Configuration section name whose plugin needs to be resolved.
@return
If plugin is resolved using configuration section data: \n
Expanded file path to plugin file used to process configuration section. \n
If plugin is not found or error encountered: \n
None
'''
if isString(section_name) is False:
raise TypeError('Invalid argument used as section name')
elif self.ztpDict.get(section_name) is None:
logger.error('Configuration Section %s not found.' % section_name)
return None
plugin_data = self.ztpDict.get(section_name).get('plugin')
name = None
if plugin_data is not None and isinstance(plugin_data, dict):
logger.debug(
'User defined plugin detected for configuration section %s.' %
section_name)
plugin_file = getCfg(
'ztp-tmp-persistent') + '/' + section_name + '/' + 'plugin'
try:
# Re-use the plugin if already present
if os.path.isfile(plugin_file) is True:
return plugin_file
if plugin_data.get('dynamic-url'):
dyn_url_data = plugin_data.get('dynamic-url')
if isinstance(dyn_url_data, dict) and dyn_url_data.get(
'destination') is not None:
objDynUrl = DynamicURL(dyn_url_data)
else:
objDynUrl = DynamicURL(dyn_url_data, plugin_file)
rc, plugin_file = objDynUrl.download()
return plugin_file
elif plugin_data.get('url'):
url_data = plugin_data.get('url')
if isinstance(
url_data,
dict) and url_data.get('destination') is not None:
objUrl = URL(url_data)
else:
objUrl = URL(url_data, plugin_file)
updateActivity(
'Downloading plugin \'%s\' for configuration section %s'
% (objUrl.getSource(), section_name))
rc, plugin_file = objUrl.download()
if rc != 0:
logger.error(
'Failed to download plugin \'%s\' for configuration section %s.'
% (objUrl.getSource(), section_name))
return plugin_file
elif plugin_data.get('name') is not None:
name = plugin_data.get('name')
except (TypeError, ValueError, OSError, IOError) as e:
logger.error(
'Exception [%s] encountered while determining plugin for configuration section %s.'
% (str(e), section_name))
return None
elif plugin_data is not None and isString(plugin_data):
name = plugin_data
elif plugin_data is not None:
logger.error(
'Invalid plugin data type used for configuration section %s.' %
section_name)
return None
# plugin name is not provided in section data, use section name as plugin name
if name is None:
res = re.split("^[0-9]+-", section_name, maxsplit=1)
if len(res) > 1:
name = res[1]
else:
name = res[0]
logger.debug(
'ZTP provided plugin %s is being used for configuration section %s.'
% (name, section_name))
if os.path.isfile(getCfg('plugins-dir') + '/' + name) is True:
return getCfg('plugins-dir') + '/' + name
return None
