def wait_for_do_to_complete(): """Blocks until the f5-declarative-onboarding is complete""" LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.debug('waiting for iControl LX to become available') tmos_onboard_utils.wait_for_icontrollx() LOG.debug('starting f5-declarative-onboarding watcher') end_time = time.time() + DO_TIMEOUT while (end_time - time.time()) > 0: try: response = tmos_onboard_utils.get_do_declaration() if hasattr(response, 'status_code'): if response.status_code == 200: LOG.info('f5-declarative-onboarding shows complete') return True if response.status_code > 399: json_resp = response.json() LOG.debug('f5-declarative-onboarding error %s - %s', response.status_code, json_resp) if 'status' in json_resp and 'declaration' in json_resp: return False time.sleep(1) except Exception as excpt: LOG.error('f5-declarative-onboarding threw an execption: %s', excpt) time.sleep(1) LOG.error( 'f5-declarative-onboarding did not successfully complete after %s', DO_TIMEOUT) return False
def wait_for_do_to_complete(): """Blocks until the f5-declarative-onboarding is complete""" LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.debug('waiting for iControl LX to become available') tmos_onboard_utils.wait_for_icontrollx() LOG.debug('starting f5-declarative-onboarding watcher') restarted_services = False end_time = time.time() + DO_TIMEOUT while (end_time - time.time()) > 0: try: response = tmos_onboard_utils.get_do_status_codes() if hasattr(response, 'status_code'): if response.status_code == 200: json_resp = response.json() status_code = json_resp['result']['code'] status = json_resp['result']['status'] if status_code == 200 and status == "OK": LOG.info('f5-declarative-onboarding shows complete') return True elif status_code == 503 and status == "ERROR": LOG.error('system not responsive, aborting onboarding') return False else: if status_code > 399: if 'errors' in json_resp['result']: errors = json_resp['result'] if len(errors) > 0: LOG.error( 'f5-declarative-onboarding error %s - %s', status_code, json_resp['errors'][0]) return False else: LOG.error( 'f5-declarative-onboarding error %s - %s - waiting for error reason to populate', status_code, status) else: LOG.error( 'f5-declarative-onboarding error %s - No errors specified in results', status_code) return False elif status == "ROLLING_BACK": LOG.error( 'f5-declarative-onboarding rolling back with status %s - %s', status_code, status) return False time.sleep(1) except Exception as excpt: LOG.error('f5-declarative-onboarding threw an execption: %s', excpt) time.sleep(1) LOG.error( 'f5-declarative-onboarding did not successfully complete after %s', DO_TIMEOUT) return False
def wait_for_do_to_complete(): """Blocks until the f5-declarative-onboarding is complete""" LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.debug('waiting for iControl LX to become available') tmos_onboard_utils.wait_for_icontrollx() LOG.debug('starting f5-declarative-onboarding watcher') end_time = time.time() + DO_TIMEOUT restart_retry = time.time() + ONBOARD_TIMEOUT while (end_time - time.time()) > 0: try: response = tmos_onboard_utils.get_do_declaration() if hasattr(response, 'status_code'): if response.status_code == 200: LOG.info('f5-declarative-onboarding shows complete') return True if response.status_code > 399: json_resp = response.json() LOG.debug('f5-declarative-onboarding error %s - %s', response.status_code, json_resp) if 'status' in json_resp and 'declaration' in json_resp: return False if (restart_retry - time.time()) < 0: restart_retry = time.time() + ONBOARD_TIMEOUT tmos_onboard_utils.bigstart_restart() icontrol_available = False for _ in range(5): if tmos_onboard_utils.wait_for_icontrollx(timeout=30): icontrol_available = True break time.sleep(1) if icontrol_available: LOG.debug( 'waiting for f5-declarative-onboarding to become available' ) do_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/declarative-onboarding', timeout=30): do_available = True break time.sleep(1) if do_available: LOG.error( 'f5-declarative-onboarding retry post after restart services' ) tmos_onboard_utils.do_declare() time.sleep(10) time.sleep(1) except Exception as excpt: seconds_to_restart = restart_retry - time.time() if seconds_to_restart > 0: LOG.error( 'f5-declarative-onboarding threw an execption: %s service restarts in %d seconds', excpt, seconds_to_restart) else: LOG.error( 'f5-declarative-onboarding threw an execption: %s force restarting services', excpt, seconds_to_restart) restart_retry = time.time() + ONBOARD_TIMEOUT tmos_onboard_utils.bigstart_restart() icontrol_available = False for _ in range(5): if tmos_onboard_utils.wait_for_icontrollx(timeout=30): icontrol_available = True break time.sleep(1) if icontrol_available: LOG.debug( 'waiting for f5-declarative-onboarding to become available' ) do_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/declarative-onboarding', timeout=30): do_available = True break time.sleep(1) if do_available: LOG.error( 'f5-declarative-onboarding retry post after restart services' ) tmos_onboard_utils.do_declare() time.sleep(10) LOG.error( 'f5-declarative-onboarding did not successfully complete after %s', DO_TIMEOUT) return False
def handle(name, cloud_config, cloud, log, args): """Cloud-init processing function""" onboard_status = SUCCESS try: if wait_for_do_to_complete(): # reset restnoded to over come AS3 issue # https://github.com/F5Networks/f5-appsvcs-extension/issues/108 tmos_onboard_utils.bigstart_restart('restnoded') tmos_onboard_utils.wait_for_icontrollx() if tmos_onboard_utils.ts_declaration_persisted() and ( not onboard_status == ERROR): ts_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/telemetry/declare', timeout=30): ts_available = True break time.sleep(1) if ts_available: # check if there was a previous declaration.. if so don't issue response = tmos_onboard_utils.get_ts_declaration() # we have to check the format of the reponse because AS3 # does not assure the response is valid JSON if hasattr(response, 'status_code'): if response.status_code == 204 or response.status_code == 200: LOG.info( 'making initial f5-telemetry-streaming declaration' ) ts_declared = tmos_onboard_utils.ts_declare() if not ts_declared: LOG.error( 'f5-telemetry-streaming inital declaration failed' ) onboard_status = ERROR else: LOG.info( 'f5-telemetry-streaming initial declaration submitted successfully' ) else: LOG.warn( 'f5-telemetry-streaming has a previously errored declaration, not deploying' ) onboard_status = ERROR else: LOG.error( 'f5-telemetry-streaming has failed, not deploying initial declaration declaration %s - %s', response.status_code, response.text) onboard_status = ERROR else: LOG.error( 'f5-telemetry-streaming never became available.. timeout...' ) LOG.error( 'check /var/log/restnoded/restnoded.log for errors') onboard_status = ERROR if tmos_onboard_utils.as3_declaration_persisted() and ( not onboard_status == ERROR): as3_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/appsvcs/declare', timeout=30): as3_available = True break time.sleep(1) if as3_available: # check if there was a previous declaration.. if so don't issue response = tmos_onboard_utils.get_as3_declaration() if hasattr(response, 'status_code'): # we have to check the format of the reponse because AS3 # does not assure the response is valid JSON if response.status_code == 204 or response.status_code == 200: LOG.info( 'making initial f5-appsvcs-extension declaration' ) as3_declared = tmos_onboard_utils.as3_declare() if not as3_declared: LOG.error( 'f5-appsvcs-extension inital declaration failed' ) onboard_status = ERROR else: LOG.info( 'f5-appsvcs-extension initial declaration submitted successfully' ) else: LOG.warn( 'f5-appsvcs-extension has a previously errored declaration, not deploying' ) onboard_status = ERROR else: LOG.error( 'f5-appsvcs-extension has failed, not deploying initial declaration declaration %s - %s', response.status_code, response.text) onboard_status = ERROR else: LOG.error( 'f5-appsvcs-extension never became available.. timeout...' ) LOG.error( 'check /var/log/restnoded/restnoded.log for errors') onboard_status = ERROR else: LOG.error( 'f5-declarative-onboarding has failed, not deploying f5-appsvcs-extension initial declaration' ) onboard_status = ERROR except Exception as err: LOG.error( 'onboard exception prior to f5-appsvcs-extension declaration - %s', err) if os.path.exists(CLOUD_CONFIG_FILE): # read config phone_home_url = None if 'phone_home_url' in cloud_config[name]: LOG.info('phone_home_url from module %s found', name) phone_home_url = cloud_config[name]['phone_home_url'] phone_home_url_verify_tls = True if 'phone_home_url_verify_tls' in cloud_config[name]: phone_home_url_verify_tls = cloud_config[name][ 'phone_home_url_verify_tls'] phone_home_url_metadata = {} if 'phone_home_url_metadata' in cloud_config[name]: phone_home_url_metadata = cloud_config[name][ 'phone_home_url_metadata'] phone_home_cli = None if 'phone_home_cli' in cloud_config[name]: phone_home_cli = cloud_config[name]['phone_home_cli'] post_onboard_enabled = False if 'post_onboard_enabled' in cloud_config[name]: post_onboard_enabled = bool( cloud_config[name]['post_onboard_enabled']) if post_onboard_enabled and onboard_status == SUCCESS: if not runs_post_onboard_commands(name): onboard_status = ERROR # call phone_home if phone_home_url: tmos_onboard_utils.phone_home( phone_home_url, True, tmos_onboard_utils.ts_declaration_persisted(), tmos_onboard_utils.as3_declaration_persisted(), onboard_status, phone_home_url_verify_tls, phone_home_url_metadata) if phone_home_cli and onboard_status == SUCCESS: tmos_onboard_utils.run_cmd(phone_home_cli) clean() LOG.info('onboarding ended with status: %s', onboard_status)
def onboard(do_declaration, as3_declaration, ts_declaration, trusted_sources, post_onboard_enabled, post_onboard_commands, phone_home_url, phone_home_url_verify_tls, phone_home_url_metadata, phone_home_cli): """Implements the onboarding business logic""" do_enabled = True if do_declaration: try: tmos_onboard_utils.persist_do_declaration(do_declaration, None) except Exception as err: LOG.error( 'could not persist f5-declarative-onboarding declaration: %s', err) do_enabled = False else: do_enabled = False ts_enabled = True if ts_declaration: try: tmos_onboard_utils.persist_ts_declaration(ts_declaration) except Exception as err: LOG.error( 'could not persist f5-telemetry-streaming declaration %s', err) ts_enabled = False else: as3_enabled = False as3_enabled = True if as3_declaration: try: tmos_onboard_utils.persist_as3_declaration(as3_declaration) except Exception as err: LOG.error('could not persist f5-appsvcs-extension declaration %s', err) as3_enabled = False else: as3_enabled = False if post_onboard_enabled: tmsh_cmd_dir_exists() create_post_onboard_script(post_onboard_commands) create_post_onboard() if not tmos_onboard_utils.is_mgmt_ip: LOG.debug('waiting on mgmt interface provisioning to complete') tmos_onboard_utils.wait_for_mgmt_dhcp(timeout=600) LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.info('installing discovered iControl LX extensions') tmos_onboard_utils.install_extensions(trusted_sources) if do_enabled or ts_enabled or as3_enabled: icontrol_available = False for _ in range(5): if tmos_onboard_utils.wait_for_icontrollx(timeout=30): icontrol_available = True break time.sleep(1) if not icontrol_available: LOG.error('iControl LX never became available.. timeout...') LOG.error('check /var/log/restnoded/restnoded.log for errors') LOG.error('disabling iControl LX declarations..') do_enabled = False as3_enabled = False LOG.debug('waiting for iControl LX interfaces to be available') tmos_onboard_utils.wait_for_icontrollx() do_posted = False if do_enabled: do_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/declarative-onboarding', timeout=30): do_available = True break time.sleep(1) if do_available: if tmos_onboard_utils.do_declare(): do_posted = True else: LOG.error( 'f5-declarative-onboarding never became available.. timeout...' ) LOG.error('check /var/log/restnoded/restnoded.log for errors') onboard_status = SUCCESS if do_enabled and not do_posted: # DO enabled, initial post failed (bad declaration syntax) # fail the phone_home_url onboard_status = ERROR elif do_posted and (ts_enabled or as3_enabled): # DO enabled, posted async, need to background AS3 - phone home deferred to background AS3 process run_declarations_deffered() return elif do_posted and not (ts_enabled or as3_enabled): run_do_wait() return elif ts_enabled or as3_enabled: # DO not enabled, run AS3 declaration now # reset restnoded to over come AS3 issue # https://github.com/F5Networks/f5-appsvcs-extension/issues/108 tmos_onboard_utils.bigstart_restart('restnoded') tmos_onboard_utils.wait_for_icontrollx() if ts_enabled and (not onboard_status == ERROR): ts_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/telemetry/declare', timeout=30): ts_available = True break time.sleep(1) if ts_available: # check if there was a previous declaration.. if so don't issue response = tmos_onboard_utils.get_ts_declaration() # we have to check the format of the reponse because AS3 # does not assure the response is valid JSON if hasattr(response, 'status_code'): if response.status_code == 204 or response.status_code == 200: LOG.info( 'making initial f5-telemetry-streaming declaration' ) ts_declared = tmos_onboard_utils.ts_declare() if not ts_declared: LOG.error( 'f5-telemetry-streaming inital declaration failed' ) onboard_status = ERROR else: LOG.info( 'f5-telemetry-streaming initial declaration submitted successfully' ) if post_onboard_enabled: if not run_post_onboard_commands: onboard_status = ERROR else: LOG.warn( 'f5-telemetry-streaming has a previously errored declaration, not deploying' ) onboard_status = ERROR else: LOG.error( 'f5-telemetry-streaming has failed, not deploying initial declaration declaration %s - %s', response.status_code, response.text) onboard_status = ERROR else: LOG.error( 'f5-telemetry-streaming never became available.. timeout...' ) LOG.error('check /var/log/restnoded/restnoded.log for errors') onboard_status = ERROR if as3_enabled and (not onboard_status == ERROR): # this logic was added because AS3 performs hidden tasks # after it is started that can only be assured when # AS3 endpoint returns a response as3_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/appsvcs/declare', timeout=30): as3_available = True break time.sleep(1) if as3_available: # check if there was a previous declaration.. if so don't issue response = tmos_onboard_utils.get_as3_declaration() # we have to check the format of the reponse because AS3 # does not assure the response is valid JSON if hasattr(response, 'status_code'): if response.status_code == 204 or response.status_code == 200: LOG.info( 'making initial f5-appsvcs-extension declaration') as3_declared = tmos_onboard_utils.as3_declare() if not as3_declared: LOG.error( 'f5-appsvcs-extension inital declaration failed' ) onboard_status = ERROR else: LOG.info( 'f5-appsvcs-extension initial declaration submitted successfully' ) if post_onboard_enabled: if not run_post_onboard_commands: onboard_status = ERROR else: LOG.warn( 'f5-appsvcs-extension has a previously errored declaration, not deploying' ) onboard_status = ERROR else: LOG.error( 'f5-appsvcs-extension has failed, not deploying initial declaration declaration %s - %s', response.status_code, response.text) onboard_status = ERROR else: LOG.error( 'f5-appsvcs-extension never became available.. timeout...') LOG.error('check /var/log/restnoded/restnoded.log for errors') onboard_status = ERROR else: # neither DO or AS3 enabled if post_onboard_enabled: if not run_post_onboard_commands(): onboard_status = ERROR if phone_home_url: tmos_onboard_utils.phone_home(phone_home_url, do_enabled, ts_enabled, as3_enabled, onboard_status, phone_home_url_verify_tls, phone_home_url_metadata) if phone_home_cli and onboard_status == SUCCESS: tmos_onboard_utils.run_cmd(phone_home_cli) LOG.info('onboarding ended with status: %s', onboard_status)