def wait_for_do_to_complete(): """Blocks until the f5-declarative-onboarding is complete""" LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.debug('waiting for iControl LX to become available') tmos_onboard_utils.wait_for_icontrollx() LOG.debug('starting f5-declarative-onboarding watcher') end_time = time.time() + DO_TIMEOUT while (end_time - time.time()) > 0: try: response = tmos_onboard_utils.get_do_declaration() if hasattr(response, 'status_code'): if response.status_code == 200: LOG.info('f5-declarative-onboarding shows complete') return True if response.status_code > 399: json_resp = response.json() LOG.debug('f5-declarative-onboarding error %s - %s', response.status_code, json_resp) if 'status' in json_resp and 'declaration' in json_resp: return False time.sleep(1) except Exception as excpt: LOG.error('f5-declarative-onboarding threw an execption: %s', excpt) time.sleep(1) LOG.error( 'f5-declarative-onboarding did not successfully complete after %s', DO_TIMEOUT) return False
def wait_for_do_to_complete(): """Blocks until the f5-declarative-onboarding is complete""" LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.debug('waiting for iControl LX to become available') tmos_onboard_utils.wait_for_icontrollx() LOG.debug('starting f5-declarative-onboarding watcher') restarted_services = False end_time = time.time() + DO_TIMEOUT while (end_time - time.time()) > 0: try: response = tmos_onboard_utils.get_do_status_codes() if hasattr(response, 'status_code'): if response.status_code == 200: json_resp = response.json() status_code = json_resp['result']['code'] status = json_resp['result']['status'] if status_code == 200 and status == "OK": LOG.info('f5-declarative-onboarding shows complete') return True elif status_code == 503 and status == "ERROR": LOG.error('system not responsive, aborting onboarding') return False else: if status_code > 399: if 'errors' in json_resp['result']: errors = json_resp['result'] if len(errors) > 0: LOG.error( 'f5-declarative-onboarding error %s - %s', status_code, json_resp['errors'][0]) return False else: LOG.error( 'f5-declarative-onboarding error %s - %s - waiting for error reason to populate', status_code, status) else: LOG.error( 'f5-declarative-onboarding error %s - No errors specified in results', status_code) return False elif status == "ROLLING_BACK": LOG.error( 'f5-declarative-onboarding rolling back with status %s - %s', status_code, status) return False time.sleep(1) except Exception as excpt: LOG.error('f5-declarative-onboarding threw an execption: %s', excpt) time.sleep(1) LOG.error( 'f5-declarative-onboarding did not successfully complete after %s', DO_TIMEOUT) return False
def handle(name, cloud_config, cloud, log, args): """Cloud-init processing function""" tag = MODULE_NAME # force /PLATFORM to KVM generic with open('/PLATFORM', 'w') as platform: platform.write( "platform=Z100\nfamily=0xC0000000\nhost=Z100\nsystype=0x71\n") # force more memory for control plane sevices to complete tmos_onboard_utils.run_cmd("/usr/bin/setdb provision.extramb 500") tmos_onboard_utils.run_cmd("/usr/bin/setdb restjavad.useextramb true") # find SSH key in vendor_data file if os.path.exists(VENDOR_DATA_RAW_FILE): LOG.debug('attempting to extract SSH key from vendor_data') public_keys = [] with open(VENDOR_DATA_RAW_FILE, 'r') as vdf: for line in vdf: if 'ssh-rsa' in line: public_keys.append(line[line.index('ssh-rsa'):]) if public_keys: LOG.debug('injecting %d SSH authorized keys for root' % len(public_keys)) tmos_onboard_utils.inject_public_ssh_keys(public_keys) # randomize root and admin accounts LOG.debug('randomizing default account passwords') root_password = '******' + ''.join([ random.choice(string.ascii_letters + string.digits) for n in range(32) ]) + '\n' try: util.subp(['chpasswd'], root_password) except Exception as e: LOG.error('failed to randomize password for root user: %s', e) admin_password = '******' + ''.join([ random.choice(string.ascii_letters + string.digits) for n in range(32) ]) + '\n' try: util.subp(['chpasswd'], admin_password) except Exception as e: LOG.error('failed to randomize password for admin user: %s', e) try: # forcing hostname from metadata file if "local-hostname" in cloud.datasource.metadata: LOG.debug('forcing localhost name to %s' % cloud.datasource.metadata['local-hostname']) tmos_onboard_utils.wait_for_mcpd() tmos_onboard_utils.run_cmd( "tmsh modify sys global-settings hostname %s.local" % cloud.datasource.metadata['local-hostname']) except Exception as ex: LOG.debug('exception setting hostname from metadata: %s', ex)
def wait_for_do_to_complete(): """Blocks until the f5-declarative-onboarding is complete""" LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.debug('waiting for iControl LX to become available') tmos_onboard_utils.wait_for_icontrollx() LOG.debug('starting f5-declarative-onboarding watcher') end_time = time.time() + DO_TIMEOUT restart_retry = time.time() + ONBOARD_TIMEOUT while (end_time - time.time()) > 0: try: response = tmos_onboard_utils.get_do_declaration() if hasattr(response, 'status_code'): if response.status_code == 200: LOG.info('f5-declarative-onboarding shows complete') return True if response.status_code > 399: json_resp = response.json() LOG.debug('f5-declarative-onboarding error %s - %s', response.status_code, json_resp) if 'status' in json_resp and 'declaration' in json_resp: return False if (restart_retry - time.time()) < 0: restart_retry = time.time() + ONBOARD_TIMEOUT tmos_onboard_utils.bigstart_restart() icontrol_available = False for _ in range(5): if tmos_onboard_utils.wait_for_icontrollx(timeout=30): icontrol_available = True break time.sleep(1) if icontrol_available: LOG.debug( 'waiting for f5-declarative-onboarding to become available' ) do_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/declarative-onboarding', timeout=30): do_available = True break time.sleep(1) if do_available: LOG.error( 'f5-declarative-onboarding retry post after restart services' ) tmos_onboard_utils.do_declare() time.sleep(10) time.sleep(1) except Exception as excpt: seconds_to_restart = restart_retry - time.time() if seconds_to_restart > 0: LOG.error( 'f5-declarative-onboarding threw an execption: %s service restarts in %d seconds', excpt, seconds_to_restart) else: LOG.error( 'f5-declarative-onboarding threw an execption: %s force restarting services', excpt, seconds_to_restart) restart_retry = time.time() + ONBOARD_TIMEOUT tmos_onboard_utils.bigstart_restart() icontrol_available = False for _ in range(5): if tmos_onboard_utils.wait_for_icontrollx(timeout=30): icontrol_available = True break time.sleep(1) if icontrol_available: LOG.debug( 'waiting for f5-declarative-onboarding to become available' ) do_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/declarative-onboarding', timeout=30): do_available = True break time.sleep(1) if do_available: LOG.error( 'f5-declarative-onboarding retry post after restart services' ) tmos_onboard_utils.do_declare() time.sleep(10) LOG.error( 'f5-declarative-onboarding did not successfully complete after %s', DO_TIMEOUT) return False
def onboard(do_declaration, as3_declaration, ts_declaration, trusted_sources, post_onboard_enabled, post_onboard_commands, phone_home_url, phone_home_url_verify_tls, phone_home_url_metadata, phone_home_cli): """Implements the onboarding business logic""" do_enabled = True if do_declaration: try: tmos_onboard_utils.persist_do_declaration(do_declaration, None) except Exception as err: LOG.error( 'could not persist f5-declarative-onboarding declaration: %s', err) do_enabled = False else: do_enabled = False ts_enabled = True if ts_declaration: try: tmos_onboard_utils.persist_ts_declaration(ts_declaration) except Exception as err: LOG.error( 'could not persist f5-telemetry-streaming declaration %s', err) ts_enabled = False else: as3_enabled = False as3_enabled = True if as3_declaration: try: tmos_onboard_utils.persist_as3_declaration(as3_declaration) except Exception as err: LOG.error('could not persist f5-appsvcs-extension declaration %s', err) as3_enabled = False else: as3_enabled = False if post_onboard_enabled: tmsh_cmd_dir_exists() create_post_onboard_script(post_onboard_commands) create_post_onboard() if not tmos_onboard_utils.is_mgmt_ip: LOG.debug('waiting on mgmt interface provisioning to complete') tmos_onboard_utils.wait_for_mgmt_dhcp(timeout=600) LOG.debug('waiting for mcpd to become available') tmos_onboard_utils.wait_for_mcpd() LOG.debug('waiting for iControl to become available') tmos_onboard_utils.wait_for_icontrol() LOG.info('installing discovered iControl LX extensions') tmos_onboard_utils.install_extensions(trusted_sources) if do_enabled or ts_enabled or as3_enabled: icontrol_available = False for _ in range(5): if tmos_onboard_utils.wait_for_icontrollx(timeout=30): icontrol_available = True break time.sleep(1) if not icontrol_available: LOG.error('iControl LX never became available.. timeout...') LOG.error('check /var/log/restnoded/restnoded.log for errors') LOG.error('disabling iControl LX declarations..') do_enabled = False as3_enabled = False LOG.debug('waiting for iControl LX interfaces to be available') tmos_onboard_utils.wait_for_icontrollx() do_posted = False if do_enabled: do_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/declarative-onboarding', timeout=30): do_available = True break time.sleep(1) if do_available: if tmos_onboard_utils.do_declare(): do_posted = True else: LOG.error( 'f5-declarative-onboarding never became available.. timeout...' ) LOG.error('check /var/log/restnoded/restnoded.log for errors') onboard_status = SUCCESS if do_enabled and not do_posted: # DO enabled, initial post failed (bad declaration syntax) # fail the phone_home_url onboard_status = ERROR elif do_posted and (ts_enabled or as3_enabled): # DO enabled, posted async, need to background AS3 - phone home deferred to background AS3 process run_declarations_deffered() return elif do_posted and not (ts_enabled or as3_enabled): run_do_wait() return elif ts_enabled or as3_enabled: # DO not enabled, run AS3 declaration now # reset restnoded to over come AS3 issue # https://github.com/F5Networks/f5-appsvcs-extension/issues/108 tmos_onboard_utils.bigstart_restart('restnoded') tmos_onboard_utils.wait_for_icontrollx() if ts_enabled and (not onboard_status == ERROR): ts_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/telemetry/declare', timeout=30): ts_available = True break time.sleep(1) if ts_available: # check if there was a previous declaration.. if so don't issue response = tmos_onboard_utils.get_ts_declaration() # we have to check the format of the reponse because AS3 # does not assure the response is valid JSON if hasattr(response, 'status_code'): if response.status_code == 204 or response.status_code == 200: LOG.info( 'making initial f5-telemetry-streaming declaration' ) ts_declared = tmos_onboard_utils.ts_declare() if not ts_declared: LOG.error( 'f5-telemetry-streaming inital declaration failed' ) onboard_status = ERROR else: LOG.info( 'f5-telemetry-streaming initial declaration submitted successfully' ) if post_onboard_enabled: if not run_post_onboard_commands: onboard_status = ERROR else: LOG.warn( 'f5-telemetry-streaming has a previously errored declaration, not deploying' ) onboard_status = ERROR else: LOG.error( 'f5-telemetry-streaming has failed, not deploying initial declaration declaration %s - %s', response.status_code, response.text) onboard_status = ERROR else: LOG.error( 'f5-telemetry-streaming never became available.. timeout...' ) LOG.error('check /var/log/restnoded/restnoded.log for errors') onboard_status = ERROR if as3_enabled and (not onboard_status == ERROR): # this logic was added because AS3 performs hidden tasks # after it is started that can only be assured when # AS3 endpoint returns a response as3_available = False for _ in range(5): if tmos_onboard_utils.wait_for_rest_worker( '/mgmt/shared/appsvcs/declare', timeout=30): as3_available = True break time.sleep(1) if as3_available: # check if there was a previous declaration.. if so don't issue response = tmos_onboard_utils.get_as3_declaration() # we have to check the format of the reponse because AS3 # does not assure the response is valid JSON if hasattr(response, 'status_code'): if response.status_code == 204 or response.status_code == 200: LOG.info( 'making initial f5-appsvcs-extension declaration') as3_declared = tmos_onboard_utils.as3_declare() if not as3_declared: LOG.error( 'f5-appsvcs-extension inital declaration failed' ) onboard_status = ERROR else: LOG.info( 'f5-appsvcs-extension initial declaration submitted successfully' ) if post_onboard_enabled: if not run_post_onboard_commands: onboard_status = ERROR else: LOG.warn( 'f5-appsvcs-extension has a previously errored declaration, not deploying' ) onboard_status = ERROR else: LOG.error( 'f5-appsvcs-extension has failed, not deploying initial declaration declaration %s - %s', response.status_code, response.text) onboard_status = ERROR else: LOG.error( 'f5-appsvcs-extension never became available.. timeout...') LOG.error('check /var/log/restnoded/restnoded.log for errors') onboard_status = ERROR else: # neither DO or AS3 enabled if post_onboard_enabled: if not run_post_onboard_commands(): onboard_status = ERROR if phone_home_url: tmos_onboard_utils.phone_home(phone_home_url, do_enabled, ts_enabled, as3_enabled, onboard_status, phone_home_url_verify_tls, phone_home_url_metadata) if phone_home_cli and onboard_status == SUCCESS: tmos_onboard_utils.run_cmd(phone_home_cli) LOG.info('onboarding ended with status: %s', onboard_status)