def wait_for_do_to_complete():
"""Blocks until the f5-declarative-onboarding is complete"""
LOG.debug('waiting for mcpd to become available')
tmos_onboard_utils.wait_for_mcpd()
LOG.debug('waiting for iControl to become available')
tmos_onboard_utils.wait_for_icontrol()
LOG.debug('waiting for iControl LX to become available')
tmos_onboard_utils.wait_for_icontrollx()
LOG.debug('starting f5-declarative-onboarding watcher')
end_time = time.time() + DO_TIMEOUT
while (end_time - time.time()) > 0:
try:
response = tmos_onboard_utils.get_do_declaration()
if hasattr(response, 'status_code'):
if response.status_code == 200:
LOG.info('f5-declarative-onboarding shows complete')
return True
if response.status_code > 399:
json_resp = response.json()
LOG.debug('f5-declarative-onboarding error %s - %s',
response.status_code, json_resp)
if 'status' in json_resp and 'declaration' in json_resp:
return False
time.sleep(1)
except Exception as excpt:
LOG.error('f5-declarative-onboarding threw an execption: %s', excpt)
time.sleep(1)
LOG.error(
'f5-declarative-onboarding did not successfully complete after %s', DO_TIMEOUT)
return False
def wait_for_do_to_complete():
"""Blocks until the f5-declarative-onboarding is complete"""
LOG.debug('waiting for mcpd to become available')
tmos_onboard_utils.wait_for_mcpd()
LOG.debug('waiting for iControl to become available')
tmos_onboard_utils.wait_for_icontrol()
LOG.debug('waiting for iControl LX to become available')
tmos_onboard_utils.wait_for_icontrollx()
LOG.debug('starting f5-declarative-onboarding watcher')
restarted_services = False
end_time = time.time() + DO_TIMEOUT
while (end_time - time.time()) > 0:
try:
response = tmos_onboard_utils.get_do_status_codes()
if hasattr(response, 'status_code'):
if response.status_code == 200:
json_resp = response.json()
status_code = json_resp['result']['code']
status = json_resp['result']['status']
if status_code == 200 and status == "OK":
LOG.info('f5-declarative-onboarding shows complete')
return True
elif status_code == 503 and status == "ERROR":
LOG.error('system not responsive, aborting onboarding')
return False
else:
if status_code > 399:
if 'errors' in json_resp['result']:
errors = json_resp['result']
if len(errors) > 0:
LOG.error(
'f5-declarative-onboarding error %s - %s',
status_code, json_resp['errors'][0])
return False
else:
LOG.error(
'f5-declarative-onboarding error %s - %s - waiting for error reason to populate',
status_code, status)
else:
LOG.error(
'f5-declarative-onboarding error %s - No errors specified in results',
status_code)
return False
elif status == "ROLLING_BACK":
LOG.error(
'f5-declarative-onboarding rolling back with status %s - %s',
status_code, status)
return False
time.sleep(1)
except Exception as excpt:
LOG.error('f5-declarative-onboarding threw an execption: %s',
excpt)
time.sleep(1)
LOG.error(
'f5-declarative-onboarding did not successfully complete after %s',
DO_TIMEOUT)
return False
def wait_for_do_to_complete():
"""Blocks until the f5-declarative-onboarding is complete"""
LOG.debug('waiting for mcpd to become available')
tmos_onboard_utils.wait_for_mcpd()
LOG.debug('waiting for iControl to become available')
tmos_onboard_utils.wait_for_icontrol()
LOG.debug('waiting for iControl LX to become available')
tmos_onboard_utils.wait_for_icontrollx()
LOG.debug('starting f5-declarative-onboarding watcher')
end_time = time.time() + DO_TIMEOUT
restart_retry = time.time() + ONBOARD_TIMEOUT
while (end_time - time.time()) > 0:
try:
response = tmos_onboard_utils.get_do_declaration()
if hasattr(response, 'status_code'):
if response.status_code == 200:
LOG.info('f5-declarative-onboarding shows complete')
return True
if response.status_code > 399:
json_resp = response.json()
LOG.debug('f5-declarative-onboarding error %s - %s',
response.status_code, json_resp)
if 'status' in json_resp and 'declaration' in json_resp:
return False
if (restart_retry - time.time()) < 0:
restart_retry = time.time() + ONBOARD_TIMEOUT
tmos_onboard_utils.bigstart_restart()
icontrol_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_icontrollx(timeout=30):
icontrol_available = True
break
time.sleep(1)
if icontrol_available:
LOG.debug(
'waiting for f5-declarative-onboarding to become available'
)
do_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_rest_worker(
'/mgmt/shared/declarative-onboarding',
timeout=30):
do_available = True
break
time.sleep(1)
if do_available:
LOG.error(
'f5-declarative-onboarding retry post after restart services'
)
tmos_onboard_utils.do_declare()
time.sleep(10)
time.sleep(1)
except Exception as excpt:
seconds_to_restart = restart_retry - time.time()
if seconds_to_restart > 0:
LOG.error(
'f5-declarative-onboarding threw an execption: %s service restarts in %d seconds',
excpt, seconds_to_restart)
else:
LOG.error(
'f5-declarative-onboarding threw an execption: %s force restarting services',
excpt, seconds_to_restart)
restart_retry = time.time() + ONBOARD_TIMEOUT
tmos_onboard_utils.bigstart_restart()
icontrol_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_icontrollx(timeout=30):
icontrol_available = True
break
time.sleep(1)
if icontrol_available:
LOG.debug(
'waiting for f5-declarative-onboarding to become available'
)
do_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_rest_worker(
'/mgmt/shared/declarative-onboarding',
timeout=30):
do_available = True
break
time.sleep(1)
if do_available:
LOG.error(
'f5-declarative-onboarding retry post after restart services'
)
tmos_onboard_utils.do_declare()
time.sleep(10)
LOG.error(
'f5-declarative-onboarding did not successfully complete after %s',
DO_TIMEOUT)
return False
def handle(name, cloud_config, cloud, log, args):
"""Cloud-init processing function"""
onboard_status = SUCCESS
try:
if wait_for_do_to_complete():
# reset restnoded to over come AS3 issue
# https://github.com/F5Networks/f5-appsvcs-extension/issues/108
tmos_onboard_utils.bigstart_restart('restnoded')
tmos_onboard_utils.wait_for_icontrollx()
if tmos_onboard_utils.ts_declaration_persisted() and (
not onboard_status == ERROR):
ts_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_rest_worker(
'/mgmt/shared/telemetry/declare', timeout=30):
ts_available = True
break
time.sleep(1)
if ts_available:
# check if there was a previous declaration.. if so don't issue
response = tmos_onboard_utils.get_ts_declaration()
# we have to check the format of the reponse because AS3
# does not assure the response is valid JSON
if hasattr(response, 'status_code'):
if response.status_code == 204 or response.status_code == 200:
LOG.info(
'making initial f5-telemetry-streaming declaration'
)
ts_declared = tmos_onboard_utils.ts_declare()
if not ts_declared:
LOG.error(
'f5-telemetry-streaming inital declaration failed'
)
onboard_status = ERROR
else:
LOG.info(
'f5-telemetry-streaming initial declaration submitted successfully'
)
else:
LOG.warn(
'f5-telemetry-streaming has a previously errored declaration, not deploying'
)
onboard_status = ERROR
else:
LOG.error(
'f5-telemetry-streaming has failed, not deploying initial declaration declaration %s - %s',
response.status_code, response.text)
onboard_status = ERROR
else:
LOG.error(
'f5-telemetry-streaming never became available.. timeout...'
)
LOG.error(
'check /var/log/restnoded/restnoded.log for errors')
onboard_status = ERROR
if tmos_onboard_utils.as3_declaration_persisted() and (
not onboard_status == ERROR):
as3_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_rest_worker(
'/mgmt/shared/appsvcs/declare', timeout=30):
as3_available = True
break
time.sleep(1)
if as3_available:
# check if there was a previous declaration.. if so don't issue
response = tmos_onboard_utils.get_as3_declaration()
if hasattr(response, 'status_code'):
# we have to check the format of the reponse because AS3
# does not assure the response is valid JSON
if response.status_code == 204 or response.status_code == 200:
LOG.info(
'making initial f5-appsvcs-extension declaration'
)
as3_declared = tmos_onboard_utils.as3_declare()
if not as3_declared:
LOG.error(
'f5-appsvcs-extension inital declaration failed'
)
onboard_status = ERROR
else:
LOG.info(
'f5-appsvcs-extension initial declaration submitted successfully'
)
else:
LOG.warn(
'f5-appsvcs-extension has a previously errored declaration, not deploying'
)
onboard_status = ERROR
else:
LOG.error(
'f5-appsvcs-extension has failed, not deploying initial declaration declaration %s - %s',
response.status_code, response.text)
onboard_status = ERROR
else:
LOG.error(
'f5-appsvcs-extension never became available.. timeout...'
)
LOG.error(
'check /var/log/restnoded/restnoded.log for errors')
onboard_status = ERROR
else:
LOG.error(
'f5-declarative-onboarding has failed, not deploying f5-appsvcs-extension initial declaration'
)
onboard_status = ERROR
except Exception as err:
LOG.error(
'onboard exception prior to f5-appsvcs-extension declaration - %s',
err)
if os.path.exists(CLOUD_CONFIG_FILE):
# read config
phone_home_url = None
if 'phone_home_url' in cloud_config[name]:
LOG.info('phone_home_url from module %s found', name)
phone_home_url = cloud_config[name]['phone_home_url']
phone_home_url_verify_tls = True
if 'phone_home_url_verify_tls' in cloud_config[name]:
phone_home_url_verify_tls = cloud_config[name][
'phone_home_url_verify_tls']
phone_home_url_metadata = {}
if 'phone_home_url_metadata' in cloud_config[name]:
phone_home_url_metadata = cloud_config[name][
'phone_home_url_metadata']
phone_home_cli = None
if 'phone_home_cli' in cloud_config[name]:
phone_home_cli = cloud_config[name]['phone_home_cli']
post_onboard_enabled = False
if 'post_onboard_enabled' in cloud_config[name]:
post_onboard_enabled = bool(
cloud_config[name]['post_onboard_enabled'])
if post_onboard_enabled and onboard_status == SUCCESS:
if not runs_post_onboard_commands(name):
onboard_status = ERROR
# call phone_home
if phone_home_url:
tmos_onboard_utils.phone_home(
phone_home_url, True,
tmos_onboard_utils.ts_declaration_persisted(),
tmos_onboard_utils.as3_declaration_persisted(), onboard_status,
phone_home_url_verify_tls, phone_home_url_metadata)
if phone_home_cli and onboard_status == SUCCESS:
tmos_onboard_utils.run_cmd(phone_home_cli)
clean()
LOG.info('onboarding ended with status: %s', onboard_status)
def onboard(do_declaration, as3_declaration, ts_declaration, trusted_sources,
post_onboard_enabled, post_onboard_commands, phone_home_url,
phone_home_url_verify_tls, phone_home_url_metadata,
phone_home_cli):
"""Implements the onboarding business logic"""
do_enabled = True
if do_declaration:
try:
tmos_onboard_utils.persist_do_declaration(do_declaration, None)
except Exception as err:
LOG.error(
'could not persist f5-declarative-onboarding declaration: %s',
err)
do_enabled = False
else:
do_enabled = False
ts_enabled = True
if ts_declaration:
try:
tmos_onboard_utils.persist_ts_declaration(ts_declaration)
except Exception as err:
LOG.error(
'could not persist f5-telemetry-streaming declaration %s', err)
ts_enabled = False
else:
as3_enabled = False
as3_enabled = True
if as3_declaration:
try:
tmos_onboard_utils.persist_as3_declaration(as3_declaration)
except Exception as err:
LOG.error('could not persist f5-appsvcs-extension declaration %s',
err)
as3_enabled = False
else:
as3_enabled = False
if post_onboard_enabled:
tmsh_cmd_dir_exists()
create_post_onboard_script(post_onboard_commands)
create_post_onboard()
if not tmos_onboard_utils.is_mgmt_ip:
LOG.debug('waiting on mgmt interface provisioning to complete')
tmos_onboard_utils.wait_for_mgmt_dhcp(timeout=600)
LOG.debug('waiting for mcpd to become available')
tmos_onboard_utils.wait_for_mcpd()
LOG.debug('waiting for iControl to become available')
tmos_onboard_utils.wait_for_icontrol()
LOG.info('installing discovered iControl LX extensions')
tmos_onboard_utils.install_extensions(trusted_sources)
if do_enabled or ts_enabled or as3_enabled:
icontrol_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_icontrollx(timeout=30):
icontrol_available = True
break
time.sleep(1)
if not icontrol_available:
LOG.error('iControl LX never became available.. timeout...')
LOG.error('check /var/log/restnoded/restnoded.log for errors')
LOG.error('disabling iControl LX declarations..')
do_enabled = False
as3_enabled = False
LOG.debug('waiting for iControl LX interfaces to be available')
tmos_onboard_utils.wait_for_icontrollx()
do_posted = False
if do_enabled:
do_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_rest_worker(
'/mgmt/shared/declarative-onboarding', timeout=30):
do_available = True
break
time.sleep(1)
if do_available:
if tmos_onboard_utils.do_declare():
do_posted = True
else:
LOG.error(
'f5-declarative-onboarding never became available.. timeout...'
)
LOG.error('check /var/log/restnoded/restnoded.log for errors')
onboard_status = SUCCESS
if do_enabled and not do_posted:
# DO enabled, initial post failed (bad declaration syntax)
# fail the phone_home_url
onboard_status = ERROR
elif do_posted and (ts_enabled or as3_enabled):
# DO enabled, posted async, need to background AS3 - phone home deferred to background AS3 process
run_declarations_deffered()
return
elif do_posted and not (ts_enabled or as3_enabled):
run_do_wait()
return
elif ts_enabled or as3_enabled:
# DO not enabled, run AS3 declaration now
# reset restnoded to over come AS3 issue
# https://github.com/F5Networks/f5-appsvcs-extension/issues/108
tmos_onboard_utils.bigstart_restart('restnoded')
tmos_onboard_utils.wait_for_icontrollx()
if ts_enabled and (not onboard_status == ERROR):
ts_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_rest_worker(
'/mgmt/shared/telemetry/declare', timeout=30):
ts_available = True
break
time.sleep(1)
if ts_available:
# check if there was a previous declaration.. if so don't issue
response = tmos_onboard_utils.get_ts_declaration()
# we have to check the format of the reponse because AS3
# does not assure the response is valid JSON
if hasattr(response, 'status_code'):
if response.status_code == 204 or response.status_code == 200:
LOG.info(
'making initial f5-telemetry-streaming declaration'
)
ts_declared = tmos_onboard_utils.ts_declare()
if not ts_declared:
LOG.error(
'f5-telemetry-streaming inital declaration failed'
)
onboard_status = ERROR
else:
LOG.info(
'f5-telemetry-streaming initial declaration submitted successfully'
)
if post_onboard_enabled:
if not run_post_onboard_commands:
onboard_status = ERROR
else:
LOG.warn(
'f5-telemetry-streaming has a previously errored declaration, not deploying'
)
onboard_status = ERROR
else:
LOG.error(
'f5-telemetry-streaming has failed, not deploying initial declaration declaration %s - %s',
response.status_code, response.text)
onboard_status = ERROR
else:
LOG.error(
'f5-telemetry-streaming never became available.. timeout...'
)
LOG.error('check /var/log/restnoded/restnoded.log for errors')
onboard_status = ERROR
if as3_enabled and (not onboard_status == ERROR):
# this logic was added because AS3 performs hidden tasks
# after it is started that can only be assured when
# AS3 endpoint returns a response
as3_available = False
for _ in range(5):
if tmos_onboard_utils.wait_for_rest_worker(
'/mgmt/shared/appsvcs/declare', timeout=30):
as3_available = True
break
time.sleep(1)
if as3_available:
# check if there was a previous declaration.. if so don't issue
response = tmos_onboard_utils.get_as3_declaration()
# we have to check the format of the reponse because AS3
# does not assure the response is valid JSON
if hasattr(response, 'status_code'):
if response.status_code == 204 or response.status_code == 200:
LOG.info(
'making initial f5-appsvcs-extension declaration')
as3_declared = tmos_onboard_utils.as3_declare()
if not as3_declared:
LOG.error(
'f5-appsvcs-extension inital declaration failed'
)
onboard_status = ERROR
else:
LOG.info(
'f5-appsvcs-extension initial declaration submitted successfully'
)
if post_onboard_enabled:
if not run_post_onboard_commands:
onboard_status = ERROR
else:
LOG.warn(
'f5-appsvcs-extension has a previously errored declaration, not deploying'
)
onboard_status = ERROR
else:
LOG.error(
'f5-appsvcs-extension has failed, not deploying initial declaration declaration %s - %s',
response.status_code, response.text)
onboard_status = ERROR
else:
LOG.error(
'f5-appsvcs-extension never became available.. timeout...')
LOG.error('check /var/log/restnoded/restnoded.log for errors')
onboard_status = ERROR
else:
# neither DO or AS3 enabled
if post_onboard_enabled:
if not run_post_onboard_commands():
onboard_status = ERROR
if phone_home_url:
tmos_onboard_utils.phone_home(phone_home_url, do_enabled, ts_enabled,
as3_enabled, onboard_status,
phone_home_url_verify_tls,
phone_home_url_metadata)
if phone_home_cli and onboard_status == SUCCESS:
tmos_onboard_utils.run_cmd(phone_home_cli)
LOG.info('onboarding ended with status: %s', onboard_status)