def _load_template_definition_from_catalog(
self, msg_update_callback=utils.NullPrinter()): # noqa: E501
msg = "Loading k8s template definition from catalog"
logger.SERVER_LOGGER.info(msg)
msg_update_callback.general_no_color(msg)
client = None
try:
log_filename = None
log_wire = \
utils.str_to_bool(self.config['service'].get('log_wire'))
if log_wire:
log_filename = logger.SERVER_DEBUG_WIRELOG_FILEPATH
client = Client(self.config['vcd']['host'],
api_version=self.config['vcd']['api_version'],
verify_ssl_certs=self.config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(self.config['vcd']['username'],
SYSTEM_ORG_NAME,
self.config['vcd']['password'])
client.set_credentials(credentials)
org_name = self.config['broker']['org']
catalog_name = self.config['broker']['catalog']
k8_templates = ltm.get_all_k8s_local_template_definition(
client=client, catalog_name=catalog_name, org_name=org_name)
if not k8_templates:
msg = "No valid K8 templates were found in catalog " \
f"'{catalog_name}'. Unable to start CSE server."
msg_update_callback.error(msg)
logger.SERVER_LOGGER.error(msg)
sys.exit(1)
# Check that default k8s template exists in vCD at the correct
# revision
default_template_name = \
self.config['broker']['default_template_name']
default_template_revision = \
str(self.config['broker']['default_template_revision'])
found_default_template = False
for template in k8_templates:
if str(template[LocalTemplateKey.REVISION]
) == default_template_revision and template[
LocalTemplateKey.
NAME] == default_template_name: # noqa: E501
found_default_template = True
msg = f"Found K8 template '{template['name']}' at revision " \
f"{template['revision']} in catalog '{catalog_name}'"
msg_update_callback.general(msg)
logger.SERVER_LOGGER.info(msg)
if not found_default_template:
msg = f"Default template {default_template_name} with " \
f"revision {default_template_revision} not found." \
" Unable to start CSE server."
msg_update_callback.error(msg)
logger.SERVER_LOGGER.error(msg)
sys.exit(1)
self.config['broker']['templates'] = k8_templates
finally:
if client:
client.logout()
def run(self, msg_update_callback=None):
configure_server_logger()
self.config = get_validated_config(
self.config_file, msg_update_callback=msg_update_callback)
populate_vsphere_list(self.config['vcs'])
# Read K8 catalog definition from catalog item metadata and append
# to server config
client = None
try:
client = Client(self.config['vcd']['host'],
api_version=self.config['vcd']['api_version'],
verify_ssl_certs=self.config['vcd']['verify'],
log_file=SERVER_DEBUG_WIRELOG_FILEPATH,
log_requests=True,
log_headers=True,
log_bodies=True)
credentials = BasicLoginCredentials(self.config['vcd']['username'],
SYSTEM_ORG_NAME,
self.config['vcd']['password'])
client.set_credentials(credentials)
org_name = self.config['broker']['org']
catalog_name = self.config['broker']['catalog']
k8_templates = get_all_k8s_local_template_definition(
client=client, catalog_name=catalog_name, org_name=org_name)
if not k8_templates:
msg = "No valid K8 templates were found in catalog " \
f"'{catalog_name}'. Unable to start CSE server."
if msg_update_callback:
msg_update_callback.error(msg)
LOGGER.error(msg)
sys.exit(1)
# Check that deafult K8 template exists in vCD at the correct
# revision
default_template_name = \
self.config['broker']['default_template_name']
default_template_revision = \
str(self.config['broker']['default_template_revision'])
found_default_template = False
for template in k8_templates:
if str(template['revision']) == default_template_revision \
and template['name'] == default_template_name:
found_default_template = True
msg = f"Found K8 template '{template['name']}' at revision " \
f"{template['revision']} in catalog '{catalog_name}'"
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
if not found_default_template:
msg = f"Default template {default_template_name} with " \
f"revision {default_template_revision} not found." \
" Unable to start CSE server."
if msg_update_callback:
msg_update_callback.error(msg)
LOGGER.error(msg)
sys.exit(1)
self.config['broker']['templates'] = k8_templates
finally:
if client:
client.logout()
# TODO Rule framework, update config with rules
if self.should_check_config:
check_cse_installation(self.config,
msg_update_callback=msg_update_callback)
if self.config.get('pks_config'):
pks_config = self.config.get('pks_config')
self.pks_cache = PksCache(
pks_servers=pks_config.get('pks_api_servers', []),
pks_accounts=pks_config.get('pks_accounts', []),
pvdcs=pks_config.get('pvdcs', []),
orgs=pks_config.get('orgs', []),
nsxt_servers=pks_config.get('nsxt_servers', []))
amqp = self.config['amqp']
num_consumers = self.config['service']['listeners']
for n in range(num_consumers):
try:
c = MessageConsumer(amqp['host'], amqp['port'], amqp['ssl'],
amqp['vhost'], amqp['username'],
amqp['password'], amqp['exchange'],
amqp['routing_key'])
name = 'MessageConsumer-%s' % n
t = Thread(name=name, target=consumer_thread, args=(c, ))
t.daemon = True
t.start()
msg = f"Started thread '{name} ({t.ident})'"
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
self.threads.append(t)
self.consumers.append(c)
time.sleep(0.25)
except KeyboardInterrupt:
break
except Exception:
LOGGER.error(traceback.format_exc())
LOGGER.info(f"Number of threads started: {len(self.threads)}")
self._state = ServerState.RUNNING
message = f"Container Service Extension for vCloud Director" \
f"\nServer running using config file: {self.config_file}" \
f"\nLog files: {SERVER_INFO_LOG_FILEPATH}, " \
f"{SERVER_DEBUG_LOG_FILEPATH}" \
f"\nwaiting for requests (ctrl+c to close)"
signal.signal(signal.SIGINT, signal_handler)
if msg_update_callback:
msg_update_callback.general_no_color(message)
LOGGER.info(message)
while True:
try:
time.sleep(1)
if self._state == ServerState.STOPPING and \
self.active_requests_count() == 0:
break
except KeyboardInterrupt:
break
except Exception:
if msg_update_callback:
msg_update_callback.general_no_color(
traceback.format_exc())
LOGGER.error(traceback.format_exc())
sys.exit(1)
LOGGER.info("Stop detected")
LOGGER.info("Closing connections...")
for c in self.consumers:
try:
c.stop()
except Exception:
pass
self._state = ServerState.STOPPED
LOGGER.info("Done")
def list_template(ctx, config_file_name, skip_config_decryption,
display_option):
"""List CSE k8s templates."""
if skip_config_decryption:
password = None
else:
password = os.getenv('CSE_CONFIG_PASSWORD') or prompt_text(
PASSWORD_FOR_CONFIG_DECRYPTION_MSG, color='green', hide_input=True)
try:
console_message_printer = ConsoleMessagePrinter()
try:
check_python_version()
except Exception as err:
click.secho(str(err), fg='red')
sys.exit(1)
# We don't want to validate config file, because server startup or
# installation is not being perfomred. If values in config file are
# missing or bad, appropriate exception will be raised while accessing
# or using them.
if skip_config_decryption:
with open(config_file_name) as config_file:
config_dict = yaml.safe_load(config_file) or {}
else:
console_message_printer.info(f"Decrypting '{config_file_name}'")
config_dict = yaml.safe_load(
get_decrypted_file_contents(config_file_name, password)) or {}
local_templates = []
if display_option in (DISPLAY_ALL, DISPLAY_DIFF, DISPLAY_LOCAL):
client = None
try:
# To suppress the warning message that pyvcloud prints if
# ssl_cert verification is skipped.
if not config_dict['vcd']['verify']:
requests.packages.urllib3.disable_warnings()
client = Client(config_dict['vcd']['host'],
api_version=config_dict['vcd']['api_version'],
verify_ssl_certs=config_dict['vcd']['verify'])
credentials = BasicLoginCredentials(
config_dict['vcd']['username'], SYSTEM_ORG_NAME,
config_dict['vcd']['password'])
client.set_credentials(credentials)
org_name = config_dict['broker']['org']
catalog_name = config_dict['broker']['catalog']
local_template_definitions = \
get_all_k8s_local_template_definition(
client=client,
catalog_name=catalog_name,
org_name=org_name)
default_template_name = \
config_dict['broker']['default_template_name']
default_template_revision = \
str(config_dict['broker']['default_template_revision'])
for definition in local_template_definitions:
template = {}
template['name'] = definition[LocalTemplateKey.NAME]
template['revision'] = \
definition[LocalTemplateKey.REVISION]
template['compute_policy'] = \
definition[LocalTemplateKey.COMPUTE_POLICY]
template['local'] = True
template['remote'] = False
if str(definition[LocalTemplateKey.REVISION]
) == default_template_revision and definition[
LocalTemplateKey.
NAME] == default_template_name: # noqa: E501
template['default'] = True
else:
template['default'] = False
template['deprecated'] = \
str_to_bool(definition[LocalTemplateKey.DEPRECATED])
template['cpu'] = definition[LocalTemplateKey.CPU]
template['memory'] = definition[LocalTemplateKey.MEMORY]
template['description'] = \
definition[LocalTemplateKey.DESCRIPTION]
local_templates.append(template)
finally:
if client:
client.logout()
remote_templates = []
if display_option in (DISPLAY_ALL, DISPLAY_DIFF, DISPLAY_REMOTE):
rtm = RemoteTemplateManager(
remote_template_cookbook_url=config_dict['broker']
['remote_template_cookbook_url'], # noqa: E501
msg_update_callback=ConsoleMessagePrinter())
remote_template_cookbook = rtm.get_remote_template_cookbook()
remote_template_definitions = remote_template_cookbook['templates']
for definition in remote_template_definitions:
template = {}
template['name'] = definition[RemoteTemplateKey.NAME]
template['revision'] = definition[RemoteTemplateKey.REVISION]
template['compute_policy'] = \
definition[RemoteTemplateKey.COMPUTE_POLICY]
template['local'] = False
template['remote'] = True
template['default'] = False
template['deprecated'] = \
str_to_bool(definition[RemoteTemplateKey.DEPRECATED])
template['cpu'] = definition[RemoteTemplateKey.CPU]
template['memory'] = definition[RemoteTemplateKey.MEMORY]
template['description'] = \
definition[RemoteTemplateKey.DESCRIPTION]
remote_templates.append(template)
result = []
if display_option is DISPLAY_ALL:
result = remote_templates
# If local copy of template exists, update the remote definition
# with relevant values, else add the local definition to the result
# list.
for local_template in local_templates:
found = False
for remote_template in remote_templates:
if str(local_template[LocalTemplateKey.REVISION]) == str(
remote_template[RemoteTemplateKey.REVISION]
) and local_template[
LocalTemplateKey.NAME] == remote_template[
RemoteTemplateKey.NAME]: # noqa: E501
remote_template['compute_policy'] = \
local_template['compute_policy']
remote_template['local'] = local_template['local']
remote_template['default'] = local_template['default']
found = True
break
if not found:
result.append(local_template)
elif display_option in DISPLAY_DIFF:
for remote_template in remote_templates:
found = False
for local_template in local_templates:
if str(local_template[LocalTemplateKey.REVISION]) == str(
remote_template[RemoteTemplateKey.REVISION]
) and local_template[
LocalTemplateKey.NAME] == remote_template[
RemoteTemplateKey.NAME]: # noqa: E501
found = True
break
if not found:
result.append(remote_template)
elif display_option in DISPLAY_LOCAL:
result = local_templates
elif display_option in DISPLAY_REMOTE:
result = remote_templates
stdout(result, ctx, sort_headers=False)
except cryptography.fernet.InvalidToken:
click.secho(CONFIG_DECRYPTION_ERROR_MSG, fg='red')
except Exception as err:
click.secho(str(err), fg='red')
def _load_template_definition_from_catalog(self,
msg_update_callback=utils.NullPrinter()): # noqa: E501
# NOTE: If `enable_tkg_plus` in the config file is set to false,
# CSE server will skip loading the TKG+ template this will prevent
# users from performing TKG+ related operations.
msg = "Loading k8s template definition from catalog"
logger.SERVER_LOGGER.info(msg)
msg_update_callback.general_no_color(msg)
client = None
try:
log_filename = None
log_wire = \
utils.str_to_bool(self.config['service'].get('log_wire'))
if log_wire:
log_filename = logger.SERVER_DEBUG_WIRELOG_FILEPATH
client = Client(self.config['vcd']['host'],
api_version=self.config['vcd']['api_version'],
verify_ssl_certs=self.config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(self.config['vcd']['username'],
server_constants.SYSTEM_ORG_NAME, # noqa: E501
self.config['vcd']['password'])
client.set_credentials(credentials)
is_tkg_plus_enabled = utils.is_tkg_plus_enabled(self.config)
org_name = self.config['broker']['org']
catalog_name = self.config['broker']['catalog']
k8_templates = ltm.get_all_k8s_local_template_definition(
client=client, catalog_name=catalog_name, org_name=org_name,
logger_debug=logger.SERVER_LOGGER)
if not k8_templates:
msg = "No valid K8 templates were found in catalog " \
f"'{catalog_name}'. Unable to start CSE server."
msg_update_callback.error(msg)
logger.SERVER_LOGGER.error(msg)
sys.exit(1)
# Check that default k8s template exists in vCD at the correct
# revision
default_template_name = \
self.config['broker']['default_template_name']
default_template_revision = \
str(self.config['broker']['default_template_revision'])
found_default_template = False
for template in k8_templates:
api_version = float(client.get_api_version())
if api_version >= float(vCDApiVersion.VERSION_35.value) and \
template[server_constants.LocalTemplateKey.KIND] == \
shared_constants.ClusterEntityKind.TKG_PLUS.value and \
not is_tkg_plus_enabled:
# TKG+ is not enabled on CSE config. Skip the template and
# log the relevant information.
msg = "Skipping loading template data for " \
f"'{template[server_constants.LocalTemplateKey.NAME]}' as " \
"TKG+ is not enabled" # noqa: E501
logger.SERVER_LOGGER.debug(msg)
k8_templates.remove(template)
continue
if str(template[server_constants.LocalTemplateKey.REVISION]) == default_template_revision and \
template[server_constants.LocalTemplateKey.NAME] == default_template_name: # noqa: E501
found_default_template = True
msg = f"Found K8 template '{template['name']}' at revision " \
f"{template['revision']} in catalog '{catalog_name}'"
msg_update_callback.general(msg)
logger.SERVER_LOGGER.info(msg)
if not found_default_template:
msg = f"Default template {default_template_name} with " \
f"revision {default_template_revision} not found." \
" Unable to start CSE server."
msg_update_callback.error(msg)
logger.SERVER_LOGGER.error(msg)
sys.exit(1)
self.config['broker']['templates'] = k8_templates
finally:
if client:
client.logout()
