def configure_vcd_amqp(client,
exchange_name,
host,
port,
prefix,
ssl_accept_all,
use_ssl,
vhost,
username,
password,
quiet=False):
"""Configures vCD AMQP settings/exchange using parameter values.
:param pyvcloud.vcd.client.Client client:
:param str exchange_name: name of exchange.
:param str host: AMQP host name.
:param int port: AMQP port.
:param str prefix:
:param bool ssl_accept_all:
:param bool use_ssl: Enable ssl.
:param str vhost: AMQP vhost.
:param str username: AMQP username.
:param str password: AMQP password.
:param bool quiet: if True, disables all console and log output
:raises Exception: if could not set AMQP configuration.
"""
amqp_service = AmqpService(client)
amqp = {
'AmqpExchange': exchange_name,
'AmqpHost': host,
'AmqpPort': port,
'AmqpPrefix': prefix,
'AmqpSslAcceptAll': ssl_accept_all,
'AmqpUseSSL': use_ssl,
'AmqpUsername': username,
'AmqpVHost': vhost
}
# This block sets the AMQP setting values on the
# vCD "System Administration Extensibility page"
result = amqp_service.test_config(amqp, password)
if not quiet:
msg = f"AMQP test settings, result: {result['Valid'].text}"
click.secho(msg, fg='yellow')
LOGGER.info(msg)
if result['Valid'].text == 'true':
amqp_service.set_config(amqp, password)
if not quiet:
msg = "Updated vCD AMQP configuration"
click.secho(msg, fg='green')
LOGGER.info(msg)
else:
msg = "Couldn't set vCD AMQP configuration"
if not quiet:
click.secho(msg, fg='red')
LOGGER.error(msg, exc_info=True)
# TODO replace raw exception with specific
raise Exception(msg)
def _create_amqp_exchange(exchange_name,
host,
port,
vhost,
use_ssl,
username,
password,
msg_update_callback=None):
"""Create the specified AMQP exchange if it does not exist.
If specified AMQP exchange exists already, does nothing.
:param str exchange_name: The AMQP exchange name to check for or create.
:param str host: AMQP host name.
:param str password: AMQP password.
:param int port: AMQP port number.
:param bool use_ssl: Enable ssl.
:param str username: AMQP username.
:param str vhost: AMQP vhost.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object
that writes messages onto console.
:raises AmqpError: if AMQP exchange could not be created.
"""
msg = f"Checking for AMQP exchange '{exchange_name}'"
if msg_update_callback:
msg_update_callback.info(msg)
LOGGER.info(msg)
credentials = pika.PlainCredentials(username, password)
parameters = pika.ConnectionParameters(host,
port,
vhost,
credentials,
ssl=use_ssl,
connection_attempts=3,
retry_delay=2,
socket_timeout=5)
connection = None
try:
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.exchange_declare(exchange=exchange_name,
exchange_type=EXCHANGE_TYPE,
durable=True,
auto_delete=False)
except Exception as err:
msg = f"Cannot create AMQP exchange '{exchange_name}'"
if msg_update_callback:
msg_update_callback.error(msg)
LOGGER.error(msg, exc_info=True)
raise AmqpError(msg, str(err))
finally:
if connection is not None:
connection.close()
msg = f"AMQP exchange '{exchange_name}' is ready"
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
def install_template(template_name,
template_revision,
config_file_name,
force_create,
retain_temp_vapp,
ssh_key,
skip_config_decryption=False,
decryption_password=None,
msg_update_callback=None):
"""Install a particular template in CSE.
If template_name and revision are wild carded to *, all templates defined
in remote template cookbook will be installed.
:param str template_name:
:param str template_revision:
:param str config_file_name: config file name.
:param bool force_create: if True and template already exists in vCD,
overwrites existing template.
:param str ssh_key: public ssh key to place into template vApp(s).
:param bool retain_temp_vapp: if True, temporary vApp will not destroyed,
so the user can ssh into and debug the vm.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object
that writes messages onto console.
"""
configure_install_logger()
config = get_validated_config(
config_file_name,
skip_config_decryption=skip_config_decryption,
decryption_password=decryption_password,
msg_update_callback=msg_update_callback)
populate_vsphere_list(config['vcs'])
msg = f"Installing template '{template_name}' at revision " \
f"'{template_revision}' on vCloud Director using config file " \
f"'{config_file_name}'"
if msg_update_callback:
msg_update_callback.info(msg)
LOGGER.info(msg)
client = None
try:
log_filename = None
log_wire = str_to_bool(config['service'].get('log_wire'))
if log_wire:
log_filename = INSTALL_WIRELOG_FILEPATH
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
# read remote template cookbook
rtm = RemoteTemplateManager(
remote_template_cookbook_url=config['broker']
['remote_template_cookbook_url'], # noqa: E501
logger=LOGGER,
msg_update_callback=msg_update_callback)
remote_template_cookbook = rtm.get_remote_template_cookbook()
found_template = False
for template in remote_template_cookbook['templates']:
template_name_matched = template_name in (
template[RemoteTemplateKey.NAME], '*') # noqa: E501
template_revision_matched = str(template_revision) in (str(
template[RemoteTemplateKey.REVISION]), '*') # noqa: E501
if template_name_matched and template_revision_matched:
found_template = True
_install_template(
client=client,
remote_template_manager=rtm,
template=template,
org_name=config['broker']['org'],
vdc_name=config['broker']['vdc'],
catalog_name=config['broker']['catalog'],
network_name=config['broker']['network'],
ip_allocation_mode=config['broker']['ip_allocation_mode'],
storage_profile=config['broker']['storage_profile'],
force_update=force_create,
retain_temp_vapp=retain_temp_vapp,
ssh_key=ssh_key,
msg_update_callback=msg_update_callback)
if not found_template:
msg = f"Template '{template_name}' at revision " \
f"'{template_revision}' not found in remote template " \
"cookbook."
if msg_update_callback:
msg_update_callback.error(msg)
LOGGER.error(msg, exc_info=True)
except Exception:
if msg_update_callback:
msg_update_callback.error(
"Template Installation Error. Check CSE install logs")
LOGGER.error("Template Installation Error", exc_info=True)
finally:
if client is not None:
client.logout()
def install_cse(config_file_name,
skip_template_creation,
force_update,
ssh_key,
retain_temp_vapp,
pks_config_file_name=None,
skip_config_decryption=False,
decryption_password=None,
msg_update_callback=None):
"""Handle logistics for CSE installation.
Handles decision making for configuring AMQP exchange/settings,
extension registration, catalog setup, and template creation.
:param str config_file_name: config file name.
:param bool skip_template_creation: If True, skip creating the templates.
:param bool force_update: if True and templates already exist in vCD,
overwrites existing templates.
:param str ssh_key: public ssh key to place into template vApp(s).
:param bool retain_temp_vapp: if True, temporary vApp will not destroyed,
so the user can ssh into and debug the vm.
:param str pks_config_file_name: pks config file name.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object
that writes messages onto console.
:raises AmqpError: if AMQP exchange could not be created.
"""
configure_install_logger()
config = get_validated_config(
config_file_name,
pks_config_file_name=pks_config_file_name,
skip_config_decryption=skip_config_decryption,
decryption_password=decryption_password,
msg_update_callback=msg_update_callback)
populate_vsphere_list(config['vcs'])
msg = f"Installing CSE on vCloud Director using config file " \
f"'{config_file_name}'"
if msg_update_callback:
msg_update_callback.info(msg)
LOGGER.info(msg)
client = None
try:
log_filename = None
log_wire = str_to_bool(config['service'].get('log_wire'))
if log_wire:
log_filename = INSTALL_WIRELOG_FILEPATH
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
# create amqp exchange if it doesn't exist
amqp = config['amqp']
_create_amqp_exchange(amqp['exchange'],
amqp['host'],
amqp['port'],
amqp['vhost'],
amqp['ssl'],
amqp['username'],
amqp['password'],
msg_update_callback=msg_update_callback)
# register or update cse on vCD
_register_cse(client,
amqp['routing_key'],
amqp['exchange'],
msg_update_callback=msg_update_callback)
# register rights to vCD
# TODO() should also remove rights when unregistering CSE
_register_right(client,
right_name=CSE_NATIVE_DEPLOY_RIGHT_NAME,
description=CSE_NATIVE_DEPLOY_RIGHT_DESCRIPTION,
category=CSE_NATIVE_DEPLOY_RIGHT_CATEGORY,
bundle_key=CSE_NATIVE_DEPLOY_RIGHT_BUNDLE_KEY,
msg_update_callback=msg_update_callback)
_register_right(client,
right_name=CSE_PKS_DEPLOY_RIGHT_NAME,
description=CSE_PKS_DEPLOY_RIGHT_DESCRIPTION,
category=CSE_PKS_DEPLOY_RIGHT_CATEGORY,
bundle_key=CSE_PKS_DEPLOY_RIGHT_BUNDLE_KEY,
msg_update_callback=msg_update_callback)
# set up cse catalog
org = get_org(client, org_name=config['broker']['org'])
create_and_share_catalog(org,
config['broker']['catalog'],
catalog_desc='CSE templates',
msg_update_callback=msg_update_callback)
if skip_template_creation:
msg = "Skipping creation of templates."
if msg_update_callback:
msg_update_callback.info(msg)
LOGGER.warning(msg)
else:
# read remote template cookbook, download all scripts
rtm = RemoteTemplateManager(
remote_template_cookbook_url=config['broker']
['remote_template_cookbook_url'], # noqa: E501
logger=LOGGER,
msg_update_callback=msg_update_callback)
remote_template_cookbook = rtm.get_remote_template_cookbook()
# create all templates defined in cookbook
for template in remote_template_cookbook['templates']:
_install_template(
client=client,
remote_template_manager=rtm,
template=template,
org_name=config['broker']['org'],
vdc_name=config['broker']['vdc'],
catalog_name=config['broker']['catalog'],
network_name=config['broker']['network'],
ip_allocation_mode=config['broker']['ip_allocation_mode'],
storage_profile=config['broker']['storage_profile'],
force_update=force_update,
retain_temp_vapp=retain_temp_vapp,
ssh_key=ssh_key,
msg_update_callback=msg_update_callback)
# if it's a PKS setup, setup NSX-T constructs
if config.get('pks_config'):
nsxt_servers = config.get('pks_config')['nsxt_servers']
for nsxt_server in nsxt_servers:
msg = f"Configuring NSX-T server ({nsxt_server.get('name')})" \
" for CSE. Please check install logs for details."
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
nsxt_client = NSXTClient(host=nsxt_server.get('host'),
username=nsxt_server.get('username'),
password=nsxt_server.get('password'),
http_proxy=nsxt_server.get('proxy'),
https_proxy=nsxt_server.get('proxy'),
verify_ssl=nsxt_server.get('verify'),
logger_instance=LOGGER,
log_requests=True,
log_headers=True,
log_body=True)
setup_nsxt_constructs(
nsxt_client=nsxt_client,
nodes_ip_block_id=nsxt_server.get('nodes_ip_block_ids'),
pods_ip_block_id=nsxt_server.get('pods_ip_block_ids'),
ncp_boundary_firewall_section_anchor_id=nsxt_server.get(
'distributed_firewall_section_anchor_id')
) # noqa: E501
except Exception:
if msg_update_callback:
msg_update_callback.error(
"CSE Installation Error. Check CSE install logs")
LOGGER.error("CSE Installation Error", exc_info=True)
raise # TODO() need installation relevant exceptions for rollback
finally:
if client is not None:
client.logout()
def _register_def_schema(client: Client,
msg_update_callback=utils.NullPrinter(),
log_wire=False):
"""Register defined entity interface and defined entity type.
If vCD api version is >= 35, register the vCD api version based
defined entity interface and defined entity type. Read the schema present
in the location dictated by api version to register the
defined entity type.
:param pyvcloud.vcd.client.Client client:
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:param bool log_wire: wire logging enabled
"""
msg = "Registering defined entity schema"
msg_update_callback.info(msg)
INSTALL_LOGGER.debug(msg)
logger_wire = SERVER_CLOUDAPI_WIRE_LOGGER if log_wire else NULL_LOGGER
cloudapi_client = vcd_utils.get_cloudapi_client_from_vcd_client(
client=client, # noqa: E501
logger_debug=INSTALL_LOGGER, # noqa: E501
logger_wire=logger_wire) # noqa: E501
schema_file = None
try:
def_utils.raise_error_if_def_not_supported(cloudapi_client)
schema_svc = def_schema_svc.DefSchemaService(cloudapi_client)
keys_map = def_utils.MAP_API_VERSION_TO_KEYS[float(
client.get_api_version())] # noqa: E501
defKey = def_utils.DefKey
native_interface = def_models.\
DefInterface(name=keys_map[defKey.INTERFACE_NAME],
vendor=keys_map[defKey.VENDOR],
nss=keys_map[defKey.INTERFACE_NSS],
version=keys_map[defKey.INTERFACE_VERSION], # noqa: E501
readonly=False)
msg = ""
try:
schema_svc.get_interface(native_interface.get_id())
msg = "defined entity interface already exists." \
" Skipping defined entity interface creation"
except HTTPError:
# TODO handle this part only if the interface was not found
native_interface = schema_svc.create_interface(native_interface)
msg = "Successfully created defined entity interface"
msg_update_callback.general(msg)
INSTALL_LOGGER.debug(msg)
# TODO stop-gap fix - find efficient way to import schema
import importlib
import importlib.resources as pkg_resources
schema_module = importlib.import_module(
f'{def_utils.DEF_SCHEMA_DIRECTORY}.{keys_map[defKey.ENTITY_TYPE_SCHEMA_VERSION]}'
) # noqa: E501
schema_file = pkg_resources.open_text(
schema_module, def_utils.DEF_ENTITY_TYPE_SCHEMA_FILE) # noqa: E501
native_entity_type = def_models.\
DefEntityType(name=keys_map[defKey.ENTITY_TYPE_NAME],
description='',
vendor=keys_map[defKey.VENDOR],
nss=keys_map[defKey.ENTITY_TYPE_NSS],
version=keys_map[defKey.ENTITY_TYPE_VERSION],
schema=json.load(schema_file),
interfaces=[native_interface.get_id()],
readonly=False)
msg = ""
try:
schema_svc.get_entity_type(native_entity_type.get_id())
msg = "defined entity type already exists." \
" Skipping defined entity type creation"
except HTTPError:
# TODO handle this part only if the entity type was not found
native_entity_type = schema_svc.create_entity_type(
native_entity_type) # noqa: E501
msg = "Successfully registered defined entity type"
msg_update_callback.general(msg)
INSTALL_LOGGER.debug(msg)
except cse_exception.DefNotSupportedException:
msg = "Skipping defined entity type and defined entity interface" \
" registration"
msg_update_callback.general(msg)
INSTALL_LOGGER.debug(msg)
except Exception as e:
msg = f"Error occured while registering defined entity schema: {str(e)}" # noqa: E501
msg_update_callback.error(msg)
INSTALL_LOGGER.error(msg)
raise (e)
finally:
try:
schema_file.close()
except Exception:
pass
def install_template(template_name,
template_revision,
config_file_name,
force_create,
retain_temp_vapp,
ssh_key,
skip_config_decryption=False,
decryption_password=None,
msg_update_callback=utils.NullPrinter()):
"""Install a particular template in CSE.
If template_name and revision are wild carded to *, all templates defined
in remote template cookbook will be installed.
:param str template_name:
:param str template_revision:
:param str config_file_name: config file name.
:param bool force_create: if True and template already exists in vCD,
overwrites existing template.
:param str ssh_key: public ssh key to place into template vApp(s).
:param bool retain_temp_vapp: if True, temporary vApp will not destroyed,
so the user can ssh into and debug the vm.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
"""
config = get_validated_config(
config_file_name,
skip_config_decryption=skip_config_decryption,
decryption_password=decryption_password,
log_wire_file=INSTALL_WIRELOG_FILEPATH,
logger_debug=INSTALL_LOGGER,
msg_update_callback=msg_update_callback)
populate_vsphere_list(config['vcs'])
msg = f"Installing template '{template_name}' at revision " \
f"'{template_revision}' on vCloud Director using config file " \
f"'{config_file_name}'"
msg_update_callback.info(msg)
INSTALL_LOGGER.info(msg)
client = None
try:
# Telemetry data construction
cse_params = {
PayloadKey.TEMPLATE_NAME: template_name,
PayloadKey.TEMPLATE_REVISION: template_revision,
PayloadKey.WAS_DECRYPTION_SKIPPED: bool(skip_config_decryption),
PayloadKey.WERE_TEMPLATES_FORCE_UPDATED: bool(force_create),
PayloadKey.WAS_TEMP_VAPP_RETAINED: bool(retain_temp_vapp),
PayloadKey.WAS_SSH_KEY_SPECIFIED: bool(ssh_key)
}
# Record telemetry data
record_user_action_details(
cse_operation=CseOperation.TEMPLATE_INSTALL,
cse_params=cse_params,
telemetry_settings=config['service']['telemetry'])
log_filename = None
log_wire = utils.str_to_bool(config['service'].get('log_wire'))
if log_wire:
log_filename = INSTALL_WIRELOG_FILEPATH
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(config['vcd']['username'],
server_constants.SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
msg_update_callback.general(msg)
INSTALL_LOGGER.info(msg)
# read remote template cookbook
rtm = RemoteTemplateManager(
remote_template_cookbook_url=config['broker']
['remote_template_cookbook_url'], # noqa: E501
logger=INSTALL_LOGGER,
msg_update_callback=msg_update_callback)
remote_template_cookbook = rtm.get_remote_template_cookbook()
found_template = False
for template in remote_template_cookbook['templates']:
template_name_matched = template_name in (
template[server_constants.RemoteTemplateKey.NAME], '*'
) # noqa: E501
template_revision_matched = \
str(template_revision) in (str(template[server_constants.RemoteTemplateKey.REVISION]), '*') # noqa: E501
if template_name_matched and template_revision_matched:
found_template = True
_install_template(
client=client,
remote_template_manager=rtm,
template=template,
org_name=config['broker']['org'],
vdc_name=config['broker']['vdc'],
catalog_name=config['broker']['catalog'],
network_name=config['broker']['network'],
ip_allocation_mode=config['broker']['ip_allocation_mode'],
storage_profile=config['broker']['storage_profile'],
force_update=force_create,
retain_temp_vapp=retain_temp_vapp,
ssh_key=ssh_key,
msg_update_callback=msg_update_callback)
if not found_template:
msg = f"Template '{template_name}' at revision " \
f"'{template_revision}' not found in remote template " \
"cookbook."
msg_update_callback.error(msg)
INSTALL_LOGGER.error(msg, exc_info=True)
raise Exception(msg)
# Record telemetry data on successful template install
record_user_action(
cse_operation=CseOperation.TEMPLATE_INSTALL,
status=OperationStatus.SUCCESS,
telemetry_settings=config['service']['telemetry']) # noqa: E501
except Exception:
msg_update_callback.error(
"Template Installation Error. Check CSE install logs")
INSTALL_LOGGER.error("Template Installation Error", exc_info=True)
# Record telemetry data on template install failure
record_user_action(cse_operation=CseOperation.TEMPLATE_INSTALL,
status=OperationStatus.FAILED,
telemetry_settings=config['service']['telemetry'])
finally:
if client is not None:
client.logout()
def install_cse(config_file_name,
skip_template_creation,
force_update,
ssh_key,
retain_temp_vapp,
pks_config_file_name=None,
skip_config_decryption=False,
decryption_password=None,
msg_update_callback=utils.NullPrinter()):
"""Handle logistics for CSE installation.
Handles decision making for configuring AMQP exchange/settings,
defined entity schema registration for vCD api version >= 35,
extension registration, catalog setup and template creation.
Also records telemetry data on installation details.
:param str config_file_name: config file name.
:param bool skip_template_creation: If True, skip creating the templates.
:param bool force_update: if True and templates already exist in vCD,
overwrites existing templates.
:param str ssh_key: public ssh key to place into template vApp(s).
:param bool retain_temp_vapp: if True, temporary vApp will not destroyed,
so the user can ssh into and debug the vm.
:param str pks_config_file_name: pks config file name.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:raises cse_exception.AmqpError: if AMQP exchange could not be created.
"""
config = get_validated_config(
config_file_name,
pks_config_file_name=pks_config_file_name,
skip_config_decryption=skip_config_decryption,
decryption_password=decryption_password,
log_wire_file=INSTALL_WIRELOG_FILEPATH,
logger_debug=INSTALL_LOGGER,
msg_update_callback=msg_update_callback)
populate_vsphere_list(config['vcs'])
msg = f"Installing CSE on vCloud Director using config file " \
f"'{config_file_name}'"
msg_update_callback.info(msg)
INSTALL_LOGGER.info(msg)
client = None
try:
# Telemetry - Construct telemetry data
telemetry_data = {
PayloadKey.WAS_DECRYPTION_SKIPPED:
bool(skip_config_decryption), # noqa: E501
PayloadKey.WAS_PKS_CONFIG_FILE_PROVIDED:
bool(pks_config_file_name), # noqa: E501
PayloadKey.WERE_TEMPLATES_SKIPPED:
bool(skip_template_creation), # noqa: E501
PayloadKey.WERE_TEMPLATES_FORCE_UPDATED:
bool(force_update), # noqa: E501
PayloadKey.WAS_TEMP_VAPP_RETAINED:
bool(retain_temp_vapp), # noqa: E501
PayloadKey.WAS_SSH_KEY_SPECIFIED: bool(ssh_key) # noqa: E501
}
# Telemetry - Record detailed telemetry data on install
record_user_action_details(
CseOperation.SERVICE_INSTALL,
telemetry_data,
telemetry_settings=config['service']['telemetry']) # noqa: E501
log_filename = None
log_wire = utils.str_to_bool(config['service'].get('log_wire'))
if log_wire:
log_filename = INSTALL_WIRELOG_FILEPATH
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(config['vcd']['username'],
server_constants.SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
msg_update_callback.general(msg)
INSTALL_LOGGER.info(msg)
# create amqp exchange if it doesn't exist
amqp = config['amqp']
_create_amqp_exchange(amqp['exchange'],
amqp['host'],
amqp['port'],
amqp['vhost'],
amqp['ssl'],
amqp['username'],
amqp['password'],
msg_update_callback=msg_update_callback)
# register or update cse on vCD
_register_cse(client,
amqp['routing_key'],
amqp['exchange'],
msg_update_callback=msg_update_callback)
# register cse def schema on VCD
# schema should be located at
# ~/.cse-schema/api-v<API VERSION>/schema.json
_register_def_schema(client,
msg_update_callback=msg_update_callback,
log_wire=log_wire)
# Since we use CSE extension id as our telemetry instance_id, the
# validated config won't have the instance_id yet. Now that CSE has
# been registered as an extension, we should update the telemetry
# config with the correct instance_id
if config['service']['telemetry']['enable']:
store_telemetry_settings(config)
# register rights to vCD
# TODO() should also remove rights when unregistering CSE
_register_right(
client,
right_name=server_constants.
CSE_NATIVE_DEPLOY_RIGHT_NAME, # noqa: E501
description=server_constants.
CSE_NATIVE_DEPLOY_RIGHT_DESCRIPTION, # noqa: E501
category=server_constants.
CSE_NATIVE_DEPLOY_RIGHT_CATEGORY, # noqa: E501
bundle_key=server_constants.
CSE_NATIVE_DEPLOY_RIGHT_BUNDLE_KEY, # noqa: E501
msg_update_callback=msg_update_callback)
_register_right(
client,
right_name=server_constants.
CSE_PKS_DEPLOY_RIGHT_NAME, # noqa: E501
description=server_constants.
CSE_PKS_DEPLOY_RIGHT_DESCRIPTION, # noqa: E501
category=server_constants.
CSE_PKS_DEPLOY_RIGHT_CATEGORY, # noqa: E501
bundle_key=server_constants.
CSE_PKS_DEPLOY_RIGHT_BUNDLE_KEY, # noqa: E501
msg_update_callback=msg_update_callback)
# set up placement policies for all types of clusters
_setup_placement_policies(
client,
policy_list=server_constants.
CLUSTER_PLACEMENT_POLICIES, # noqa: E501
msg_update_callback=msg_update_callback,
log_wire=log_wire)
# set up cse catalog
org = vcd_utils.get_org(client, org_name=config['broker']['org'])
vcd_utils.create_and_share_catalog(
org,
config['broker']['catalog'],
catalog_desc='CSE templates',
logger=INSTALL_LOGGER,
msg_update_callback=msg_update_callback)
if skip_template_creation:
msg = "Skipping creation of templates."
msg_update_callback.info(msg)
INSTALL_LOGGER.warning(msg)
else:
# read remote template cookbook, download all scripts
rtm = RemoteTemplateManager(
remote_template_cookbook_url=config['broker']
['remote_template_cookbook_url'], # noqa: E501
logger=INSTALL_LOGGER,
msg_update_callback=msg_update_callback)
remote_template_cookbook = rtm.get_remote_template_cookbook()
# create all templates defined in cookbook
for template in remote_template_cookbook['templates']:
# TODO tag created templates with placement policies
_install_template(
client=client,
remote_template_manager=rtm,
template=template,
org_name=config['broker']['org'],
vdc_name=config['broker']['vdc'],
catalog_name=config['broker']['catalog'],
network_name=config['broker']['network'],
ip_allocation_mode=config['broker']['ip_allocation_mode'],
storage_profile=config['broker']['storage_profile'],
force_update=force_update,
retain_temp_vapp=retain_temp_vapp,
ssh_key=ssh_key,
msg_update_callback=msg_update_callback)
# if it's a PKS setup, setup NSX-T constructs
if config.get('pks_config'):
nsxt_servers = config['pks_config']['nsxt_servers']
wire_logger = NULL_LOGGER
if log_wire:
wire_logger = SERVER_NSXT_WIRE_LOGGER
for nsxt_server in nsxt_servers:
msg = f"Configuring NSX-T server ({nsxt_server.get('name')})" \
" for CSE. Please check install logs for details."
msg_update_callback.general(msg)
INSTALL_LOGGER.info(msg)
nsxt_client = NSXTClient(host=nsxt_server.get('host'),
username=nsxt_server.get('username'),
password=nsxt_server.get('password'),
logger_debug=INSTALL_LOGGER,
logger_wire=wire_logger,
http_proxy=nsxt_server.get('proxy'),
https_proxy=nsxt_server.get('proxy'),
verify_ssl=nsxt_server.get('verify'))
setup_nsxt_constructs(
nsxt_client=nsxt_client,
nodes_ip_block_id=nsxt_server.get('nodes_ip_block_ids'),
pods_ip_block_id=nsxt_server.get('pods_ip_block_ids'),
ncp_boundary_firewall_section_anchor_id=nsxt_server.get(
'distributed_firewall_section_anchor_id')
) # noqa: E501
# Telemetry - Record successful install action
record_user_action(CseOperation.SERVICE_INSTALL,
telemetry_settings=config['service']['telemetry'])
except Exception:
msg_update_callback.error(
"CSE Installation Error. Check CSE install logs")
INSTALL_LOGGER.error("CSE Installation Error", exc_info=True)
# Telemetry - Record failed install action
record_user_action(CseOperation.SERVICE_INSTALL,
status=OperationStatus.FAILED,
telemetry_settings=config['service']['telemetry'])
raise # TODO() need installation relevant exceptions for rollback
finally:
if client is not None:
client.logout()
def install_cse(ctx,
config_file_name='config.yaml',
skip_template_creation=True,
force_update=False,
ssh_key=None,
retain_temp_vapp=False,
msg_update_callback=None):
"""Handle logistics for CSE installation.
Handles decision making for configuring AMQP exchange/settings,
extension registration, catalog setup, and template creation.
:param click.core.Context ctx:
:param str config_file_name: config file name.
:param bool skip_template_creation: If True, skip creating the templates.
:param bool force_update: if True and templates already exist in vCD,
overwrites existing templates.
:param str ssh_key: public ssh key to place into template vApp(s).
:param bool retain_temp_vapp: if True, temporary vApp will not destroyed,
so the user can ssh into and debug the vm.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object
that writes messages onto console.
:raises AmqpError: if AMQP exchange could not be created.
"""
configure_install_logger()
config = get_validated_config(config_file_name,
msg_update_callback=msg_update_callback)
populate_vsphere_list(config['vcs'])
msg = f"Installing CSE on vCloud Director using config file " \
f"'{config_file_name}'"
if msg_update_callback:
msg_update_callback.info(msg)
LOGGER.info(msg)
client = None
try:
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=INSTALL_WIRELOG_FILEPATH,
log_requests=True,
log_headers=True,
log_bodies=True)
credentials = BasicLoginCredentials(config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
# create amqp exchange if it doesn't exist
amqp = config['amqp']
_create_amqp_exchange(amqp['exchange'],
amqp['host'],
amqp['port'],
amqp['vhost'],
amqp['ssl'],
amqp['username'],
amqp['password'],
msg_update_callback=msg_update_callback)
# register or update cse on vCD
_register_cse(client,
amqp['routing_key'],
amqp['exchange'],
msg_update_callback=msg_update_callback)
# register rights to vCD
# TODO() should also remove rights when unregistering CSE
_register_right(client,
right_name=CSE_NATIVE_DEPLOY_RIGHT_NAME,
description=CSE_NATIVE_DEPLOY_RIGHT_DESCRIPTION,
category=CSE_NATIVE_DEPLOY_RIGHT_CATEGORY,
bundle_key=CSE_NATIVE_DEPLOY_RIGHT_BUNDLE_KEY,
msg_update_callback=msg_update_callback)
_register_right(client,
right_name=CSE_PKS_DEPLOY_RIGHT_NAME,
description=CSE_PKS_DEPLOY_RIGHT_DESCRIPTION,
category=CSE_PKS_DEPLOY_RIGHT_CATEGORY,
bundle_key=CSE_PKS_DEPLOY_RIGHT_BUNDLE_KEY,
msg_update_callback=msg_update_callback)
org_name = config['broker']['org']
catalog_name = config['broker']['catalog']
# set up cse catalog
org = get_org(client, org_name=org_name)
create_and_share_catalog(org,
catalog_name,
catalog_desc='CSE templates',
msg_update_callback=msg_update_callback)
if skip_template_creation:
msg = "Skipping creation of templates."
if msg_update_callback:
msg_update_callback.info(msg)
LOGGER.warning(msg)
else:
# read remote template cookbook, download all scripts
rtm = RemoteTemplateManager(
remote_template_cookbook_url=config['broker']
['remote_template_cookbook_url'], # noqa: E501
logger=LOGGER,
msg_update_callback=ConsoleMessagePrinter())
remote_template_cookbook = rtm.get_remote_template_cookbook()
# create all templates defined in cookbook
for template in remote_template_cookbook['templates']:
rtm.download_template_scripts(
template_name=template[RemoteTemplateKey.NAME],
revision=template[RemoteTemplateKey.REVISION],
force_overwrite=force_update)
catalog_item_name = get_revisioned_template_name(
template[RemoteTemplateKey.NAME],
template[RemoteTemplateKey.REVISION])
build_params = {
'template_name':
template[RemoteTemplateKey.NAME],
'template_revision':
template[RemoteTemplateKey.REVISION],
'source_ova_name':
template[RemoteTemplateKey.SOURCE_OVA_NAME], # noqa: E501
'source_ova_href':
template[RemoteTemplateKey.SOURCE_OVA_HREF], # noqa: E501
'source_ova_sha256':
template[
RemoteTemplateKey.SOURCE_OVA_SHA256], # noqa: E501
'org_name':
org_name,
'vdc_name':
config['broker']['vdc'],
'catalog_name':
catalog_name,
'catalog_item_name':
catalog_item_name,
'catalog_item_description':
template[RemoteTemplateKey.DESCRIPTION], # noqa: E501
'temp_vapp_name':
template[RemoteTemplateKey.NAME] + '_temp', # noqa: E501
'cpu':
template[RemoteTemplateKey.CPU],
'memory':
template[RemoteTemplateKey.MEMORY],
'network_name':
config['broker']['network'],
'ip_allocation_mode':
config['broker']['ip_allocation_mode'], # noqa: E501
'storage_profile':
config['broker']['storage_profile']
}
builder = TemplateBuilder(
client,
client,
build_params,
ssh_key=ssh_key,
logger=LOGGER,
msg_update_callback=ConsoleMessagePrinter())
builder.build(force_recreate=force_update,
retain_temp_vapp=retain_temp_vapp)
# remote definition is a super set of local definition, barring
# the key 'catalog_item_name'
template_definition = dict(template)
template_definition['catalog_item_name'] = catalog_item_name
save_k8s_local_template_definition_as_metadata(
client=client,
catalog_name=catalog_name,
catalog_item_name=catalog_item_name,
template_definition=template_definition,
org_name=org_name)
# if it's a PKS setup, setup NSX-T constructs
if config.get('pks_config'):
nsxt_servers = config.get('pks_config')['nsxt_servers']
for nsxt_server in nsxt_servers:
msg = f"Configuring NSX-T server ({nsxt_server.get('name')})" \
" for CSE. Please check install logs for details."
if msg_update_callback:
msg_update_callback.general(msg)
LOGGER.info(msg)
nsxt_client = NSXTClient(host=nsxt_server.get('host'),
username=nsxt_server.get('username'),
password=nsxt_server.get('password'),
http_proxy=nsxt_server.get('proxy'),
https_proxy=nsxt_server.get('proxy'),
verify_ssl=nsxt_server.get('verify'),
logger_instance=LOGGER,
log_requests=True,
log_headers=True,
log_body=True)
setup_nsxt_constructs(
nsxt_client=nsxt_client,
nodes_ip_block_id=nsxt_server.get('nodes_ip_block_ids'),
pods_ip_block_id=nsxt_server.get('pods_ip_block_ids'),
ncp_boundary_firewall_section_anchor_id=nsxt_server.get(
'distributed_firewall_section_anchor_id')
) # noqa: E501
except Exception:
if msg_update_callback:
msg_update_callback.error(
"CSE Installation Error. Check CSE install logs")
LOGGER.error("CSE Installation Error", exc_info=True)
raise # TODO() need installation relevant exceptions for rollback
finally:
if client is not None:
client.logout()
def _customize_vm(ctx, config, vapp, vm_name, cust_script, is_photon=False):
"""Customizes a VM in a VApp using the customization script @cust_script.
:param click.core.Context ctx: click context object. Needed to pass to
stdout.
:param dict config: CSE config.
:param pyvcloud.vcd.vapp.VApp vapp:
:param str vm_name:
:param str cust_script: the customization script to run on
:param bool is_photon: True if the vapp was instantiated from
a 'photon' ova file, False otherwise (False is safe even if
the vapp is photon-based).
:raises Exception: if unable to execute the customization script in
VSphere.
"""
callback = vgr_callback(prepend_msg='Waiting for guest tools, status: "')
if not is_photon:
vs = get_vsphere(config, vapp, vm_name, logger=LOGGER)
wait_until_tools_ready(vapp, vs, callback=callback)
vapp.reload()
task = vapp.shutdown()
stdout(task, ctx=ctx)
vapp.reload()
task = vapp.power_on()
stdout(task, ctx=ctx)
vapp.reload()
vs = get_vsphere(config, vapp, vm_name, logger=LOGGER)
wait_until_tools_ready(vapp, vs, callback=callback)
password_auto = vapp.get_admin_password(vm_name)
try:
result = vs.execute_script_in_guest(vs.get_vm_by_moid(
vapp.get_vm_moid(vm_name)),
'root',
password_auto,
cust_script,
target_file=None,
wait_for_completion=True,
wait_time=10,
get_output=True,
delete_script=True,
callback=vgr_callback())
except Exception as err:
# TODO replace raw exception with specific exception
# unsure all errors execute_script_in_guest can result in
# Docker TLS handshake timeout can occur when internet is slow
click.secho("Failed VM customization. Check CSE install log", fg='red')
LOGGER.error(f"Failed VM customization with error: f{err}",
exc_info=True)
raise
if len(result) > 0:
msg = f'Result: {result}'
click.echo(msg)
LOGGER.debug(msg)
result_stdout = result[1].content.decode()
result_stderr = result[2].content.decode()
msg = 'stderr:'
click.echo(msg)
LOGGER.debug(msg)
if len(result_stderr) > 0:
click.echo(result_stderr)
LOGGER.debug(result_stderr)
msg = 'stdout:'
click.echo(msg)
LOGGER.debug(msg)
if len(result_stdout) > 0:
click.echo(result_stdout)
LOGGER.debug(result_stdout)
if len(result) == 0 or result[0] != 0:
msg = "Failed VM customization"
click.secho(f"{msg}. Check CSE install log", fg='red')
LOGGER.error(msg, exc_info=True)
# TODO replace raw exception with specific exception
raise Exception(msg)
def install_cse(ctx,
config_file_name='config.yaml',
template_name='*',
update=False,
no_capture=False,
ssh_key=None,
amqp_install='prompt',
ext_install='prompt'):
"""Handles logistics for CSE installation.
Handles decision making for configuring AMQP exchange/settings,
extension registration, catalog setup, and template creation.
:param click.core.Context ctx:
:param str config_file_name: config file name.
:param str template_name: which templates to create/update. A value of '*'
means to create/update all templates specified in config file.
:param bool update: if True and templates already exist in vCD,
overwrites existing templates.
:param bool no_capture: if True, temporary vApp will not be captured or
destroyed, so the user can ssh into and debug the VM.
:param str ssh_key: public ssh key to place into template vApp(s).
:param str amqp_install: 'prompt' asks the user if vCD AMQP should be
configured. 'skip' does not configure vCD AMQP. 'config' configures
vCD AMQP without asking the user.
:param str ext_install: 'prompt' asks the user if CSE should be registered
to vCD. 'skip' does not register CSE to vCD. 'config' registers CSE
to vCD without asking the user.
:raises AmqpError: if AMQP exchange could not be created.
"""
config = get_validated_config(config_file_name)
configure_install_logger()
msg = f"Installing CSE on vCloud Director using config file " \
f"'{config_file_name}'"
click.secho(msg, fg='yellow')
LOGGER.info(msg)
client = None
try:
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=INSTALL_LOG_FILEPATH,
log_headers=True,
log_bodies=True)
credentials = BasicLoginCredentials(config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
click.secho(msg, fg='green')
LOGGER.info(msg)
# configure amqp
amqp = config['amqp']
create_amqp_exchange(amqp['exchange'], amqp['host'], amqp['port'],
amqp['vhost'], amqp['ssl'], amqp['username'],
amqp['password'])
if should_configure_amqp(client, amqp, amqp_install):
configure_vcd_amqp(client, amqp['exchange'], amqp['host'],
amqp['port'], amqp['prefix'],
amqp['ssl_accept_all'], amqp['ssl'],
amqp['vhost'], amqp['username'],
amqp['password'])
# register cse as extension to vCD
if should_register_cse(client, ext_install):
register_cse(client, amqp['routing_key'], amqp['exchange'])
# set up cse catalog
org = get_org(client, org_name=config['broker']['org'])
create_and_share_catalog(org,
config['broker']['catalog'],
catalog_desc='CSE templates')
# create, customize, capture VM templates
for template in config['broker']['templates']:
if template_name == '*' or template['name'] == template_name:
create_template(ctx,
client,
config,
template,
update=update,
no_capture=no_capture,
ssh_key=ssh_key,
org=org)
except Exception:
click.secho("CSE Installation Error. Check CSE install logs", fg='red')
LOGGER.error("CSE Installation Error", exc_info=True)
raise # TODO need installation relevant exceptions for rollback
finally:
if client is not None:
client.logout()
