def _validate_vcd_config(
vcd_dict,
msg_update_callback=NullPrinter(),
log_file=None,
log_wire=False
):
"""Ensure that 'vcd' section of config is correct.
Checks that
* 'vcd' section of config has correct keys and value types.
* vCD is accessible.
:param dict vcd_dict: 'vcd' section of config file as a dict.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:param str log_file: log_file for pyvcloud wire log.
:param bool log_wire: If pyvcloud requests should be logged.
:raises KeyError: if @vcd_dict has missing
:raises TypeError: if the value type for a @vcd_dict is incorrect.
"""
check_keys_and_value_types(
vcd_dict,
SAMPLE_VCD_CONFIG['vcd'],
location="config file 'vcd' section",
msg_update_callback=msg_update_callback
)
if not vcd_dict['verify']:
msg_update_callback.general(
'InsecureRequestWarning: Unverified HTTPS request is '
'being made. Adding certificate verification is '
'strongly advised.'
)
requests.packages.urllib3.disable_warnings()
client = None
try:
_validate_vcd_url_scheme(vcd_dict['host'])
client = Client(
vcd_dict['host'],
verify_ssl_certs=vcd_dict['verify'],
log_file=log_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire
)
client.set_credentials(
BasicLoginCredentials(
vcd_dict['username'],
SYSTEM_ORG_NAME,
vcd_dict['password']
)
)
msg_update_callback.general(
"Connected to vCloud Director "
f"({vcd_dict['host']}:{vcd_dict['port']})"
)
finally:
if client is not None:
client.logout()
def _validate_broker_config(
broker_dict,
legacy_mode=False,
msg_update_callback=NullPrinter(),
logger_debug=NULL_LOGGER
):
"""Ensure that 'broker' section of config is correct.
Checks that 'broker' section of config has correct keys and value
types. Also checks that 'default_broker' property is a valid template.
:param dict broker_dict: 'broker' section of config file as a dict.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:raises KeyError: if @broker_dict has missing properties.
:raises TypeError: if the value type for a @broker_dict property is
incorrect.
:raises ValueError: if 'ip_allocation_mode' is not 'dhcp' or 'pool'. Or
if remote_template_cookbook_url is invalid.
"""
check_keys_and_value_types(
broker_dict,
SAMPLE_BROKER_CONFIG['broker'],
location="config file 'broker' section",
msg_update_callback=msg_update_callback
)
valid_ip_allocation_modes = [
'dhcp',
'pool'
]
if broker_dict['ip_allocation_mode'] not in valid_ip_allocation_modes:
raise ValueError(
"IP allocation mode is "
f"'{broker_dict['ip_allocation_mode']}' when it "
"should be either 'dhcp' or 'pool'"
)
rtm = RemoteTemplateManager(
remote_template_cookbook_url=broker_dict['remote_template_cookbook_url'], # noqa: E501
legacy_mode=legacy_mode,
logger=logger_debug
)
remote_template_cookbook = rtm.get_filtered_remote_template_cookbook()
if not remote_template_cookbook:
raise Exception("Remote template cookbook is invalid.")
def _validate_amqp_config(amqp_dict, msg_update_callback=NullPrinter()):
"""Ensure that 'amqp' section of config is correct.
Checks that 'amqp' section of config has correct keys and value types.
Also ensures that connection to AMQP server is valid.
:param dict amqp_dict: 'amqp' section of config file as a dict.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:raises KeyError: if @amqp_dict has missing or extra properties.
:raises TypeError: if the value type for an @amqp_dict property
is incorrect.
:raises AmqpConnectionError: if AMQP connection failed.
"""
check_keys_and_value_types(amqp_dict,
SAMPLE_AMQP_CONFIG['amqp'],
location="config file 'amqp' section",
msg_update_callback=msg_update_callback)
credentials = pika.PlainCredentials(amqp_dict['username'],
amqp_dict['password'])
parameters = pika.ConnectionParameters(amqp_dict['host'],
amqp_dict['port'],
amqp_dict['vhost'],
credentials,
connection_attempts=3,
retry_delay=2,
socket_timeout=5)
connection = None
try:
connection = pika.BlockingConnection(parameters)
msg_update_callback.general(
"Connected to AMQP server "
f"({amqp_dict['host']}:{amqp_dict['port']})")
except Exception as err:
raise AmqpConnectionError(f"AMQP server error : {str(err)}")
finally:
if connection is not None:
connection.close()
def _validate_pks_config_structure(
pks_config,
msg_update_callback=NullPrinter()
):
sample_config = {
**SAMPLE_PKS_SERVERS_SECTION,
**SAMPLE_PKS_ACCOUNTS_SECTION,
**SAMPLE_PKS_ORGS_SECTION,
**SAMPLE_PKS_PVDCS_SECTION,
**SAMPLE_PKS_NSXT_SERVERS_SECTION
}
check_keys_and_value_types(
pks_config,
sample_config,
location='pks config file',
excluded_keys=[PKS_ORGS_SECTION_KEY],
msg_update_callback=msg_update_callback
)
pks_servers = pks_config[PKS_SERVERS_SECTION_KEY]
for index, pks_server in enumerate(pks_servers, 1):
check_keys_and_value_types(
pks_server,
SAMPLE_PKS_SERVERS_SECTION[PKS_SERVERS_SECTION_KEY][0],
location=f"pks config file '{PKS_SERVERS_SECTION_KEY}' "
f"section, pks server #{index}",
excluded_keys=['proxy'],
msg_update_callback=msg_update_callback
)
pks_accounts = pks_config[PKS_ACCOUNTS_SECTION_KEY]
for index, pks_account in enumerate(pks_accounts, 1):
check_keys_and_value_types(
pks_account,
SAMPLE_PKS_ACCOUNTS_SECTION[PKS_ACCOUNTS_SECTION_KEY][0],
location=f"pks config file '{PKS_ACCOUNTS_SECTION_KEY}' "
f"section, pks account #{index}",
msg_update_callback=msg_update_callback
)
if PKS_ORGS_SECTION_KEY in pks_config.keys():
orgs = pks_config[PKS_ORGS_SECTION_KEY]
for index, org in enumerate(orgs, 1):
check_keys_and_value_types(
org,
SAMPLE_PKS_ORGS_SECTION[PKS_ORGS_SECTION_KEY][0],
location=f"pks config file '{PKS_ORGS_SECTION_KEY}' "
f"section, org #{index}",
msg_update_callback=msg_update_callback
)
pvdcs = pks_config[PKS_PVDCS_SECTION_KEY]
for index, pvdc in enumerate(pvdcs, 1):
check_keys_and_value_types(
pvdc,
SAMPLE_PKS_PVDCS_SECTION[PKS_PVDCS_SECTION_KEY][0],
location=f"pks config file '{PKS_PVDCS_SECTION_KEY}' "
f"section, pvdc #{index}",
msg_update_callback=msg_update_callback
)
nsxt_servers = pks_config[PKS_NSXT_SERVERS_SECTION_KEY]
for index, nsxt_server in enumerate(nsxt_servers, 1):
check_keys_and_value_types(
nsxt_server,
SAMPLE_PKS_NSXT_SERVERS_SECTION[PKS_NSXT_SERVERS_SECTION_KEY][0],
location=f"pks config file '{PKS_NSXT_SERVERS_SECTION_KEY}' "
f"section, nsxt server #{index}",
excluded_keys=['proxy'],
msg_update_callback=msg_update_callback
)
def get_validated_config(
config_file_name,
pks_config_file_name=None,
skip_config_decryption=False,
decryption_password=None,
log_wire_file=None,
logger_debug=NULL_LOGGER,
msg_update_callback=NullPrinter()
):
"""Get the config file as a dictionary and check for validity.
Ensures that all properties exist and all values are the expected type.
Checks that AMQP connection is available, and vCD/VCs are valid.
Does not guarantee that CSE has been installed according to this
config file. Additionally populates certain key-value pairs in the
config dict to avoid repeated computation of those e.g.
supported api versions, feature flags, RDE version in use etc.
:param str config_file_name: path to config file.
:param str pks_config_file_name: path to PKS config file.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param str log_wire_file: log_wire_file to use if needed to wire log
pyvcloud requests and responses
:param logging.Logger logger_debug: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return: CSE config
:rtype: dict
:raises KeyError: if config file has missing or extra properties.
:raises TypeError: if the value type for a config file property
is incorrect.
:raises container_service_extension.exceptions.AmqpConnectionError:
(when not using MQTT) if AMQP connection failed (host, password, port,
username, vhost is invalid).
:raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid.
:raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or
'password' is invalid.
:raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password'
is invalid.
"""
check_file_permissions(config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(config_file_name) as config_file:
config = yaml.safe_load(config_file) or {}
else:
msg_update_callback.info(
f"Decrypting '{config_file_name}'")
try:
config = yaml.safe_load(
get_decrypted_file_contents(
config_file_name,
decryption_password
)
) or {}
except cryptography.fernet.InvalidToken:
raise Exception(CONFIG_DECRYPTION_ERROR_MSG)
msg_update_callback.info(
f"Validating config file '{config_file_name}'"
)
is_no_vc_communication_mode = \
server_utils.is_no_vc_communication_mode(ServerConfig(config))
use_mqtt = server_utils.should_use_mqtt_protocol(ServerConfig(config))
sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \
else SAMPLE_MQTT_CONFIG
# This allows us to compare top-level config keys and value types
sample_config = {
**sample_message_queue_config,
**SAMPLE_VCD_CONFIG,
**SAMPLE_SERVICE_CONFIG,
**SAMPLE_BROKER_CONFIG
}
if not is_no_vc_communication_mode:
sample_config.update(SAMPLE_VCS_CONFIG)
else:
if 'vcs' in config:
del config['vcs']
log_wire = str_to_bool(config.get('service', {}).get('log_wire'))
nsxt_wire_logger = NULL_LOGGER
if not log_wire:
log_wire_file = None
nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER
check_keys_and_value_types(
config,
sample_config,
location='config file',
msg_update_callback=msg_update_callback
)
# MQTT validation not required because no MQTT host, exchange, etc.
# is needed in the config file since the server code creates and
# registers the MQTT extension directly using server constants
if not use_mqtt:
_validate_amqp_config(config['amqp'], msg_update_callback)
# Validation of service properties is done first as those properties are
# used in broker validation.
check_keys_and_value_types(
config['service'],
SAMPLE_SERVICE_CONFIG['service'],
location="config file 'service' section",
excluded_keys=['log_wire'],
msg_update_callback=msg_update_callback
)
try:
if is_no_vc_communication_mode:
_validate_vcd_config(
config['vcd'],
msg_update_callback,
log_file=log_wire_file,
log_wire=log_wire
)
else:
_validate_vcd_and_vcs_config(
config['vcd'],
config['vcs'],
msg_update_callback,
log_file=log_wire_file,
log_wire=log_wire
)
except vim.fault.InvalidLogin:
raise Exception(VCENTER_LOGIN_ERROR_MSG)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
except requests.exceptions.ConnectionError as err:
raise Exception(f"Cannot connect to {err.request.url}.")
check_keys_and_value_types(
config['service']['telemetry'],
SAMPLE_SERVICE_CONFIG['service']['telemetry'],
location="config file 'service->telemetry' section",
msg_update_callback=msg_update_callback
)
_validate_broker_config(
config['broker'],
legacy_mode=config['service']['legacy_mode'],
msg_update_callback=msg_update_callback,
logger_debug=logger_debug
)
config = add_additional_details_to_config(
config=config,
vcd_host=config['vcd']['host'],
vcd_username=config['vcd']['username'],
vcd_password=config['vcd']['password'],
verify_ssl=config['vcd']['verify'],
is_legacy_mode=config['service']['legacy_mode'],
is_mqtt_exchange=server_utils.should_use_mqtt_protocol(
ServerConfig(config)
),
log_wire=log_wire,
log_wire_file=log_wire_file
)
_raise_error_if_amqp_not_supported(
use_mqtt,
config['service']['default_api_version'],
logger=logger_debug
)
msg_update_callback.general(
f"Config file '{config_file_name}' is valid"
)
if pks_config_file_name:
check_file_permissions(pks_config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(pks_config_file_name) as f:
pks_config = yaml.safe_load(f) or {}
else:
msg_update_callback.info(
f"Decrypting '{pks_config_file_name}'")
pks_config = yaml.safe_load(
get_decrypted_file_contents(pks_config_file_name,
decryption_password)) or {}
msg_update_callback.info(
f"Validating PKS config file '{pks_config_file_name}'")
_validate_pks_config_structure(pks_config, msg_update_callback)
try:
_validate_pks_config_data_integrity(pks_config,
msg_update_callback,
logger_debug=logger_debug,
logger_wire=nsxt_wire_logger)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
msg_update_callback.general(
f"PKS Config file '{pks_config_file_name}' is valid")
config['pks_config'] = pks_config
else:
config['pks_config'] = None
return config
def _validate_vcd_and_vcs_config(
vcd_dict,
vcs,
msg_update_callback=NullPrinter(),
log_file=None,
log_wire=False
):
"""Ensure that 'vcd' and vcs' section of config are correct.
Checks that
* 'vcd' and 'vcs' section of config have correct keys and value types.
* vCD and all registered VCs in vCD are accessible.
* api version specified for vcd is supported by CSE.
:param dict vcd_dict: 'vcd' section of config file as a dict.
:param list vcs: 'vcs' section of config file as a list of dicts.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:param str log_file: log_file for pyvcloud wire log.
:param bool log_wire: If pyvcloud requests should be logged.
:raises KeyError: if @vcd_dict or a vc in @vcs has missing or
extra properties.
:raises TypeError: if the value type for a @vcd_dict or vc property
is incorrect.
:raises ValueError: if vCD has a VC that is not listed in the config file.
"""
check_keys_and_value_types(vcd_dict, SAMPLE_VCD_CONFIG['vcd'],
location="config file 'vcd' section",
msg_update_callback=msg_update_callback)
if not vcd_dict['verify']:
msg_update_callback.general(
'InsecureRequestWarning: Unverified HTTPS request is '
'being made. Adding certificate verification is '
'strongly advised.'
)
requests.packages.urllib3.disable_warnings()
client = None
try:
_validate_vcd_url_scheme(vcd_dict['host'])
client = Client(
vcd_dict['host'],
verify_ssl_certs=vcd_dict['verify'],
log_file=log_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire
)
client.set_credentials(
BasicLoginCredentials(
vcd_dict['username'],
SYSTEM_ORG_NAME,
vcd_dict['password']
)
)
msg_update_callback.general(
"Connected to vCloud Director "
f"({vcd_dict['host']}:{vcd_dict['port']})"
)
for index, vc in enumerate(vcs, 1):
check_keys_and_value_types(
vc,
SAMPLE_VCS_CONFIG['vcs'][0],
location=f"config file 'vcs' section, vc #{index}",
msg_update_callback=msg_update_callback
)
# Check that all registered VCs in vCD are listed in config file
platform = Platform(client)
config_vc_names = [vc['name'] for vc in vcs]
for platform_vc in platform.list_vcenters():
platform_vc_name = platform_vc.get('name')
if platform_vc_name not in config_vc_names:
raise ValueError(
f"vCenter '{platform_vc_name}' registered in "
"vCD but not found in config file"
)
# Check that all VCs listed in config file are registered in vCD
for vc in vcs:
vcenter = platform.get_vcenter(vc['name'])
if not (hasattr(vcenter, 'IsConnected') and vcenter.IsConnected):
msg = f"vCenter Server '{vc['name']}' not available"
msg_update_callback.info(msg)
continue
vsphere_url = urlparse(vcenter.Url.text)
vsphere_url_port = vsphere_url.port
if vsphere_url_port:
v = VSphere(
vsphere_url.hostname,
vc['username'],
vc['password'],
vsphere_url.port
)
else:
v = VSphere(
vsphere_url.hostname,
vc['username'],
vc['password']
)
v.connect()
msg = f"Connected to vCenter Server '{vc['name']}' as " \
f"'{vc['username']}' ({vsphere_url.hostname}"
if vsphere_url_port:
msg += f":{vsphere_url.port}"
msg += ")"
msg_update_callback.general(msg)
finally:
if client is not None:
client.logout()
def get_validated_config(config_file_name,
pks_config_file_name=None,
skip_config_decryption=False,
decryption_password=None,
log_wire_file=None,
logger_debug=NULL_LOGGER,
msg_update_callback=NullPrinter()):
"""Get the config file as a dictionary and check for validity.
Ensures that all properties exist and all values are the expected type.
Checks that AMQP connection is available, and vCD/VCs are valid.
Does not guarantee that CSE has been installed according to this
config file.
:param str config_file_name: path to config file.
:param str pks_config_file_name: path to PKS config file.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param str log_wire_file: log_wire_file to use if needed to wire log
pyvcloud requests and responses
:param logging.Logger logger_debug: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return: CSE config
:rtype: dict
:raises KeyError: if config file has missing or extra properties.
:raises TypeError: if the value type for a config file property
is incorrect.
:raises container_service_extension.exceptions.AmqpConnectionError:
(when not using MQTT) if AMQP connection failed (host, password, port,
username, vhost is invalid).
:raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid.
:raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or
'password' is invalid.
:raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password'
is invalid.
"""
check_file_permissions(config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(config_file_name) as config_file:
config = yaml.safe_load(config_file) or {}
else:
msg_update_callback.info(
f"Decrypting '{config_file_name}'")
try:
config = yaml.safe_load(
get_decrypted_file_contents(config_file_name,
decryption_password)) or {}
except cryptography.fernet.InvalidToken:
raise Exception(CONFIG_DECRYPTION_ERROR_MSG)
msg_update_callback.info(
f"Validating config file '{config_file_name}'")
# This allows us to compare top-level config keys and value types
use_mqtt = should_use_mqtt_protocol(config)
sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \
else SAMPLE_MQTT_CONFIG
sample_config = {
**sample_message_queue_config, **SAMPLE_VCD_CONFIG,
**SAMPLE_VCS_CONFIG, **SAMPLE_SERVICE_CONFIG,
**SAMPLE_BROKER_CONFIG
}
log_wire = str_to_bool(config.get('service', {}).get('log_wire'))
nsxt_wire_logger = NULL_LOGGER
if not log_wire:
log_wire_file = None
nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER
check_keys_and_value_types(config, sample_config, location='config file',
msg_update_callback=msg_update_callback)
# MQTT validation not required because no MQTT host, exchange, etc.
# is needed in the config file since the server code creates and
# registers the MQTT extension directly using server constants
if not use_mqtt:
_validate_amqp_config(config['amqp'], msg_update_callback)
try:
_validate_vcd_and_vcs_config(config['vcd'], config['vcs'],
msg_update_callback,
log_file=log_wire_file,
log_wire=log_wire)
except vim.fault.InvalidLogin:
raise Exception(VCENTER_LOGIN_ERROR_MSG)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
except requests.exceptions.ConnectionError as err:
raise Exception(f"Cannot connect to {err.request.url}.")
_validate_broker_config(config['broker'],
legacy_mode=config['service']['legacy_mode'],
msg_update_callback=msg_update_callback,
logger_debug=logger_debug)
check_keys_and_value_types(config['service'],
SAMPLE_SERVICE_CONFIG['service'],
location="config file 'service' section",
excluded_keys=['log_wire'],
msg_update_callback=msg_update_callback)
check_keys_and_value_types(config['service']['telemetry'],
SAMPLE_SERVICE_CONFIG['service']['telemetry'],
location="config file 'service->telemetry' "
"section",
msg_update_callback=msg_update_callback)
msg_update_callback.general(
f"Config file '{config_file_name}' is valid")
if pks_config_file_name:
check_file_permissions(pks_config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(pks_config_file_name) as f:
pks_config = yaml.safe_load(f) or {}
else:
msg_update_callback.info(
f"Decrypting '{pks_config_file_name}'")
pks_config = yaml.safe_load(
get_decrypted_file_contents(pks_config_file_name,
decryption_password)) or {}
msg_update_callback.info(
f"Validating PKS config file '{pks_config_file_name}'")
_validate_pks_config_structure(pks_config, msg_update_callback)
try:
_validate_pks_config_data_integrity(pks_config,
msg_update_callback,
logger_debug=logger_debug,
logger_wire=nsxt_wire_logger)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
msg_update_callback.general(
f"PKS Config file '{pks_config_file_name}' is valid")
config['pks_config'] = pks_config
else:
config['pks_config'] = None
# Compute common supported api versions by the CSE server and vCD
sysadmin_client = None
try:
sysadmin_client = Client(
config['vcd']['host'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_wire_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
sysadmin_client.set_credentials(BasicLoginCredentials(
config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password']))
vcd_supported_api_versions = \
set(sysadmin_client.get_supported_versions_list())
cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS)
common_supported_api_versions = \
list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501
common_supported_api_versions.sort()
config['service']['supported_api_versions'] = \
common_supported_api_versions
finally:
if sysadmin_client:
sysadmin_client.logout()
# Convert legacy_mode flag in service_section to corresponding
# feature flags
is_legacy_mode = config['service']['legacy_mode']
if 'feature_flags' not in config:
config['feature_flags'] = {}
config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode)
config['feature_flags']['non_legacy_api'] = \
not str_to_bool(is_legacy_mode)
# Temporary work around before api version is completely removed from
# config
if is_legacy_mode:
supported_api_versions_float = \
[float(x) for x in config['service']['supported_api_versions']
if float(x) < 35.0]
else:
supported_api_versions_float = \
[float(x) for x in config['service']['supported_api_versions']
if float(x) >= 35.0]
config['vcd']['api_version'] = str(max(supported_api_versions_float))
# Store telemetry instance id, url and collector id in config
# This steps needs to be done after api_version has been computed
# and stored in the config
store_telemetry_settings(config)
return config
