def init_rde_environment(config_filepath=BASE_CONFIG_FILEPATH,
logger=NULL_LOGGER): # noqa: E501
"""Set up module variables according to config dict.
:param str config_filepath:
:param logging.Logger logger:
"""
global CLIENT, ORG_HREF, VDC_HREF, \
CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \
TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \
CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_LOGIN_CMD, \
USERNAME_TO_LOGIN_CMD, USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, \
TEST_VDC_HREF, VCD_API_VERSION_TO_USE, VCD_SITE
logger.debug("Setting RDE environement")
config = testutils.yaml_to_dict(config_filepath)
logger.debug(f"Config file used: {config}")
sysadmin_client = Client(config['vcd']['host'],
verify_ssl_certs=config['vcd']['verify'])
sysadmin_client.set_credentials(
BasicLoginCredentials(config['vcd']['username'],
shared_constants.SYSTEM_ORG_NAME,
config['vcd']['password']))
vcd_supported_api_versions = \
set(sysadmin_client.get_supported_versions_list())
cse_supported_api_versions = set(
shared_constants.SUPPORTED_VCD_API_VERSIONS) # noqa: E501
common_supported_api_versions = \
list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501
common_supported_api_versions.sort()
max_api_version = get_max_api_version(common_supported_api_versions)
CLIENT = Client(config['vcd']['host'],
api_version=max_api_version,
verify_ssl_certs=config['vcd']['verify'])
credentials = BasicLoginCredentials(config['vcd']['username'],
shared_constants.SYSTEM_ORG_NAME,
config['vcd']['password'])
CLIENT.set_credentials(credentials)
VCD_API_VERSION_TO_USE = max_api_version
logger.debug(f"Using VCD api version: {VCD_API_VERSION_TO_USE}")
CATALOG_NAME = config['broker']['catalog']
VCD_SITE = f"https://{config['vcd']['host']}"
SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \
f"{config['vcd']['username']} " \
f"-iwp {config['vcd']['password']} " \
f"-V {VCD_API_VERSION_TO_USE}"
CLUSTER_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \
f"{TEST_ORG}" \
f" {CLUSTER_ADMIN_NAME} " \
f"-iwp {CLUSTER_ADMIN_PASSWORD} " \
f"-V {VCD_API_VERSION_TO_USE}"
CLUSTER_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \
f"{TEST_ORG}" \
f" {CLUSTER_AUTHOR_NAME} " \
f"-iwp {CLUSTER_AUTHOR_PASSWORD} " \
f"-V {VCD_API_VERSION_TO_USE}"
USERNAME_TO_LOGIN_CMD = {
SYS_ADMIN_NAME: SYS_ADMIN_LOGIN_CMD,
CLUSTER_ADMIN_NAME: CLUSTER_ADMIN_LOGIN_CMD,
CLUSTER_AUTHOR_NAME: CLUSTER_AUTHOR_LOGIN_CMD
}
# hrefs for Org and VDC that hosts the catalog
org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org'])
vdc = pyvcloud_utils.get_vdc(CLIENT,
vdc_name=config['broker']['vdc'],
org=org)
ORG_HREF = org.href
VDC_HREF = vdc.href
logger.debug(f"Using template org {org.get_name()} with href {ORG_HREF}")
logger.debug(f"Using template vdc {vdc.name} with href {VDC_HREF}")
# hrefs for Org and VDC that tests cluster operations
test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG)
test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org)
TEST_ORG_HREF = test_org.href
TEST_VDC_HREF = test_vdc.href
logger.debug(f"Using test org {test_org.get_name()} "
f"with href {TEST_ORG_HREF}")
logger.debug(f"Using test vdc {test_vdc.name} with href {TEST_VDC_HREF}")
if SHOULD_INSTALL_PREREQUISITES:
create_cluster_admin_role(config['vcd'], logger=logger)
create_cluster_author_role(config['vcd'], logger=logger)
# create and publish sizing class sc1 to TEST_VDC
cpm = ComputePolicyManager(sysadmin_client=sysadmin_client,
log_wire=True)
created_policy = None
try:
created_policy = cpm.add_vdc_compute_policy(
SIZING_CLASS_NAME,
description=SIZING_CLASS_DESCRIPTION,
cpu_count=2,
memory_mb=2048)
except HTTPError as err:
if 'already exists' in err.response.text:
logger.debug(
f"Compute policy {SIZING_CLASS_NAME} already exists"
) # noqa: E501
created_policy = cpm.get_vdc_compute_policy(SIZING_CLASS_NAME)
else:
logger.error(
f"Request to create sizing policy {SIZING_CLASS_NAME} failed."
) # noqa: E501
raise
try:
cpm.add_compute_policy_to_vdc(
pyvcloud_utils.extract_id(
test_vdc.get_resource_admin().get('id')), # noqa: E501
created_policy['id'])
except Exception as err:
logger.error(
f"Error publishing sizing policy {SIZING_CLASS_NAME} to vdc {TEST_VDC}: {err}"
) # noqa: E501
create_cluster_admin_role(config['vcd'], logger=logger)
create_cluster_author_role(config['vcd'], logger=logger)
def init_rde_environment(config_filepath=BASE_CONFIG_FILEPATH,
logger=NULL_LOGGER): # noqa: E501
"""Set up module variables according to config dict.
:param str config_filepath:
"""
global CLIENT, ORG_HREF, VDC_HREF, \
CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \
TEMPLATE_DEFINITIONS, TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \
CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_LOGIN_CMD, \
USERNAME_TO_LOGIN_CMD, USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, \
TEST_VDC_HREF, VCD_API_VERSION_TO_USE
logger.debug("Setting RDE environement")
config = testutils.yaml_to_dict(config_filepath)
logger.debug(f"Config file used: {config}")
# download all remote template scripts
rtm = RemoteTemplateManager(
config['broker']['remote_template_cookbook_url'],
legacy_mode=config['service']['legacy_mode'])
template_cookbook = rtm.get_filtered_remote_template_cookbook()
TEMPLATE_DEFINITIONS = template_cookbook['templates']
rtm.download_all_template_scripts(force_overwrite=True)
sysadmin_client = Client(config['vcd']['host'],
verify_ssl_certs=config['vcd']['verify'])
sysadmin_client.set_credentials(
BasicLoginCredentials(config['vcd']['username'],
shared_constants.SYSTEM_ORG_NAME,
config['vcd']['password']))
vcd_supported_api_versions = \
set(sysadmin_client.get_supported_versions_list())
cse_supported_api_versions = set(
shared_constants.SUPPORTED_VCD_API_VERSIONS) # noqa: E501
common_supported_api_versions = \
list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501
common_supported_api_versions.sort()
max_api_version = get_max_api_version(common_supported_api_versions)
CLIENT = Client(config['vcd']['host'],
api_version=max_api_version,
verify_ssl_certs=config['vcd']['verify'])
credentials = BasicLoginCredentials(config['vcd']['username'],
shared_constants.SYSTEM_ORG_NAME,
config['vcd']['password'])
CLIENT.set_credentials(credentials)
VCD_API_VERSION_TO_USE = max_api_version
logger.debug(f"Using VCD api version: {VCD_API_VERSION_TO_USE}")
CATALOG_NAME = config['broker']['catalog']
SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \
f"{config['vcd']['username']} " \
f"-iwp {config['vcd']['password']} " \
f"-V {VCD_API_VERSION_TO_USE}"
CLUSTER_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \
f"{TEST_ORG}" \
f" {CLUSTER_ADMIN_NAME} " \
f"-iwp {CLUSTER_ADMIN_PASSWORD} " \
f"-V {VCD_API_VERSION_TO_USE}"
CLUSTER_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \
f"{TEST_ORG}" \
f" {CLUSTER_AUTHOR_NAME} " \
f"-iwp {CLUSTER_AUTHOR_PASSWORD} " \
f"-V {VCD_API_VERSION_TO_USE}"
USERNAME_TO_LOGIN_CMD = {
SYS_ADMIN_NAME: SYS_ADMIN_LOGIN_CMD,
CLUSTER_ADMIN_NAME: CLUSTER_ADMIN_LOGIN_CMD,
CLUSTER_AUTHOR_NAME: CLUSTER_AUTHOR_LOGIN_CMD
}
# hrefs for Org and VDC that hosts the catalog
org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org'])
vdc = pyvcloud_utils.get_vdc(CLIENT,
vdc_name=config['broker']['vdc'],
org=org)
ORG_HREF = org.href
VDC_HREF = vdc.href
logger.debug(f"Using template org {org.get_name()} with href {ORG_HREF}")
logger.debug(f"Using template vdc {vdc.name} with href {VDC_HREF}")
# hrefs for Org and VDC that tests cluster operations
test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG)
test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org)
TEST_ORG_HREF = test_org.href
TEST_VDC_HREF = test_vdc.href
logger.debug(f"Using test org {test_org.get_name()} "
f"with href {TEST_ORG_HREF}")
logger.debug(f"Using test vdc {test_vdc.name} with href {TEST_VDC_HREF}")
create_cluster_admin_role(config['vcd'], logger=logger)
create_cluster_author_role(config['vcd'], logger=logger)
def add_additional_details_to_config(
config: Dict,
vcd_host: str,
vcd_username: str,
vcd_password: str,
verify_ssl: bool,
is_legacy_mode: bool,
is_mqtt_exchange: bool,
log_wire: bool,
log_wire_file: str
):
"""Update config dict with computed key-value pairs.
:param dict config:
:param str vcd_host:
:param str vcd_username:
:param str vcd_password:
:param bool verify_ssl:
:param bool is_legacy_mode:
:param bool is_mqtt_exchange:
:param bool log_wire:
:param str log_wire_file:
:return: the updated config file
:rtype: dict
"""
# Compute common supported api versions by the CSE server and vCD
sysadmin_client = None
try:
sysadmin_client = Client(
vcd_host,
verify_ssl_certs=verify_ssl,
log_file=log_wire_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire
)
sysadmin_client.set_credentials(
BasicLoginCredentials(
vcd_username,
SYSTEM_ORG_NAME,
vcd_password
)
)
vcd_supported_api_versions = \
set(sysadmin_client.get_supported_versions_list())
cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS)
common_supported_api_versions = \
list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501
common_supported_api_versions.sort()
if is_legacy_mode:
common_supported_api_versions = \
[x for x in common_supported_api_versions
if VCDApiVersion(x) < VcdApiVersionObj.VERSION_35.value]
else:
common_supported_api_versions = \
[x for x in common_supported_api_versions
if VCDApiVersion(x) >= VcdApiVersionObj.VERSION_35.value]
config['service']['supported_api_versions'] = \
common_supported_api_versions
finally:
if sysadmin_client:
sysadmin_client.logout()
# Convert legacy_mode flag in service_section to corresponding
# feature flags
if 'feature_flags' not in config:
config['feature_flags'] = {}
config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode)
config['feature_flags']['non_legacy_api'] = \
not str_to_bool(is_legacy_mode)
# Compute the default api version as the max supported version
# Also compute the RDE version in use
max_vcd_api_version_supported: str = get_max_api_version(config['service']['supported_api_versions']) # noqa: E501
config['service']['default_api_version'] = max_vcd_api_version_supported
config['service']['rde_version_in_use'] = semantic_version.Version(
rde_utils.get_runtime_rde_version_by_vcd_api_version(
max_vcd_api_version_supported
)
)
# Update the config dict with telemetry specific key value pairs
update_with_telemetry_settings(
config_dict=config,
vcd_host=vcd_host,
vcd_username=vcd_username,
vcd_password=vcd_password,
verify_ssl=verify_ssl,
is_mqtt_exchange=is_mqtt_exchange
)
return config
def get_validated_config(config_file_name,
pks_config_file_name=None,
skip_config_decryption=False,
decryption_password=None,
log_wire_file=None,
logger_debug=NULL_LOGGER,
msg_update_callback=NullPrinter()):
"""Get the config file as a dictionary and check for validity.
Ensures that all properties exist and all values are the expected type.
Checks that AMQP connection is available, and vCD/VCs are valid.
Does not guarantee that CSE has been installed according to this
config file.
:param str config_file_name: path to config file.
:param str pks_config_file_name: path to PKS config file.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param str log_wire_file: log_wire_file to use if needed to wire log
pyvcloud requests and responses
:param logging.Logger logger_debug: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return: CSE config
:rtype: dict
:raises KeyError: if config file has missing or extra properties.
:raises TypeError: if the value type for a config file property
is incorrect.
:raises container_service_extension.exceptions.AmqpConnectionError:
(when not using MQTT) if AMQP connection failed (host, password, port,
username, vhost is invalid).
:raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid.
:raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or
'password' is invalid.
:raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password'
is invalid.
"""
check_file_permissions(config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(config_file_name) as config_file:
config = yaml.safe_load(config_file) or {}
else:
msg_update_callback.info(
f"Decrypting '{config_file_name}'")
try:
config = yaml.safe_load(
get_decrypted_file_contents(config_file_name,
decryption_password)) or {}
except cryptography.fernet.InvalidToken:
raise Exception(CONFIG_DECRYPTION_ERROR_MSG)
msg_update_callback.info(
f"Validating config file '{config_file_name}'")
# This allows us to compare top-level config keys and value types
use_mqtt = should_use_mqtt_protocol(config)
sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \
else SAMPLE_MQTT_CONFIG
sample_config = {
**sample_message_queue_config, **SAMPLE_VCD_CONFIG,
**SAMPLE_VCS_CONFIG, **SAMPLE_SERVICE_CONFIG,
**SAMPLE_BROKER_CONFIG
}
log_wire = str_to_bool(config.get('service', {}).get('log_wire'))
nsxt_wire_logger = NULL_LOGGER
if not log_wire:
log_wire_file = None
nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER
check_keys_and_value_types(config, sample_config, location='config file',
msg_update_callback=msg_update_callback)
# MQTT validation not required because no MQTT host, exchange, etc.
# is needed in the config file since the server code creates and
# registers the MQTT extension directly using server constants
if not use_mqtt:
_validate_amqp_config(config['amqp'], msg_update_callback)
try:
_validate_vcd_and_vcs_config(config['vcd'], config['vcs'],
msg_update_callback,
log_file=log_wire_file,
log_wire=log_wire)
except vim.fault.InvalidLogin:
raise Exception(VCENTER_LOGIN_ERROR_MSG)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
except requests.exceptions.ConnectionError as err:
raise Exception(f"Cannot connect to {err.request.url}.")
_validate_broker_config(config['broker'],
legacy_mode=config['service']['legacy_mode'],
msg_update_callback=msg_update_callback,
logger_debug=logger_debug)
check_keys_and_value_types(config['service'],
SAMPLE_SERVICE_CONFIG['service'],
location="config file 'service' section",
excluded_keys=['log_wire'],
msg_update_callback=msg_update_callback)
check_keys_and_value_types(config['service']['telemetry'],
SAMPLE_SERVICE_CONFIG['service']['telemetry'],
location="config file 'service->telemetry' "
"section",
msg_update_callback=msg_update_callback)
msg_update_callback.general(
f"Config file '{config_file_name}' is valid")
if pks_config_file_name:
check_file_permissions(pks_config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(pks_config_file_name) as f:
pks_config = yaml.safe_load(f) or {}
else:
msg_update_callback.info(
f"Decrypting '{pks_config_file_name}'")
pks_config = yaml.safe_load(
get_decrypted_file_contents(pks_config_file_name,
decryption_password)) or {}
msg_update_callback.info(
f"Validating PKS config file '{pks_config_file_name}'")
_validate_pks_config_structure(pks_config, msg_update_callback)
try:
_validate_pks_config_data_integrity(pks_config,
msg_update_callback,
logger_debug=logger_debug,
logger_wire=nsxt_wire_logger)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
msg_update_callback.general(
f"PKS Config file '{pks_config_file_name}' is valid")
config['pks_config'] = pks_config
else:
config['pks_config'] = None
# Compute common supported api versions by the CSE server and vCD
sysadmin_client = None
try:
sysadmin_client = Client(
config['vcd']['host'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_wire_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
sysadmin_client.set_credentials(BasicLoginCredentials(
config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password']))
vcd_supported_api_versions = \
set(sysadmin_client.get_supported_versions_list())
cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS)
common_supported_api_versions = \
list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501
common_supported_api_versions.sort()
config['service']['supported_api_versions'] = \
common_supported_api_versions
finally:
if sysadmin_client:
sysadmin_client.logout()
# Convert legacy_mode flag in service_section to corresponding
# feature flags
is_legacy_mode = config['service']['legacy_mode']
if 'feature_flags' not in config:
config['feature_flags'] = {}
config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode)
config['feature_flags']['non_legacy_api'] = \
not str_to_bool(is_legacy_mode)
# Temporary work around before api version is completely removed from
# config
if is_legacy_mode:
supported_api_versions_float = \
[float(x) for x in config['service']['supported_api_versions']
if float(x) < 35.0]
else:
supported_api_versions_float = \
[float(x) for x in config['service']['supported_api_versions']
if float(x) >= 35.0]
config['vcd']['api_version'] = str(max(supported_api_versions_float))
# Store telemetry instance id, url and collector id in config
# This steps needs to be done after api_version has been computed
# and stored in the config
store_telemetry_settings(config)
return config
