def init_rde_environment(config_filepath=BASE_CONFIG_FILEPATH, logger=NULL_LOGGER): # noqa: E501 """Set up module variables according to config dict. :param str config_filepath: :param logging.Logger logger: """ global CLIENT, ORG_HREF, VDC_HREF, \ CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \ TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \ CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_LOGIN_CMD, \ USERNAME_TO_LOGIN_CMD, USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, \ TEST_VDC_HREF, VCD_API_VERSION_TO_USE, VCD_SITE logger.debug("Setting RDE environement") config = testutils.yaml_to_dict(config_filepath) logger.debug(f"Config file used: {config}") sysadmin_client = Client(config['vcd']['host'], verify_ssl_certs=config['vcd']['verify']) sysadmin_client.set_credentials( BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password'])) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set( shared_constants.SUPPORTED_VCD_API_VERSIONS) # noqa: E501 common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() max_api_version = get_max_api_version(common_supported_api_versions) CLIENT = Client(config['vcd']['host'], api_version=max_api_version, verify_ssl_certs=config['vcd']['verify']) credentials = BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password']) CLIENT.set_credentials(credentials) VCD_API_VERSION_TO_USE = max_api_version logger.debug(f"Using VCD api version: {VCD_API_VERSION_TO_USE}") CATALOG_NAME = config['broker']['catalog'] VCD_SITE = f"https://{config['vcd']['host']}" SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \ f"{config['vcd']['username']} " \ f"-iwp {config['vcd']['password']} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_ADMIN_NAME} " \ f"-iwp {CLUSTER_ADMIN_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_AUTHOR_NAME} " \ f"-iwp {CLUSTER_AUTHOR_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" USERNAME_TO_LOGIN_CMD = { SYS_ADMIN_NAME: SYS_ADMIN_LOGIN_CMD, CLUSTER_ADMIN_NAME: CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_NAME: CLUSTER_AUTHOR_LOGIN_CMD } # hrefs for Org and VDC that hosts the catalog org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org']) vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=config['broker']['vdc'], org=org) ORG_HREF = org.href VDC_HREF = vdc.href logger.debug(f"Using template org {org.get_name()} with href {ORG_HREF}") logger.debug(f"Using template vdc {vdc.name} with href {VDC_HREF}") # hrefs for Org and VDC that tests cluster operations test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG) test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org) TEST_ORG_HREF = test_org.href TEST_VDC_HREF = test_vdc.href logger.debug(f"Using test org {test_org.get_name()} " f"with href {TEST_ORG_HREF}") logger.debug(f"Using test vdc {test_vdc.name} with href {TEST_VDC_HREF}") if SHOULD_INSTALL_PREREQUISITES: create_cluster_admin_role(config['vcd'], logger=logger) create_cluster_author_role(config['vcd'], logger=logger) # create and publish sizing class sc1 to TEST_VDC cpm = ComputePolicyManager(sysadmin_client=sysadmin_client, log_wire=True) created_policy = None try: created_policy = cpm.add_vdc_compute_policy( SIZING_CLASS_NAME, description=SIZING_CLASS_DESCRIPTION, cpu_count=2, memory_mb=2048) except HTTPError as err: if 'already exists' in err.response.text: logger.debug( f"Compute policy {SIZING_CLASS_NAME} already exists" ) # noqa: E501 created_policy = cpm.get_vdc_compute_policy(SIZING_CLASS_NAME) else: logger.error( f"Request to create sizing policy {SIZING_CLASS_NAME} failed." ) # noqa: E501 raise try: cpm.add_compute_policy_to_vdc( pyvcloud_utils.extract_id( test_vdc.get_resource_admin().get('id')), # noqa: E501 created_policy['id']) except Exception as err: logger.error( f"Error publishing sizing policy {SIZING_CLASS_NAME} to vdc {TEST_VDC}: {err}" ) # noqa: E501 create_cluster_admin_role(config['vcd'], logger=logger) create_cluster_author_role(config['vcd'], logger=logger)
def init_rde_environment(config_filepath=BASE_CONFIG_FILEPATH, logger=NULL_LOGGER): # noqa: E501 """Set up module variables according to config dict. :param str config_filepath: """ global CLIENT, ORG_HREF, VDC_HREF, \ CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \ TEMPLATE_DEFINITIONS, TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \ CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_LOGIN_CMD, \ USERNAME_TO_LOGIN_CMD, USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, \ TEST_VDC_HREF, VCD_API_VERSION_TO_USE logger.debug("Setting RDE environement") config = testutils.yaml_to_dict(config_filepath) logger.debug(f"Config file used: {config}") # download all remote template scripts rtm = RemoteTemplateManager( config['broker']['remote_template_cookbook_url'], legacy_mode=config['service']['legacy_mode']) template_cookbook = rtm.get_filtered_remote_template_cookbook() TEMPLATE_DEFINITIONS = template_cookbook['templates'] rtm.download_all_template_scripts(force_overwrite=True) sysadmin_client = Client(config['vcd']['host'], verify_ssl_certs=config['vcd']['verify']) sysadmin_client.set_credentials( BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password'])) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set( shared_constants.SUPPORTED_VCD_API_VERSIONS) # noqa: E501 common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() max_api_version = get_max_api_version(common_supported_api_versions) CLIENT = Client(config['vcd']['host'], api_version=max_api_version, verify_ssl_certs=config['vcd']['verify']) credentials = BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password']) CLIENT.set_credentials(credentials) VCD_API_VERSION_TO_USE = max_api_version logger.debug(f"Using VCD api version: {VCD_API_VERSION_TO_USE}") CATALOG_NAME = config['broker']['catalog'] SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \ f"{config['vcd']['username']} " \ f"-iwp {config['vcd']['password']} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_ADMIN_NAME} " \ f"-iwp {CLUSTER_ADMIN_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_AUTHOR_NAME} " \ f"-iwp {CLUSTER_AUTHOR_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" USERNAME_TO_LOGIN_CMD = { SYS_ADMIN_NAME: SYS_ADMIN_LOGIN_CMD, CLUSTER_ADMIN_NAME: CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_NAME: CLUSTER_AUTHOR_LOGIN_CMD } # hrefs for Org and VDC that hosts the catalog org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org']) vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=config['broker']['vdc'], org=org) ORG_HREF = org.href VDC_HREF = vdc.href logger.debug(f"Using template org {org.get_name()} with href {ORG_HREF}") logger.debug(f"Using template vdc {vdc.name} with href {VDC_HREF}") # hrefs for Org and VDC that tests cluster operations test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG) test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org) TEST_ORG_HREF = test_org.href TEST_VDC_HREF = test_vdc.href logger.debug(f"Using test org {test_org.get_name()} " f"with href {TEST_ORG_HREF}") logger.debug(f"Using test vdc {test_vdc.name} with href {TEST_VDC_HREF}") create_cluster_admin_role(config['vcd'], logger=logger) create_cluster_author_role(config['vcd'], logger=logger)
def add_additional_details_to_config( config: Dict, vcd_host: str, vcd_username: str, vcd_password: str, verify_ssl: bool, is_legacy_mode: bool, is_mqtt_exchange: bool, log_wire: bool, log_wire_file: str ): """Update config dict with computed key-value pairs. :param dict config: :param str vcd_host: :param str vcd_username: :param str vcd_password: :param bool verify_ssl: :param bool is_legacy_mode: :param bool is_mqtt_exchange: :param bool log_wire: :param str log_wire_file: :return: the updated config file :rtype: dict """ # Compute common supported api versions by the CSE server and vCD sysadmin_client = None try: sysadmin_client = Client( vcd_host, verify_ssl_certs=verify_ssl, log_file=log_wire_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) sysadmin_client.set_credentials( BasicLoginCredentials( vcd_username, SYSTEM_ORG_NAME, vcd_password ) ) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS) common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() if is_legacy_mode: common_supported_api_versions = \ [x for x in common_supported_api_versions if VCDApiVersion(x) < VcdApiVersionObj.VERSION_35.value] else: common_supported_api_versions = \ [x for x in common_supported_api_versions if VCDApiVersion(x) >= VcdApiVersionObj.VERSION_35.value] config['service']['supported_api_versions'] = \ common_supported_api_versions finally: if sysadmin_client: sysadmin_client.logout() # Convert legacy_mode flag in service_section to corresponding # feature flags if 'feature_flags' not in config: config['feature_flags'] = {} config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode) config['feature_flags']['non_legacy_api'] = \ not str_to_bool(is_legacy_mode) # Compute the default api version as the max supported version # Also compute the RDE version in use max_vcd_api_version_supported: str = get_max_api_version(config['service']['supported_api_versions']) # noqa: E501 config['service']['default_api_version'] = max_vcd_api_version_supported config['service']['rde_version_in_use'] = semantic_version.Version( rde_utils.get_runtime_rde_version_by_vcd_api_version( max_vcd_api_version_supported ) ) # Update the config dict with telemetry specific key value pairs update_with_telemetry_settings( config_dict=config, vcd_host=vcd_host, vcd_username=vcd_username, vcd_password=vcd_password, verify_ssl=verify_ssl, is_mqtt_exchange=is_mqtt_exchange ) return config
def get_validated_config(config_file_name, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, log_wire_file=None, logger_debug=NULL_LOGGER, msg_update_callback=NullPrinter()): """Get the config file as a dictionary and check for validity. Ensures that all properties exist and all values are the expected type. Checks that AMQP connection is available, and vCD/VCs are valid. Does not guarantee that CSE has been installed according to this config file. :param str config_file_name: path to config file. :param str pks_config_file_name: path to PKS config file. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param str log_wire_file: log_wire_file to use if needed to wire log pyvcloud requests and responses :param logging.Logger logger_debug: logger to log with. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :return: CSE config :rtype: dict :raises KeyError: if config file has missing or extra properties. :raises TypeError: if the value type for a config file property is incorrect. :raises container_service_extension.exceptions.AmqpConnectionError: (when not using MQTT) if AMQP connection failed (host, password, port, username, vhost is invalid). :raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid. :raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or 'password' is invalid. :raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password' is invalid. """ check_file_permissions(config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(config_file_name) as config_file: config = yaml.safe_load(config_file) or {} else: msg_update_callback.info( f"Decrypting '{config_file_name}'") try: config = yaml.safe_load( get_decrypted_file_contents(config_file_name, decryption_password)) or {} except cryptography.fernet.InvalidToken: raise Exception(CONFIG_DECRYPTION_ERROR_MSG) msg_update_callback.info( f"Validating config file '{config_file_name}'") # This allows us to compare top-level config keys and value types use_mqtt = should_use_mqtt_protocol(config) sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \ else SAMPLE_MQTT_CONFIG sample_config = { **sample_message_queue_config, **SAMPLE_VCD_CONFIG, **SAMPLE_VCS_CONFIG, **SAMPLE_SERVICE_CONFIG, **SAMPLE_BROKER_CONFIG } log_wire = str_to_bool(config.get('service', {}).get('log_wire')) nsxt_wire_logger = NULL_LOGGER if not log_wire: log_wire_file = None nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER check_keys_and_value_types(config, sample_config, location='config file', msg_update_callback=msg_update_callback) # MQTT validation not required because no MQTT host, exchange, etc. # is needed in the config file since the server code creates and # registers the MQTT extension directly using server constants if not use_mqtt: _validate_amqp_config(config['amqp'], msg_update_callback) try: _validate_vcd_and_vcs_config(config['vcd'], config['vcs'], msg_update_callback, log_file=log_wire_file, log_wire=log_wire) except vim.fault.InvalidLogin: raise Exception(VCENTER_LOGIN_ERROR_MSG) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") except requests.exceptions.ConnectionError as err: raise Exception(f"Cannot connect to {err.request.url}.") _validate_broker_config(config['broker'], legacy_mode=config['service']['legacy_mode'], msg_update_callback=msg_update_callback, logger_debug=logger_debug) check_keys_and_value_types(config['service'], SAMPLE_SERVICE_CONFIG['service'], location="config file 'service' section", excluded_keys=['log_wire'], msg_update_callback=msg_update_callback) check_keys_and_value_types(config['service']['telemetry'], SAMPLE_SERVICE_CONFIG['service']['telemetry'], location="config file 'service->telemetry' " "section", msg_update_callback=msg_update_callback) msg_update_callback.general( f"Config file '{config_file_name}' is valid") if pks_config_file_name: check_file_permissions(pks_config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(pks_config_file_name) as f: pks_config = yaml.safe_load(f) or {} else: msg_update_callback.info( f"Decrypting '{pks_config_file_name}'") pks_config = yaml.safe_load( get_decrypted_file_contents(pks_config_file_name, decryption_password)) or {} msg_update_callback.info( f"Validating PKS config file '{pks_config_file_name}'") _validate_pks_config_structure(pks_config, msg_update_callback) try: _validate_pks_config_data_integrity(pks_config, msg_update_callback, logger_debug=logger_debug, logger_wire=nsxt_wire_logger) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") msg_update_callback.general( f"PKS Config file '{pks_config_file_name}' is valid") config['pks_config'] = pks_config else: config['pks_config'] = None # Compute common supported api versions by the CSE server and vCD sysadmin_client = None try: sysadmin_client = Client( config['vcd']['host'], verify_ssl_certs=config['vcd']['verify'], log_file=log_wire_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) sysadmin_client.set_credentials(BasicLoginCredentials( config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password'])) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS) common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() config['service']['supported_api_versions'] = \ common_supported_api_versions finally: if sysadmin_client: sysadmin_client.logout() # Convert legacy_mode flag in service_section to corresponding # feature flags is_legacy_mode = config['service']['legacy_mode'] if 'feature_flags' not in config: config['feature_flags'] = {} config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode) config['feature_flags']['non_legacy_api'] = \ not str_to_bool(is_legacy_mode) # Temporary work around before api version is completely removed from # config if is_legacy_mode: supported_api_versions_float = \ [float(x) for x in config['service']['supported_api_versions'] if float(x) < 35.0] else: supported_api_versions_float = \ [float(x) for x in config['service']['supported_api_versions'] if float(x) >= 35.0] config['vcd']['api_version'] = str(max(supported_api_versions_float)) # Store telemetry instance id, url and collector id in config # This steps needs to be done after api_version has been computed # and stored in the config store_telemetry_settings(config) return config