def _validate_vcd_config( vcd_dict, msg_update_callback=NullPrinter(), log_file=None, log_wire=False ): """Ensure that 'vcd' section of config is correct. Checks that * 'vcd' section of config has correct keys and value types. * vCD is accessible. :param dict vcd_dict: 'vcd' section of config file as a dict. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :param str log_file: log_file for pyvcloud wire log. :param bool log_wire: If pyvcloud requests should be logged. :raises KeyError: if @vcd_dict has missing :raises TypeError: if the value type for a @vcd_dict is incorrect. """ check_keys_and_value_types( vcd_dict, SAMPLE_VCD_CONFIG['vcd'], location="config file 'vcd' section", msg_update_callback=msg_update_callback ) if not vcd_dict['verify']: msg_update_callback.general( 'InsecureRequestWarning: Unverified HTTPS request is ' 'being made. Adding certificate verification is ' 'strongly advised.' ) requests.packages.urllib3.disable_warnings() client = None try: _validate_vcd_url_scheme(vcd_dict['host']) client = Client( vcd_dict['host'], verify_ssl_certs=vcd_dict['verify'], log_file=log_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) client.set_credentials( BasicLoginCredentials( vcd_dict['username'], SYSTEM_ORG_NAME, vcd_dict['password'] ) ) msg_update_callback.general( "Connected to vCloud Director " f"({vcd_dict['host']}:{vcd_dict['port']})" ) finally: if client is not None: client.logout()
def _validate_broker_config( broker_dict, legacy_mode=False, msg_update_callback=NullPrinter(), logger_debug=NULL_LOGGER ): """Ensure that 'broker' section of config is correct. Checks that 'broker' section of config has correct keys and value types. Also checks that 'default_broker' property is a valid template. :param dict broker_dict: 'broker' section of config file as a dict. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :raises KeyError: if @broker_dict has missing properties. :raises TypeError: if the value type for a @broker_dict property is incorrect. :raises ValueError: if 'ip_allocation_mode' is not 'dhcp' or 'pool'. Or if remote_template_cookbook_url is invalid. """ check_keys_and_value_types( broker_dict, SAMPLE_BROKER_CONFIG['broker'], location="config file 'broker' section", msg_update_callback=msg_update_callback ) valid_ip_allocation_modes = [ 'dhcp', 'pool' ] if broker_dict['ip_allocation_mode'] not in valid_ip_allocation_modes: raise ValueError( "IP allocation mode is " f"'{broker_dict['ip_allocation_mode']}' when it " "should be either 'dhcp' or 'pool'" ) rtm = RemoteTemplateManager( remote_template_cookbook_url=broker_dict['remote_template_cookbook_url'], # noqa: E501 legacy_mode=legacy_mode, logger=logger_debug ) remote_template_cookbook = rtm.get_filtered_remote_template_cookbook() if not remote_template_cookbook: raise Exception("Remote template cookbook is invalid.")
def _validate_amqp_config(amqp_dict, msg_update_callback=NullPrinter()): """Ensure that 'amqp' section of config is correct. Checks that 'amqp' section of config has correct keys and value types. Also ensures that connection to AMQP server is valid. :param dict amqp_dict: 'amqp' section of config file as a dict. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :raises KeyError: if @amqp_dict has missing or extra properties. :raises TypeError: if the value type for an @amqp_dict property is incorrect. :raises AmqpConnectionError: if AMQP connection failed. """ check_keys_and_value_types(amqp_dict, SAMPLE_AMQP_CONFIG['amqp'], location="config file 'amqp' section", msg_update_callback=msg_update_callback) credentials = pika.PlainCredentials(amqp_dict['username'], amqp_dict['password']) parameters = pika.ConnectionParameters(amqp_dict['host'], amqp_dict['port'], amqp_dict['vhost'], credentials, connection_attempts=3, retry_delay=2, socket_timeout=5) connection = None try: connection = pika.BlockingConnection(parameters) msg_update_callback.general( "Connected to AMQP server " f"({amqp_dict['host']}:{amqp_dict['port']})") except Exception as err: raise AmqpConnectionError(f"AMQP server error : {str(err)}") finally: if connection is not None: connection.close()
def _validate_pks_config_structure( pks_config, msg_update_callback=NullPrinter() ): sample_config = { **SAMPLE_PKS_SERVERS_SECTION, **SAMPLE_PKS_ACCOUNTS_SECTION, **SAMPLE_PKS_ORGS_SECTION, **SAMPLE_PKS_PVDCS_SECTION, **SAMPLE_PKS_NSXT_SERVERS_SECTION } check_keys_and_value_types( pks_config, sample_config, location='pks config file', excluded_keys=[PKS_ORGS_SECTION_KEY], msg_update_callback=msg_update_callback ) pks_servers = pks_config[PKS_SERVERS_SECTION_KEY] for index, pks_server in enumerate(pks_servers, 1): check_keys_and_value_types( pks_server, SAMPLE_PKS_SERVERS_SECTION[PKS_SERVERS_SECTION_KEY][0], location=f"pks config file '{PKS_SERVERS_SECTION_KEY}' " f"section, pks server #{index}", excluded_keys=['proxy'], msg_update_callback=msg_update_callback ) pks_accounts = pks_config[PKS_ACCOUNTS_SECTION_KEY] for index, pks_account in enumerate(pks_accounts, 1): check_keys_and_value_types( pks_account, SAMPLE_PKS_ACCOUNTS_SECTION[PKS_ACCOUNTS_SECTION_KEY][0], location=f"pks config file '{PKS_ACCOUNTS_SECTION_KEY}' " f"section, pks account #{index}", msg_update_callback=msg_update_callback ) if PKS_ORGS_SECTION_KEY in pks_config.keys(): orgs = pks_config[PKS_ORGS_SECTION_KEY] for index, org in enumerate(orgs, 1): check_keys_and_value_types( org, SAMPLE_PKS_ORGS_SECTION[PKS_ORGS_SECTION_KEY][0], location=f"pks config file '{PKS_ORGS_SECTION_KEY}' " f"section, org #{index}", msg_update_callback=msg_update_callback ) pvdcs = pks_config[PKS_PVDCS_SECTION_KEY] for index, pvdc in enumerate(pvdcs, 1): check_keys_and_value_types( pvdc, SAMPLE_PKS_PVDCS_SECTION[PKS_PVDCS_SECTION_KEY][0], location=f"pks config file '{PKS_PVDCS_SECTION_KEY}' " f"section, pvdc #{index}", msg_update_callback=msg_update_callback ) nsxt_servers = pks_config[PKS_NSXT_SERVERS_SECTION_KEY] for index, nsxt_server in enumerate(nsxt_servers, 1): check_keys_and_value_types( nsxt_server, SAMPLE_PKS_NSXT_SERVERS_SECTION[PKS_NSXT_SERVERS_SECTION_KEY][0], location=f"pks config file '{PKS_NSXT_SERVERS_SECTION_KEY}' " f"section, nsxt server #{index}", excluded_keys=['proxy'], msg_update_callback=msg_update_callback )
def get_validated_config( config_file_name, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, log_wire_file=None, logger_debug=NULL_LOGGER, msg_update_callback=NullPrinter() ): """Get the config file as a dictionary and check for validity. Ensures that all properties exist and all values are the expected type. Checks that AMQP connection is available, and vCD/VCs are valid. Does not guarantee that CSE has been installed according to this config file. Additionally populates certain key-value pairs in the config dict to avoid repeated computation of those e.g. supported api versions, feature flags, RDE version in use etc. :param str config_file_name: path to config file. :param str pks_config_file_name: path to PKS config file. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param str log_wire_file: log_wire_file to use if needed to wire log pyvcloud requests and responses :param logging.Logger logger_debug: logger to log with. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :return: CSE config :rtype: dict :raises KeyError: if config file has missing or extra properties. :raises TypeError: if the value type for a config file property is incorrect. :raises container_service_extension.exceptions.AmqpConnectionError: (when not using MQTT) if AMQP connection failed (host, password, port, username, vhost is invalid). :raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid. :raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or 'password' is invalid. :raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password' is invalid. """ check_file_permissions(config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(config_file_name) as config_file: config = yaml.safe_load(config_file) or {} else: msg_update_callback.info( f"Decrypting '{config_file_name}'") try: config = yaml.safe_load( get_decrypted_file_contents( config_file_name, decryption_password ) ) or {} except cryptography.fernet.InvalidToken: raise Exception(CONFIG_DECRYPTION_ERROR_MSG) msg_update_callback.info( f"Validating config file '{config_file_name}'" ) is_no_vc_communication_mode = \ server_utils.is_no_vc_communication_mode(ServerConfig(config)) use_mqtt = server_utils.should_use_mqtt_protocol(ServerConfig(config)) sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \ else SAMPLE_MQTT_CONFIG # This allows us to compare top-level config keys and value types sample_config = { **sample_message_queue_config, **SAMPLE_VCD_CONFIG, **SAMPLE_SERVICE_CONFIG, **SAMPLE_BROKER_CONFIG } if not is_no_vc_communication_mode: sample_config.update(SAMPLE_VCS_CONFIG) else: if 'vcs' in config: del config['vcs'] log_wire = str_to_bool(config.get('service', {}).get('log_wire')) nsxt_wire_logger = NULL_LOGGER if not log_wire: log_wire_file = None nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER check_keys_and_value_types( config, sample_config, location='config file', msg_update_callback=msg_update_callback ) # MQTT validation not required because no MQTT host, exchange, etc. # is needed in the config file since the server code creates and # registers the MQTT extension directly using server constants if not use_mqtt: _validate_amqp_config(config['amqp'], msg_update_callback) # Validation of service properties is done first as those properties are # used in broker validation. check_keys_and_value_types( config['service'], SAMPLE_SERVICE_CONFIG['service'], location="config file 'service' section", excluded_keys=['log_wire'], msg_update_callback=msg_update_callback ) try: if is_no_vc_communication_mode: _validate_vcd_config( config['vcd'], msg_update_callback, log_file=log_wire_file, log_wire=log_wire ) else: _validate_vcd_and_vcs_config( config['vcd'], config['vcs'], msg_update_callback, log_file=log_wire_file, log_wire=log_wire ) except vim.fault.InvalidLogin: raise Exception(VCENTER_LOGIN_ERROR_MSG) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") except requests.exceptions.ConnectionError as err: raise Exception(f"Cannot connect to {err.request.url}.") check_keys_and_value_types( config['service']['telemetry'], SAMPLE_SERVICE_CONFIG['service']['telemetry'], location="config file 'service->telemetry' section", msg_update_callback=msg_update_callback ) _validate_broker_config( config['broker'], legacy_mode=config['service']['legacy_mode'], msg_update_callback=msg_update_callback, logger_debug=logger_debug ) config = add_additional_details_to_config( config=config, vcd_host=config['vcd']['host'], vcd_username=config['vcd']['username'], vcd_password=config['vcd']['password'], verify_ssl=config['vcd']['verify'], is_legacy_mode=config['service']['legacy_mode'], is_mqtt_exchange=server_utils.should_use_mqtt_protocol( ServerConfig(config) ), log_wire=log_wire, log_wire_file=log_wire_file ) _raise_error_if_amqp_not_supported( use_mqtt, config['service']['default_api_version'], logger=logger_debug ) msg_update_callback.general( f"Config file '{config_file_name}' is valid" ) if pks_config_file_name: check_file_permissions(pks_config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(pks_config_file_name) as f: pks_config = yaml.safe_load(f) or {} else: msg_update_callback.info( f"Decrypting '{pks_config_file_name}'") pks_config = yaml.safe_load( get_decrypted_file_contents(pks_config_file_name, decryption_password)) or {} msg_update_callback.info( f"Validating PKS config file '{pks_config_file_name}'") _validate_pks_config_structure(pks_config, msg_update_callback) try: _validate_pks_config_data_integrity(pks_config, msg_update_callback, logger_debug=logger_debug, logger_wire=nsxt_wire_logger) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") msg_update_callback.general( f"PKS Config file '{pks_config_file_name}' is valid") config['pks_config'] = pks_config else: config['pks_config'] = None return config
def _validate_vcd_and_vcs_config( vcd_dict, vcs, msg_update_callback=NullPrinter(), log_file=None, log_wire=False ): """Ensure that 'vcd' and vcs' section of config are correct. Checks that * 'vcd' and 'vcs' section of config have correct keys and value types. * vCD and all registered VCs in vCD are accessible. * api version specified for vcd is supported by CSE. :param dict vcd_dict: 'vcd' section of config file as a dict. :param list vcs: 'vcs' section of config file as a list of dicts. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :param str log_file: log_file for pyvcloud wire log. :param bool log_wire: If pyvcloud requests should be logged. :raises KeyError: if @vcd_dict or a vc in @vcs has missing or extra properties. :raises TypeError: if the value type for a @vcd_dict or vc property is incorrect. :raises ValueError: if vCD has a VC that is not listed in the config file. """ check_keys_and_value_types(vcd_dict, SAMPLE_VCD_CONFIG['vcd'], location="config file 'vcd' section", msg_update_callback=msg_update_callback) if not vcd_dict['verify']: msg_update_callback.general( 'InsecureRequestWarning: Unverified HTTPS request is ' 'being made. Adding certificate verification is ' 'strongly advised.' ) requests.packages.urllib3.disable_warnings() client = None try: _validate_vcd_url_scheme(vcd_dict['host']) client = Client( vcd_dict['host'], verify_ssl_certs=vcd_dict['verify'], log_file=log_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) client.set_credentials( BasicLoginCredentials( vcd_dict['username'], SYSTEM_ORG_NAME, vcd_dict['password'] ) ) msg_update_callback.general( "Connected to vCloud Director " f"({vcd_dict['host']}:{vcd_dict['port']})" ) for index, vc in enumerate(vcs, 1): check_keys_and_value_types( vc, SAMPLE_VCS_CONFIG['vcs'][0], location=f"config file 'vcs' section, vc #{index}", msg_update_callback=msg_update_callback ) # Check that all registered VCs in vCD are listed in config file platform = Platform(client) config_vc_names = [vc['name'] for vc in vcs] for platform_vc in platform.list_vcenters(): platform_vc_name = platform_vc.get('name') if platform_vc_name not in config_vc_names: raise ValueError( f"vCenter '{platform_vc_name}' registered in " "vCD but not found in config file" ) # Check that all VCs listed in config file are registered in vCD for vc in vcs: vcenter = platform.get_vcenter(vc['name']) if not (hasattr(vcenter, 'IsConnected') and vcenter.IsConnected): msg = f"vCenter Server '{vc['name']}' not available" msg_update_callback.info(msg) continue vsphere_url = urlparse(vcenter.Url.text) vsphere_url_port = vsphere_url.port if vsphere_url_port: v = VSphere( vsphere_url.hostname, vc['username'], vc['password'], vsphere_url.port ) else: v = VSphere( vsphere_url.hostname, vc['username'], vc['password'] ) v.connect() msg = f"Connected to vCenter Server '{vc['name']}' as " \ f"'{vc['username']}' ({vsphere_url.hostname}" if vsphere_url_port: msg += f":{vsphere_url.port}" msg += ")" msg_update_callback.general(msg) finally: if client is not None: client.logout()
def get_validated_config(config_file_name, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, log_wire_file=None, logger_debug=NULL_LOGGER, msg_update_callback=NullPrinter()): """Get the config file as a dictionary and check for validity. Ensures that all properties exist and all values are the expected type. Checks that AMQP connection is available, and vCD/VCs are valid. Does not guarantee that CSE has been installed according to this config file. :param str config_file_name: path to config file. :param str pks_config_file_name: path to PKS config file. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param str log_wire_file: log_wire_file to use if needed to wire log pyvcloud requests and responses :param logging.Logger logger_debug: logger to log with. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :return: CSE config :rtype: dict :raises KeyError: if config file has missing or extra properties. :raises TypeError: if the value type for a config file property is incorrect. :raises container_service_extension.exceptions.AmqpConnectionError: (when not using MQTT) if AMQP connection failed (host, password, port, username, vhost is invalid). :raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid. :raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or 'password' is invalid. :raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password' is invalid. """ check_file_permissions(config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(config_file_name) as config_file: config = yaml.safe_load(config_file) or {} else: msg_update_callback.info( f"Decrypting '{config_file_name}'") try: config = yaml.safe_load( get_decrypted_file_contents(config_file_name, decryption_password)) or {} except cryptography.fernet.InvalidToken: raise Exception(CONFIG_DECRYPTION_ERROR_MSG) msg_update_callback.info( f"Validating config file '{config_file_name}'") # This allows us to compare top-level config keys and value types use_mqtt = should_use_mqtt_protocol(config) sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \ else SAMPLE_MQTT_CONFIG sample_config = { **sample_message_queue_config, **SAMPLE_VCD_CONFIG, **SAMPLE_VCS_CONFIG, **SAMPLE_SERVICE_CONFIG, **SAMPLE_BROKER_CONFIG } log_wire = str_to_bool(config.get('service', {}).get('log_wire')) nsxt_wire_logger = NULL_LOGGER if not log_wire: log_wire_file = None nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER check_keys_and_value_types(config, sample_config, location='config file', msg_update_callback=msg_update_callback) # MQTT validation not required because no MQTT host, exchange, etc. # is needed in the config file since the server code creates and # registers the MQTT extension directly using server constants if not use_mqtt: _validate_amqp_config(config['amqp'], msg_update_callback) try: _validate_vcd_and_vcs_config(config['vcd'], config['vcs'], msg_update_callback, log_file=log_wire_file, log_wire=log_wire) except vim.fault.InvalidLogin: raise Exception(VCENTER_LOGIN_ERROR_MSG) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") except requests.exceptions.ConnectionError as err: raise Exception(f"Cannot connect to {err.request.url}.") _validate_broker_config(config['broker'], legacy_mode=config['service']['legacy_mode'], msg_update_callback=msg_update_callback, logger_debug=logger_debug) check_keys_and_value_types(config['service'], SAMPLE_SERVICE_CONFIG['service'], location="config file 'service' section", excluded_keys=['log_wire'], msg_update_callback=msg_update_callback) check_keys_and_value_types(config['service']['telemetry'], SAMPLE_SERVICE_CONFIG['service']['telemetry'], location="config file 'service->telemetry' " "section", msg_update_callback=msg_update_callback) msg_update_callback.general( f"Config file '{config_file_name}' is valid") if pks_config_file_name: check_file_permissions(pks_config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(pks_config_file_name) as f: pks_config = yaml.safe_load(f) or {} else: msg_update_callback.info( f"Decrypting '{pks_config_file_name}'") pks_config = yaml.safe_load( get_decrypted_file_contents(pks_config_file_name, decryption_password)) or {} msg_update_callback.info( f"Validating PKS config file '{pks_config_file_name}'") _validate_pks_config_structure(pks_config, msg_update_callback) try: _validate_pks_config_data_integrity(pks_config, msg_update_callback, logger_debug=logger_debug, logger_wire=nsxt_wire_logger) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") msg_update_callback.general( f"PKS Config file '{pks_config_file_name}' is valid") config['pks_config'] = pks_config else: config['pks_config'] = None # Compute common supported api versions by the CSE server and vCD sysadmin_client = None try: sysadmin_client = Client( config['vcd']['host'], verify_ssl_certs=config['vcd']['verify'], log_file=log_wire_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) sysadmin_client.set_credentials(BasicLoginCredentials( config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password'])) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS) common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() config['service']['supported_api_versions'] = \ common_supported_api_versions finally: if sysadmin_client: sysadmin_client.logout() # Convert legacy_mode flag in service_section to corresponding # feature flags is_legacy_mode = config['service']['legacy_mode'] if 'feature_flags' not in config: config['feature_flags'] = {} config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode) config['feature_flags']['non_legacy_api'] = \ not str_to_bool(is_legacy_mode) # Temporary work around before api version is completely removed from # config if is_legacy_mode: supported_api_versions_float = \ [float(x) for x in config['service']['supported_api_versions'] if float(x) < 35.0] else: supported_api_versions_float = \ [float(x) for x in config['service']['supported_api_versions'] if float(x) >= 35.0] config['vcd']['api_version'] = str(max(supported_api_versions_float)) # Store telemetry instance id, url and collector id in config # This steps needs to be done after api_version has been computed # and stored in the config store_telemetry_settings(config) return config