def _validate_vcd_config( vcd_dict, msg_update_callback=NullPrinter(), log_file=None, log_wire=False ): """Ensure that 'vcd' section of config is correct. Checks that * 'vcd' section of config has correct keys and value types. * vCD is accessible. :param dict vcd_dict: 'vcd' section of config file as a dict. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :param str log_file: log_file for pyvcloud wire log. :param bool log_wire: If pyvcloud requests should be logged. :raises KeyError: if @vcd_dict has missing :raises TypeError: if the value type for a @vcd_dict is incorrect. """ check_keys_and_value_types( vcd_dict, SAMPLE_VCD_CONFIG['vcd'], location="config file 'vcd' section", msg_update_callback=msg_update_callback ) if not vcd_dict['verify']: msg_update_callback.general( 'InsecureRequestWarning: Unverified HTTPS request is ' 'being made. Adding certificate verification is ' 'strongly advised.' ) requests.packages.urllib3.disable_warnings() client = None try: _validate_vcd_url_scheme(vcd_dict['host']) client = Client( vcd_dict['host'], verify_ssl_certs=vcd_dict['verify'], log_file=log_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) client.set_credentials( BasicLoginCredentials( vcd_dict['username'], SYSTEM_ORG_NAME, vcd_dict['password'] ) ) msg_update_callback.general( "Connected to vCloud Director " f"({vcd_dict['host']}:{vcd_dict['port']})" ) finally: if client is not None: client.logout()
def get_telemetry_instance_id(vcd_host: str, vcd_username: str, vcd_password: str, verify_ssl: bool, is_mqtt_exchange: bool, logger_debug=NULL_LOGGER, msg_update_callback=NullPrinter()): """Get CSE AMQP or MQTT extension id which is used as instance id. Any exception is logged as error. No exception is leaked out of this method and does not affect the server startup. :param str vcd_host: :param str vcd_username: :param str vcd_password: :param bool verify_ssl: :param bool is_mqtt_exchange: :param logging.logger logger_debug: logger instance to log any error in retrieving CSE extension id. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :return instance id to use for sending data to Vmware telemetry server :rtype str (unless no instance id found) """ client = None try: client = Client(vcd_host, verify_ssl_certs=verify_ssl) client.set_credentials( BasicLoginCredentials(vcd_username, SYSTEM_ORG_NAME, vcd_password)) if is_mqtt_exchange: # Get MQTT extension uuid mqtt_ext_manager = MQTTExtensionManager(client) ext_info = mqtt_ext_manager.get_extension_info( ext_name=CSE_SERVICE_NAME, ext_version=MQTT_EXTENSION_VERSION, ext_vendor=MQTT_EXTENSION_VENDOR) if not ext_info: logger_debug.debug("Failed to retrieve telemetry instance id") return None logger_debug.debug("Retrieved telemetry instance id") return mqtt_ext_manager.get_extension_uuid( ext_info[MQTTExtKey.EXT_URN_ID]) else: # Get AMQP extension id ext = APIExtension(client) cse_info = ext.get_extension_info(CSE_SERVICE_NAME, namespace=CSE_SERVICE_NAMESPACE) logger_debug.debug("Retrieved telemetry instance id") return cse_info.get('id') except Exception as err: msg = f"Cannot retrieve telemetry instance id:{err}" msg_update_callback.general(msg) logger_debug.error(msg, exc_info=True) finally: if client is not None: client.logout()
def read_native_template_definition_from_catalog( config: ServerConfig, msg_update_callback=utils.NullPrinter() ): # NOTE: If `enable_tkg_plus` in the config file is set to false, # CSE server will skip loading the TKG+ template this will prevent # users from performing TKG+ related operations. msg = "Loading k8s template definition from catalog" logger.SERVER_LOGGER.info(msg) msg_update_callback.general_no_color(msg) client = None try: log_filename = None log_wire = \ utils.str_to_bool(config.get_value_at('service.log_wire')) if log_wire: log_filename = logger.SERVER_DEBUG_WIRELOG_FILEPATH client = Client( uri=config.get_value_at('vcd.host'), api_version=config.get_value_at('service.default_api_version'), # noqa: E501 verify_ssl_certs=config.get_value_at('vcd.verify'), log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) credentials = BasicLoginCredentials( config.get_value_at('vcd.username'), shared_constants.SYSTEM_ORG_NAME, config.get_value_at('vcd.password') ) client.set_credentials(credentials) legacy_mode = config.get_value_at('service.legacy_mode') org_name = config.get_value_at('broker.org') catalog_name = config.get_value_at('broker.catalog') k8_templates = ltm.get_valid_k8s_local_template_definition( client=client, catalog_name=catalog_name, org_name=org_name, legacy_mode=legacy_mode, is_tkg_plus_enabled=server_utils.is_tkg_plus_enabled(config), logger_debug=logger.SERVER_LOGGER, msg_update_callback=msg_update_callback) return k8_templates finally: if client: client.logout()
def read_tkgm_template_definition_from_catalog( config: ServerConfig, msg_update_callback=utils.NullPrinter() ): msg = "Loading TKGm template definition from catalog" logger.SERVER_LOGGER.info(msg) msg_update_callback.general_no_color(msg) client = None try: log_filename = None log_wire = utils.str_to_bool( config.get_value_at('service.log_wire') ) if log_wire: log_filename = logger.SERVER_DEBUG_WIRELOG_FILEPATH client = Client( uri=config.get_value_at('vcd.host'), api_version=config.get_value_at('service.default_api_version'), # noqa: E501 verify_ssl_certs=config.get_value_at('vcd.verify'), log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) credentials = BasicLoginCredentials( config.get_value_at('vcd.username'), shared_constants.SYSTEM_ORG_NAME, config.get_value_at('vcd.password') ) client.set_credentials(credentials) org_name = config.get_value_at('broker.org') catalog_name = config.get_value_at('broker.catalog') tkgm_templates = ttm.read_all_tkgm_template( client=client, org_name=org_name, catalog_name=catalog_name, logger=logger.SERVER_LOGGER, msg_update_callback=msg_update_callback ) return tkgm_templates finally: if client: client.logout()
def get_telemetry_instance_id(config_dict, logger_debug=NULL_LOGGER, msg_update_callback=NullPrinter()): """Get CSE AMQP or MQTT extension id which is used as instance id. Any exception is logged as error. No exception is leaked out of this method and does not affect the server startup. :param dict config_dict: CSE configuration :param logging.logger logger_debug: logger instance to log any error in retrieving CSE extension id. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :return instance id to use for sending data to Vmware telemetry server :rtype str (unless no instance id found) """ vcd = config_dict['vcd'] try: client = Client(vcd['host'], api_version=vcd['api_version'], verify_ssl_certs=vcd['verify']) client.set_credentials(BasicLoginCredentials( vcd['username'], SYSTEM_ORG_NAME, vcd['password'])) if should_use_mqtt_protocol(config_dict): # Get MQTT extension uuid mqtt_ext_manager = MQTTExtensionManager(client) ext_info = mqtt_ext_manager.get_extension_info( ext_name=CSE_SERVICE_NAME, ext_version=MQTT_EXTENSION_VERSION, ext_vendor=MQTT_EXTENSION_VENDOR) if not ext_info: return None return mqtt_ext_manager.get_extension_uuid( ext_info[MQTTExtKey.EXT_URN_ID]) else: # Get AMQP extension id ext = APIExtension(client) cse_info = ext.get_extension_info(CSE_SERVICE_NAME, namespace=CSE_SERVICE_NAMESPACE) logger_debug.info("Retrieved telemetry instance id") return cse_info.get('id') except Exception as err: msg = f"Cannot retrieve telemetry instance id:{err}" msg_update_callback.general(msg) logger_debug.error(msg) finally: if client is not None: client.logout()
def get_telemetry_instance_id(vcd, logger_instance=None, msg_update_callback=None): """Get CSE extension id which is used as instance id. Any exception is logged as error. No exception is leaked out of this method and does not affect the server startup. :param dict vcd: 'vcd' section of config file as a dict. :param logging.logger logger_instance: logger instance to log any error in retrieving CSE extension id. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object that writes messages onto console. :return instance id to use for sending data to Vmware telemetry server :rtype str :raises Exception: if any exception happens while retrieving CSE extension id """ try: client = Client(vcd['host'], api_version=vcd['api_version'], verify_ssl_certs=vcd['verify']) client.set_credentials( BasicLoginCredentials(vcd['username'], SYSTEM_ORG_NAME, vcd['password'])) ext = APIExtension(client) cse_info = ext.get_extension_info(CSE_SERVICE_NAME, namespace=CSE_SERVICE_NAMESPACE) if logger_instance: logger_instance.info("Retrieved telemetry instance id") return cse_info.get('id') except Exception as err: msg = f"Cannot retrieve telemetry instance id:{err}" if msg_update_callback: msg_update_callback.general(msg) if logger_instance: logger_instance.error(msg) finally: if client is not None: client.logout()
def _validate_vcd_and_vcs_config(vcd_dict, vcs, msg_update_callback=None): """Ensure that 'vcd' and vcs' section of config are correct. Checks that 'vcd' and 'vcs' section of config have correct keys and value types. Also checks that vCD and all registered VCs in vCD are accessible. :param dict vcd_dict: 'vcd' section of config file as a dict. :param list vcs: 'vcs' section of config file as a list of dicts. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object that writes messages onto console. :raises KeyError: if @vcd_dict or a vc in @vcs has missing or extra properties. :raises TypeError: if the value type for a @vcd_dict or vc property is incorrect. :raises ValueError: if vCD has a VC that is not listed in the config file. """ check_keys_and_value_types(vcd_dict, SAMPLE_VCD_CONFIG['vcd'], location="config file 'vcd' section", msg_update_callback=msg_update_callback) if not vcd_dict['verify']: if msg_update_callback: msg_update_callback.general( 'InsecureRequestWarning: Unverified HTTPS request is ' 'being made. Adding certificate verification is ' 'strongly advised.') requests.packages.urllib3.disable_warnings() client = None try: # TODO() we get an error during client initialization if the specified # logfile points to the directory which doesn't exist. This issue # should be fixed in pyvcloud, where the logging setup creates # directories used in the log filepath if they do not exist yet. setup_log_file_directory() client = Client(vcd_dict['host'], api_version=vcd_dict['api_version'], verify_ssl_certs=vcd_dict['verify'], log_file=SERVER_DEBUG_WIRELOG_FILEPATH, log_requests=True, log_headers=True, log_bodies=True) client.set_credentials( BasicLoginCredentials(vcd_dict['username'], SYSTEM_ORG_NAME, vcd_dict['password'])) if msg_update_callback: msg_update_callback.general( "Connected to vCloud Director " f"({vcd_dict['host']}:{vcd_dict['port']})") for index, vc in enumerate(vcs, 1): check_keys_and_value_types( vc, SAMPLE_VCS_CONFIG['vcs'][0], location=f"config file 'vcs' section, vc #{index}", msg_update_callback=msg_update_callback) # Check that all registered VCs in vCD are listed in config file platform = Platform(client) config_vc_names = [vc['name'] for vc in vcs] for platform_vc in platform.list_vcenters(): platform_vc_name = platform_vc.get('name') if platform_vc_name not in config_vc_names: raise ValueError(f"vCenter '{platform_vc_name}' registered in " f"vCD but not found in config file") # Check that all VCs listed in config file are registered in vCD for vc in vcs: vcenter = platform.get_vcenter(vc['name']) vsphere_url = urlparse(vcenter.Url.text) v = VSphere(vsphere_url.hostname, vc['username'], vc['password'], vsphere_url.port) v.connect() if msg_update_callback: msg_update_callback.general( f"Connected to vCenter Server '{vc['name']}' as " f"'{vc['username']}' ({vsphere_url.hostname}:" f"{vsphere_url.port})") finally: if client is not None: client.logout()
sys.exit(1) vcd_host = sys.argv[1] org = sys.argv[2] user = sys.argv[3] password = sys.argv[4] # Disable warnings from self-signed certificates. requests.packages.urllib3.disable_warnings() # Login. SSL certificate verification is turned off to allow self-signed # certificates. You should only do this in trusted environments. print("Logging in: host={0}, org={1}, user={2}".format(vcd_host, org, user)) client = Client(vcd_host, api_version='29.0', verify_ssl_certs=False, log_file='pyvcloud.log', log_requests=True, log_headers=True, log_bodies=True) client.set_credentials(BasicLoginCredentials(user, org, password)) print("Fetching vCD installation info...") results = client.get_resource( client._session_endpoints[_WellKnownEndpoint.ADMIN]) for k, v in results.items(): print("Key: {0} Value: {1}".format(k, v)) # Log out. print("Logging out") client.logout()
def _process_template_compute_policy_compliance(self, msg_update_callback=None): msg = "Processing compute policy for k8s templates." LOGGER.info(msg) if msg_update_callback: msg_update_callback.general_no_color(msg) log_filename = None log_wire = str_to_bool(self.config['service'].get('log_wire')) if log_wire: log_filename = SERVER_DEBUG_WIRELOG_FILEPATH org_name = self.config['broker']['org'] catalog_name = self.config['broker']['catalog'] api_version = self.config['vcd']['api_version'] client = None try: if float(api_version) >= float( ApiVersion.VERSION_32.value): # noqa: E501 # TODO this api version 32 client should be removed once # vcd can handle cp removal/replacement on version 33 client = Client(self.config['vcd']['host'], api_version=ApiVersion.VERSION_32.value, verify_ssl_certs=self.config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) else: client = Client(self.config['vcd']['host'], api_version=api_version, verify_ssl_certs=self.config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(self.config['vcd']['username'], SYSTEM_ORG_NAME, self.config['vcd']['password']) client.set_credentials(credentials) cpm = ComputePolicyManager(client) try: for template in self.config['broker']['templates']: policy_name = template[LocalTemplateKey.COMPUTE_POLICY] catalog_item_name = template[ LocalTemplateKey.CATALOG_ITEM_NAME] # noqa: E501 # if policy name is not empty, stamp it on the template if policy_name: policy = cpm.get_policy(policy_name=policy_name) # create the policy if not present in system if not policy: msg = "Creating missing compute policy " \ f"'{policy_name}'." if msg_update_callback: msg_update_callback.info(msg) LOGGER.debug(msg) policy = cpm.add_policy(policy_name=policy_name) msg = f"Assigning compute policy '{policy_name}' to " \ f"template '{catalog_item_name}'." if msg_update_callback: msg_update_callback.general(msg) LOGGER.debug(msg) cpm.assign_compute_policy_to_vapp_template_vms( compute_policy_href=policy['href'], org_name=org_name, catalog_name=catalog_name, catalog_item_name=catalog_item_name) else: # empty policy name means we should remove policy from # template msg = f"Removing compute policy from template " \ f"'{catalog_item_name}'." if msg_update_callback: msg_update_callback.general(msg) LOGGER.debug(msg) cpm.remove_all_compute_policies_from_vapp_template_vms( org_name=org_name, catalog_name=catalog_name, catalog_item_name=catalog_item_name) except OperationNotSupportedException: msg = "Compute policy not supported by vCD. Skipping " \ "assigning/removing it to/from templates." if msg_update_callback: msg_update_callback.info(msg) LOGGER.debug(msg) finally: if client: client.logout()
def install_template(template_name, template_revision, config_file_name, force_create, retain_temp_vapp, ssh_key, skip_config_decryption=False, decryption_password=None, msg_update_callback=None): """Install a particular template in CSE. If template_name and revision are wild carded to *, all templates defined in remote template cookbook will be installed. :param str template_name: :param str template_revision: :param str config_file_name: config file name. :param bool force_create: if True and template already exists in vCD, overwrites existing template. :param str ssh_key: public ssh key to place into template vApp(s). :param bool retain_temp_vapp: if True, temporary vApp will not destroyed, so the user can ssh into and debug the vm. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object that writes messages onto console. """ configure_install_logger() config = get_validated_config( config_file_name, skip_config_decryption=skip_config_decryption, decryption_password=decryption_password, msg_update_callback=msg_update_callback) populate_vsphere_list(config['vcs']) msg = f"Installing template '{template_name}' at revision " \ f"'{template_revision}' on vCloud Director using config file " \ f"'{config_file_name}'" if msg_update_callback: msg_update_callback.info(msg) LOGGER.info(msg) client = None try: log_filename = None log_wire = str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = INSTALL_WIRELOG_FILEPATH client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" if msg_update_callback: msg_update_callback.general(msg) LOGGER.info(msg) # read remote template cookbook rtm = RemoteTemplateManager( remote_template_cookbook_url=config['broker'] ['remote_template_cookbook_url'], # noqa: E501 logger=LOGGER, msg_update_callback=msg_update_callback) remote_template_cookbook = rtm.get_remote_template_cookbook() found_template = False for template in remote_template_cookbook['templates']: template_name_matched = template_name in ( template[RemoteTemplateKey.NAME], '*') # noqa: E501 template_revision_matched = str(template_revision) in (str( template[RemoteTemplateKey.REVISION]), '*') # noqa: E501 if template_name_matched and template_revision_matched: found_template = True _install_template( client=client, remote_template_manager=rtm, template=template, org_name=config['broker']['org'], vdc_name=config['broker']['vdc'], catalog_name=config['broker']['catalog'], network_name=config['broker']['network'], ip_allocation_mode=config['broker']['ip_allocation_mode'], storage_profile=config['broker']['storage_profile'], force_update=force_create, retain_temp_vapp=retain_temp_vapp, ssh_key=ssh_key, msg_update_callback=msg_update_callback) if not found_template: msg = f"Template '{template_name}' at revision " \ f"'{template_revision}' not found in remote template " \ "cookbook." if msg_update_callback: msg_update_callback.error(msg) LOGGER.error(msg, exc_info=True) except Exception: if msg_update_callback: msg_update_callback.error( "Template Installation Error. Check CSE install logs") LOGGER.error("Template Installation Error", exc_info=True) finally: if client is not None: client.logout()
def _load_template_definition_from_catalog( self, msg_update_callback=utils.NullPrinter()): # noqa: E501 # NOTE: If `enable_tkg_plus` in the config file is set to false, # CSE server will skip loading the TKG+ template this will prevent # users from performing TKG+ related operations. msg = "Loading k8s template definition from catalog" logger.SERVER_LOGGER.info(msg) msg_update_callback.general_no_color(msg) client = None try: log_filename = None log_wire = \ utils.str_to_bool(self.config['service'].get('log_wire')) if log_wire: log_filename = logger.SERVER_DEBUG_WIRELOG_FILEPATH client = Client(self.config['vcd']['host'], api_version=self.config['vcd']['api_version'], verify_ssl_certs=self.config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials( self.config['vcd']['username'], server_constants.SYSTEM_ORG_NAME, # noqa: E501 self.config['vcd']['password']) client.set_credentials(credentials) is_tkg_plus_enabled = server_utils.is_tkg_plus_enabled(self.config) org_name = self.config['broker']['org'] catalog_name = self.config['broker']['catalog'] k8_templates = ltm.get_all_k8s_local_template_definition( client=client, catalog_name=catalog_name, org_name=org_name, logger_debug=logger.SERVER_LOGGER) if not k8_templates: msg = "No valid K8 templates were found in catalog " \ f"'{catalog_name}'. Unable to start CSE server." msg_update_callback.error(msg) logger.SERVER_LOGGER.error(msg) sys.exit(1) # Check that default k8s template exists in vCD at the correct # revision default_template_name = \ self.config['broker']['default_template_name'] default_template_revision = \ str(self.config['broker']['default_template_revision']) found_default_template = False for template in k8_templates: api_version = float(client.get_api_version()) if api_version >= float(vCDApiVersion.VERSION_35.value) and \ template[server_constants.LocalTemplateKey.KIND] == \ shared_constants.ClusterEntityKind.TKG_PLUS.value and \ not is_tkg_plus_enabled: # TKG+ is not enabled on CSE config. Skip the template and # log the relevant information. msg = "Skipping loading template data for " \ f"'{template[server_constants.LocalTemplateKey.NAME]}' as " \ "TKG+ is not enabled" # noqa: E501 logger.SERVER_LOGGER.debug(msg) k8_templates.remove(template) continue if str(template[server_constants.LocalTemplateKey.REVISION]) == default_template_revision and \ template[server_constants.LocalTemplateKey.NAME] == default_template_name: # noqa: E501 found_default_template = True msg = f"Found K8 template '{template['name']}' at revision " \ f"{template['revision']} in catalog '{catalog_name}'" msg_update_callback.general(msg) logger.SERVER_LOGGER.info(msg) if not found_default_template: msg = f"Default template {default_template_name} with " \ f"revision {default_template_revision} not found." \ " Unable to start CSE server." msg_update_callback.error(msg) logger.SERVER_LOGGER.error(msg) sys.exit(1) self.config['broker']['templates'] = k8_templates finally: if client: client.logout()
def install_cse(ctx, config_file_name='config.yaml', skip_template_creation=True, force_update=False, ssh_key=None, retain_temp_vapp=False, msg_update_callback=None): """Handle logistics for CSE installation. Handles decision making for configuring AMQP exchange/settings, extension registration, catalog setup, and template creation. :param click.core.Context ctx: :param str config_file_name: config file name. :param bool skip_template_creation: If True, skip creating the templates. :param bool force_update: if True and templates already exist in vCD, overwrites existing templates. :param str ssh_key: public ssh key to place into template vApp(s). :param bool retain_temp_vapp: if True, temporary vApp will not destroyed, so the user can ssh into and debug the vm. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object that writes messages onto console. :raises AmqpError: if AMQP exchange could not be created. """ configure_install_logger() config = get_validated_config(config_file_name, msg_update_callback=msg_update_callback) populate_vsphere_list(config['vcs']) msg = f"Installing CSE on vCloud Director using config file " \ f"'{config_file_name}'" if msg_update_callback: msg_update_callback.info(msg) LOGGER.info(msg) client = None try: client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=INSTALL_WIRELOG_FILEPATH, log_requests=True, log_headers=True, log_bodies=True) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" if msg_update_callback: msg_update_callback.general(msg) LOGGER.info(msg) # create amqp exchange if it doesn't exist amqp = config['amqp'] _create_amqp_exchange(amqp['exchange'], amqp['host'], amqp['port'], amqp['vhost'], amqp['ssl'], amqp['username'], amqp['password'], msg_update_callback=msg_update_callback) # register or update cse on vCD _register_cse(client, amqp['routing_key'], amqp['exchange'], msg_update_callback=msg_update_callback) # register rights to vCD # TODO() should also remove rights when unregistering CSE _register_right(client, right_name=CSE_NATIVE_DEPLOY_RIGHT_NAME, description=CSE_NATIVE_DEPLOY_RIGHT_DESCRIPTION, category=CSE_NATIVE_DEPLOY_RIGHT_CATEGORY, bundle_key=CSE_NATIVE_DEPLOY_RIGHT_BUNDLE_KEY, msg_update_callback=msg_update_callback) _register_right(client, right_name=CSE_PKS_DEPLOY_RIGHT_NAME, description=CSE_PKS_DEPLOY_RIGHT_DESCRIPTION, category=CSE_PKS_DEPLOY_RIGHT_CATEGORY, bundle_key=CSE_PKS_DEPLOY_RIGHT_BUNDLE_KEY, msg_update_callback=msg_update_callback) org_name = config['broker']['org'] catalog_name = config['broker']['catalog'] # set up cse catalog org = get_org(client, org_name=org_name) create_and_share_catalog(org, catalog_name, catalog_desc='CSE templates', msg_update_callback=msg_update_callback) if skip_template_creation: msg = "Skipping creation of templates." if msg_update_callback: msg_update_callback.info(msg) LOGGER.warning(msg) else: # read remote template cookbook, download all scripts rtm = RemoteTemplateManager( remote_template_cookbook_url=config['broker'] ['remote_template_cookbook_url'], # noqa: E501 logger=LOGGER, msg_update_callback=ConsoleMessagePrinter()) remote_template_cookbook = rtm.get_remote_template_cookbook() # create all templates defined in cookbook for template in remote_template_cookbook['templates']: rtm.download_template_scripts( template_name=template[RemoteTemplateKey.NAME], revision=template[RemoteTemplateKey.REVISION], force_overwrite=force_update) catalog_item_name = get_revisioned_template_name( template[RemoteTemplateKey.NAME], template[RemoteTemplateKey.REVISION]) build_params = { 'template_name': template[RemoteTemplateKey.NAME], 'template_revision': template[RemoteTemplateKey.REVISION], 'source_ova_name': template[RemoteTemplateKey.SOURCE_OVA_NAME], # noqa: E501 'source_ova_href': template[RemoteTemplateKey.SOURCE_OVA_HREF], # noqa: E501 'source_ova_sha256': template[ RemoteTemplateKey.SOURCE_OVA_SHA256], # noqa: E501 'org_name': org_name, 'vdc_name': config['broker']['vdc'], 'catalog_name': catalog_name, 'catalog_item_name': catalog_item_name, 'catalog_item_description': template[RemoteTemplateKey.DESCRIPTION], # noqa: E501 'temp_vapp_name': template[RemoteTemplateKey.NAME] + '_temp', # noqa: E501 'cpu': template[RemoteTemplateKey.CPU], 'memory': template[RemoteTemplateKey.MEMORY], 'network_name': config['broker']['network'], 'ip_allocation_mode': config['broker']['ip_allocation_mode'], # noqa: E501 'storage_profile': config['broker']['storage_profile'] } builder = TemplateBuilder( client, client, build_params, ssh_key=ssh_key, logger=LOGGER, msg_update_callback=ConsoleMessagePrinter()) builder.build(force_recreate=force_update, retain_temp_vapp=retain_temp_vapp) # remote definition is a super set of local definition, barring # the key 'catalog_item_name' template_definition = dict(template) template_definition['catalog_item_name'] = catalog_item_name save_k8s_local_template_definition_as_metadata( client=client, catalog_name=catalog_name, catalog_item_name=catalog_item_name, template_definition=template_definition, org_name=org_name) # if it's a PKS setup, setup NSX-T constructs if config.get('pks_config'): nsxt_servers = config.get('pks_config')['nsxt_servers'] for nsxt_server in nsxt_servers: msg = f"Configuring NSX-T server ({nsxt_server.get('name')})" \ " for CSE. Please check install logs for details." if msg_update_callback: msg_update_callback.general(msg) LOGGER.info(msg) nsxt_client = NSXTClient(host=nsxt_server.get('host'), username=nsxt_server.get('username'), password=nsxt_server.get('password'), http_proxy=nsxt_server.get('proxy'), https_proxy=nsxt_server.get('proxy'), verify_ssl=nsxt_server.get('verify'), logger_instance=LOGGER, log_requests=True, log_headers=True, log_body=True) setup_nsxt_constructs( nsxt_client=nsxt_client, nodes_ip_block_id=nsxt_server.get('nodes_ip_block_ids'), pods_ip_block_id=nsxt_server.get('pods_ip_block_ids'), ncp_boundary_firewall_section_anchor_id=nsxt_server.get( 'distributed_firewall_section_anchor_id') ) # noqa: E501 except Exception: if msg_update_callback: msg_update_callback.error( "CSE Installation Error. Check CSE install logs") LOGGER.error("CSE Installation Error", exc_info=True) raise # TODO() need installation relevant exceptions for rollback finally: if client is not None: client.logout()
def install_cse(ctx, config_file_name='config.yaml', template_name='*', update=False, no_capture=False, ssh_key=None, amqp_install='prompt', ext_install='prompt'): """Handles logistics for CSE installation. Handles decision making for configuring AMQP exchange/settings, extension registration, catalog setup, and template creation. :param click.core.Context ctx: :param str config_file_name: config file name. :param str template_name: which templates to create/update. A value of '*' means to create/update all templates specified in config file. :param bool update: if True and templates already exist in vCD, overwrites existing templates. :param bool no_capture: if True, temporary vApp will not be captured or destroyed, so the user can ssh into and debug the VM. :param str ssh_key: public ssh key to place into template vApp(s). :param str amqp_install: 'prompt' asks the user if vCD AMQP should be configured. 'skip' does not configure vCD AMQP. 'config' configures vCD AMQP without asking the user. :param str ext_install: 'prompt' asks the user if CSE should be registered to vCD. 'skip' does not register CSE to vCD. 'config' registers CSE to vCD without asking the user. :raises AmqpError: if AMQP exchange could not be created. """ config = get_validated_config(config_file_name) configure_install_logger() msg = f"Installing CSE on vCloud Director using config file " \ f"'{config_file_name}'" click.secho(msg, fg='yellow') LOGGER.info(msg) client = None try: client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=INSTALL_LOG_FILEPATH, log_headers=True, log_bodies=True) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" click.secho(msg, fg='green') LOGGER.info(msg) # configure amqp amqp = config['amqp'] create_amqp_exchange(amqp['exchange'], amqp['host'], amqp['port'], amqp['vhost'], amqp['ssl'], amqp['username'], amqp['password']) if should_configure_amqp(client, amqp, amqp_install): configure_vcd_amqp(client, amqp['exchange'], amqp['host'], amqp['port'], amqp['prefix'], amqp['ssl_accept_all'], amqp['ssl'], amqp['vhost'], amqp['username'], amqp['password']) # register cse as extension to vCD if should_register_cse(client, ext_install): register_cse(client, amqp['routing_key'], amqp['exchange']) # set up cse catalog org = get_org(client, org_name=config['broker']['org']) create_and_share_catalog(org, config['broker']['catalog'], catalog_desc='CSE templates') # create, customize, capture VM templates for template in config['broker']['templates']: if template_name == '*' or template['name'] == template_name: create_template(ctx, client, config, template, update=update, no_capture=no_capture, ssh_key=ssh_key, org=org) except Exception: click.secho("CSE Installation Error. Check CSE install logs", fg='red') LOGGER.error("CSE Installation Error", exc_info=True) raise # TODO need installation relevant exceptions for rollback finally: if client is not None: client.logout()
def check_cse_installation(config, check_template='*'): """Ensures that CSE is installed on vCD according to the config file. Checks if CSE is registered to vCD, if catalog exists, and if templates exist. :param dict config: config yaml file as a dictionary :param str check_template: which template to check for. Default value of '*' means to check all templates specified in @config :raises EntityNotFoundException: if CSE is not registered to vCD as an extension, or if specified catalog does not exist, or if specified template(s) do not exist. """ click.secho(f"Validating CSE installation according to config file", fg='yellow') err_msgs = [] client = None try: client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify']) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) # check that AMQP exchange exists amqp = config['amqp'] credentials = pika.PlainCredentials(amqp['username'], amqp['password']) parameters = pika.ConnectionParameters(amqp['host'], amqp['port'], amqp['vhost'], credentials, ssl=amqp['ssl'], connection_attempts=3, retry_delay=2, socket_timeout=5) connection = None try: connection = pika.BlockingConnection(parameters) channel = connection.channel() try: channel.exchange_declare(exchange=amqp['exchange'], exchange_type=EXCHANGE_TYPE, durable=True, passive=True, auto_delete=False) click.secho(f"AMQP exchange '{amqp['exchange']}' exists", fg='green') except pika.exceptions.ChannelClosed: msg = f"AMQP exchange '{amqp['exchange']}' does not exist" click.secho(msg, fg='red') err_msgs.append(msg) except Exception: # TODO replace raw exception with specific msg = f"Could not connect to AMQP exchange '{amqp['exchange']}'" click.secho(msg, fg='red') err_msgs.append(msg) finally: if connection is not None: connection.close() # check that CSE is registered to vCD ext = APIExtension(client) try: cse_info = ext.get_extension(CSE_NAME, namespace=CSE_NAMESPACE) if cse_info['enabled'] == 'true': click.secho( "CSE is registered to vCD and is currently " "enabled", fg='green') else: click.secho( "CSE is registered to vCD and is currently " "disabled", fg='yellow') except MissingRecordException: msg = "CSE is not registered to vCD" click.secho(msg, fg='red') err_msgs.append(msg) # check that catalog exists in vCD org = Org(client, resource=client.get_org()) catalog_name = config['broker']['catalog'] if catalog_exists(org, catalog_name): click.secho(f"Found catalog '{catalog_name}'", fg='green') # check that templates exist in vCD for template in config['broker']['templates']: if check_template != '*' and \ check_template != template['name']: continue catalog_item_name = template['catalog_item'] if catalog_item_exists(org, catalog_name, catalog_item_name): click.secho( f"Found template '{catalog_item_name}' in " f"catalog '{catalog_name}'", fg='green') else: msg = f"Template '{catalog_item_name}' not found in " \ f"catalog '{catalog_name}'" click.secho(msg, fg='red') err_msgs.append(msg) else: msg = f"Catalog '{catalog_name}' not found" click.secho(msg, fg='red') err_msgs.append(msg) finally: if client is not None: client.logout() if err_msgs: raise EntityNotFoundException(err_msgs) click.secho(f"CSE installation is valid", fg='green')
def validate_vcd_and_vcs_config(vcd_dict, vcs): """Ensures that 'vcd' and vcs' section of config are correct. Checks that 'vcd' and 'vcs' section of config have correct keys and value types. Also checks that vCD and all registered VCs in vCD are accessible. :param dict vcd_dict: 'vcd' section of config file as a dict. :param list vcs: 'vcs' section of config file as a list of dicts. :raises KeyError: if @vcd_dict or a vc in @vcs has missing or extra properties. :raises: ValueError: if the value type for a @vcd_dict or vc property is incorrect, or if vCD has a VC that is not listed in the config file. """ check_keys_and_value_types(vcd_dict, SAMPLE_VCD_CONFIG['vcd'], location="config file 'vcd' section") if not vcd_dict['verify']: click.secho( 'InsecureRequestWarning: Unverified HTTPS request is ' 'being made. Adding certificate verification is ' 'strongly advised.', fg='yellow', err=True) requests.packages.urllib3.disable_warnings() client = None try: client = Client(vcd_dict['host'], api_version=vcd_dict['api_version'], verify_ssl_certs=vcd_dict['verify']) client.set_credentials( BasicLoginCredentials(vcd_dict['username'], SYSTEM_ORG_NAME, vcd_dict['password'])) click.secho( f"Connected to vCloud Director " f"({vcd_dict['host']}:{vcd_dict['port']})", fg='green') for index, vc in enumerate(vcs, 1): check_keys_and_value_types(vc, SAMPLE_VCS_CONFIG['vcs'][0], location=f"config file 'vcs' section, " f"vc #{index}") # Check that all registered VCs in vCD are listed in config file platform = Platform(client) config_vc_names = [vc['name'] for vc in vcs] for platform_vc in platform.list_vcenters(): platform_vc_name = platform_vc.get('name') if platform_vc_name not in config_vc_names: raise ValueError(f"vCenter '{platform_vc_name}' registered in " f"vCD but not found in config file") # Check that all VCs listed in config file are registered in vCD for vc in vcs: vcenter = platform.get_vcenter(vc['name']) vsphere_url = urlparse(vcenter.Url.text) v = VSphere(vsphere_url.hostname, vc['username'], vc['password'], vsphere_url.port) v.connect() click.secho( f"Connected to vCenter Server '{vc['name']}' as " f"'{vc['username']}' ({vsphere_url.hostname}:" f"{vsphere_url.port})", fg='green') finally: if client is not None: client.logout()
def run(self, msg_update_callback=None): configure_server_logger() self.config = get_validated_config( self.config_file, msg_update_callback=msg_update_callback) populate_vsphere_list(self.config['vcs']) # Read K8 catalog definition from catalog item metadata and append # to server config client = None try: client = Client(self.config['vcd']['host'], api_version=self.config['vcd']['api_version'], verify_ssl_certs=self.config['vcd']['verify'], log_file=SERVER_DEBUG_WIRELOG_FILEPATH, log_requests=True, log_headers=True, log_bodies=True) credentials = BasicLoginCredentials(self.config['vcd']['username'], SYSTEM_ORG_NAME, self.config['vcd']['password']) client.set_credentials(credentials) org_name = self.config['broker']['org'] catalog_name = self.config['broker']['catalog'] k8_templates = get_all_k8s_local_template_definition( client=client, catalog_name=catalog_name, org_name=org_name) if not k8_templates: msg = "No valid K8 templates were found in catalog " \ f"'{catalog_name}'. Unable to start CSE server." if msg_update_callback: msg_update_callback.error(msg) LOGGER.error(msg) sys.exit(1) # Check that deafult K8 template exists in vCD at the correct # revision default_template_name = \ self.config['broker']['default_template_name'] default_template_revision = \ str(self.config['broker']['default_template_revision']) found_default_template = False for template in k8_templates: if str(template['revision']) == default_template_revision \ and template['name'] == default_template_name: found_default_template = True msg = f"Found K8 template '{template['name']}' at revision " \ f"{template['revision']} in catalog '{catalog_name}'" if msg_update_callback: msg_update_callback.general(msg) LOGGER.info(msg) if not found_default_template: msg = f"Default template {default_template_name} with " \ f"revision {default_template_revision} not found." \ " Unable to start CSE server." if msg_update_callback: msg_update_callback.error(msg) LOGGER.error(msg) sys.exit(1) self.config['broker']['templates'] = k8_templates finally: if client: client.logout() # TODO Rule framework, update config with rules if self.should_check_config: check_cse_installation(self.config, msg_update_callback=msg_update_callback) if self.config.get('pks_config'): pks_config = self.config.get('pks_config') self.pks_cache = PksCache( pks_servers=pks_config.get('pks_api_servers', []), pks_accounts=pks_config.get('pks_accounts', []), pvdcs=pks_config.get('pvdcs', []), orgs=pks_config.get('orgs', []), nsxt_servers=pks_config.get('nsxt_servers', [])) amqp = self.config['amqp'] num_consumers = self.config['service']['listeners'] for n in range(num_consumers): try: c = MessageConsumer(amqp['host'], amqp['port'], amqp['ssl'], amqp['vhost'], amqp['username'], amqp['password'], amqp['exchange'], amqp['routing_key']) name = 'MessageConsumer-%s' % n t = Thread(name=name, target=consumer_thread, args=(c, )) t.daemon = True t.start() msg = f"Started thread '{name} ({t.ident})'" if msg_update_callback: msg_update_callback.general(msg) LOGGER.info(msg) self.threads.append(t) self.consumers.append(c) time.sleep(0.25) except KeyboardInterrupt: break except Exception: LOGGER.error(traceback.format_exc()) LOGGER.info(f"Number of threads started: {len(self.threads)}") self._state = ServerState.RUNNING message = f"Container Service Extension for vCloud Director" \ f"\nServer running using config file: {self.config_file}" \ f"\nLog files: {SERVER_INFO_LOG_FILEPATH}, " \ f"{SERVER_DEBUG_LOG_FILEPATH}" \ f"\nwaiting for requests (ctrl+c to close)" signal.signal(signal.SIGINT, signal_handler) if msg_update_callback: msg_update_callback.general_no_color(message) LOGGER.info(message) while True: try: time.sleep(1) if self._state == ServerState.STOPPING and \ self.active_requests_count() == 0: break except KeyboardInterrupt: break except Exception: if msg_update_callback: msg_update_callback.general_no_color( traceback.format_exc()) LOGGER.error(traceback.format_exc()) sys.exit(1) LOGGER.info("Stop detected") LOGGER.info("Closing connections...") for c in self.consumers: try: c.stop() except Exception: pass self._state = ServerState.STOPPED LOGGER.info("Done")
def install_cse(config_file_name, skip_template_creation, force_update, ssh_key, retain_temp_vapp, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, msg_update_callback=utils.NullPrinter()): """Handle logistics for CSE installation. Handles decision making for configuring AMQP exchange/settings, defined entity schema registration for vCD api version >= 35, extension registration, catalog setup and template creation. Also records telemetry data on installation details. :param str config_file_name: config file name. :param bool skip_template_creation: If True, skip creating the templates. :param bool force_update: if True and templates already exist in vCD, overwrites existing templates. :param str ssh_key: public ssh key to place into template vApp(s). :param bool retain_temp_vapp: if True, temporary vApp will not destroyed, so the user can ssh into and debug the vm. :param str pks_config_file_name: pks config file name. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :raises cse_exception.AmqpError: if AMQP exchange could not be created. """ config = get_validated_config( config_file_name, pks_config_file_name=pks_config_file_name, skip_config_decryption=skip_config_decryption, decryption_password=decryption_password, log_wire_file=INSTALL_WIRELOG_FILEPATH, logger_debug=INSTALL_LOGGER, msg_update_callback=msg_update_callback) populate_vsphere_list(config['vcs']) msg = f"Installing CSE on vCloud Director using config file " \ f"'{config_file_name}'" msg_update_callback.info(msg) INSTALL_LOGGER.info(msg) client = None try: # Telemetry - Construct telemetry data telemetry_data = { PayloadKey.WAS_DECRYPTION_SKIPPED: bool(skip_config_decryption), # noqa: E501 PayloadKey.WAS_PKS_CONFIG_FILE_PROVIDED: bool(pks_config_file_name), # noqa: E501 PayloadKey.WERE_TEMPLATES_SKIPPED: bool(skip_template_creation), # noqa: E501 PayloadKey.WERE_TEMPLATES_FORCE_UPDATED: bool(force_update), # noqa: E501 PayloadKey.WAS_TEMP_VAPP_RETAINED: bool(retain_temp_vapp), # noqa: E501 PayloadKey.WAS_SSH_KEY_SPECIFIED: bool(ssh_key) # noqa: E501 } # Telemetry - Record detailed telemetry data on install record_user_action_details( CseOperation.SERVICE_INSTALL, telemetry_data, telemetry_settings=config['service']['telemetry']) # noqa: E501 log_filename = None log_wire = utils.str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = INSTALL_WIRELOG_FILEPATH client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], server_constants.SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" msg_update_callback.general(msg) INSTALL_LOGGER.info(msg) # create amqp exchange if it doesn't exist amqp = config['amqp'] _create_amqp_exchange(amqp['exchange'], amqp['host'], amqp['port'], amqp['vhost'], amqp['ssl'], amqp['username'], amqp['password'], msg_update_callback=msg_update_callback) # register or update cse on vCD _register_cse(client, amqp['routing_key'], amqp['exchange'], msg_update_callback=msg_update_callback) # register cse def schema on VCD # schema should be located at # ~/.cse-schema/api-v<API VERSION>/schema.json _register_def_schema(client, msg_update_callback=msg_update_callback, log_wire=log_wire) # Since we use CSE extension id as our telemetry instance_id, the # validated config won't have the instance_id yet. Now that CSE has # been registered as an extension, we should update the telemetry # config with the correct instance_id if config['service']['telemetry']['enable']: store_telemetry_settings(config) # register rights to vCD # TODO() should also remove rights when unregistering CSE _register_right( client, right_name=server_constants. CSE_NATIVE_DEPLOY_RIGHT_NAME, # noqa: E501 description=server_constants. CSE_NATIVE_DEPLOY_RIGHT_DESCRIPTION, # noqa: E501 category=server_constants. CSE_NATIVE_DEPLOY_RIGHT_CATEGORY, # noqa: E501 bundle_key=server_constants. CSE_NATIVE_DEPLOY_RIGHT_BUNDLE_KEY, # noqa: E501 msg_update_callback=msg_update_callback) _register_right( client, right_name=server_constants. CSE_PKS_DEPLOY_RIGHT_NAME, # noqa: E501 description=server_constants. CSE_PKS_DEPLOY_RIGHT_DESCRIPTION, # noqa: E501 category=server_constants. CSE_PKS_DEPLOY_RIGHT_CATEGORY, # noqa: E501 bundle_key=server_constants. CSE_PKS_DEPLOY_RIGHT_BUNDLE_KEY, # noqa: E501 msg_update_callback=msg_update_callback) # set up placement policies for all types of clusters _setup_placement_policies( client, policy_list=server_constants. CLUSTER_PLACEMENT_POLICIES, # noqa: E501 msg_update_callback=msg_update_callback, log_wire=log_wire) # set up cse catalog org = vcd_utils.get_org(client, org_name=config['broker']['org']) vcd_utils.create_and_share_catalog( org, config['broker']['catalog'], catalog_desc='CSE templates', logger=INSTALL_LOGGER, msg_update_callback=msg_update_callback) if skip_template_creation: msg = "Skipping creation of templates." msg_update_callback.info(msg) INSTALL_LOGGER.warning(msg) else: # read remote template cookbook, download all scripts rtm = RemoteTemplateManager( remote_template_cookbook_url=config['broker'] ['remote_template_cookbook_url'], # noqa: E501 logger=INSTALL_LOGGER, msg_update_callback=msg_update_callback) remote_template_cookbook = rtm.get_remote_template_cookbook() # create all templates defined in cookbook for template in remote_template_cookbook['templates']: # TODO tag created templates with placement policies _install_template( client=client, remote_template_manager=rtm, template=template, org_name=config['broker']['org'], vdc_name=config['broker']['vdc'], catalog_name=config['broker']['catalog'], network_name=config['broker']['network'], ip_allocation_mode=config['broker']['ip_allocation_mode'], storage_profile=config['broker']['storage_profile'], force_update=force_update, retain_temp_vapp=retain_temp_vapp, ssh_key=ssh_key, msg_update_callback=msg_update_callback) # if it's a PKS setup, setup NSX-T constructs if config.get('pks_config'): nsxt_servers = config['pks_config']['nsxt_servers'] wire_logger = NULL_LOGGER if log_wire: wire_logger = SERVER_NSXT_WIRE_LOGGER for nsxt_server in nsxt_servers: msg = f"Configuring NSX-T server ({nsxt_server.get('name')})" \ " for CSE. Please check install logs for details." msg_update_callback.general(msg) INSTALL_LOGGER.info(msg) nsxt_client = NSXTClient(host=nsxt_server.get('host'), username=nsxt_server.get('username'), password=nsxt_server.get('password'), logger_debug=INSTALL_LOGGER, logger_wire=wire_logger, http_proxy=nsxt_server.get('proxy'), https_proxy=nsxt_server.get('proxy'), verify_ssl=nsxt_server.get('verify')) setup_nsxt_constructs( nsxt_client=nsxt_client, nodes_ip_block_id=nsxt_server.get('nodes_ip_block_ids'), pods_ip_block_id=nsxt_server.get('pods_ip_block_ids'), ncp_boundary_firewall_section_anchor_id=nsxt_server.get( 'distributed_firewall_section_anchor_id') ) # noqa: E501 # Telemetry - Record successful install action record_user_action(CseOperation.SERVICE_INSTALL, telemetry_settings=config['service']['telemetry']) except Exception: msg_update_callback.error( "CSE Installation Error. Check CSE install logs") INSTALL_LOGGER.error("CSE Installation Error", exc_info=True) # Telemetry - Record failed install action record_user_action(CseOperation.SERVICE_INSTALL, status=OperationStatus.FAILED, telemetry_settings=config['service']['telemetry']) raise # TODO() need installation relevant exceptions for rollback finally: if client is not None: client.logout()
def convert_cluster(ctx, config_file_name, skip_config_decryption, cluster_name, admin_password, org_name, vdc_name, skip_wait_for_gc): if skip_config_decryption: decryption_password = None else: decryption_password = os.getenv('CSE_CONFIG_PASSWORD') or prompt_text( PASSWORD_FOR_CONFIG_DECRYPTION_MSG, color='green', hide_input=True) try: check_python_version() except Exception as err: click.secho(str(err), fg='red') sys.exit(1) client = None try: console_message_printer = ConsoleMessagePrinter() config = get_validated_config( config_file_name, skip_config_decryption=skip_config_decryption, decryption_password=decryption_password, msg_update_callback=console_message_printer) log_filename = None log_wire = str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = 'cluster_convert_wire.log' client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" console_message_printer.general(msg) cluster_records = get_all_clusters(client=client, cluster_name=cluster_name, org_name=org_name, ovdc_name=vdc_name) if len(cluster_records) == 0: console_message_printer.info(f"No clusters were found.") return vms = [] for cluster in cluster_records: console_message_printer.info( f"Processing cluster '{cluster['name']}'.") vapp_href = cluster['vapp_href'] vapp = VApp(client, href=vapp_href) # this step removes the old 'cse.template' metadata and adds # cse.template.name and cse.template.revision metadata # using hard-coded values taken from github history console_message_printer.info("Processing metadata of cluster.") metadata_dict = metadata_to_dict(vapp.get_metadata()) old_template_name = metadata_dict.get( ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME ) # noqa: E501 new_template_name = None cse_version = metadata_dict.get(ClusterMetadataKey.CSE_VERSION) if old_template_name: console_message_printer.info( "Determining k8s version on cluster.") if 'photon' in old_template_name: new_template_name = 'photon-v2' if cse_version in ('1.0.0'): new_template_name += '_k8s-1.8_weave-2.0.5' elif cse_version in ('1.1.0', '1.2.0', '1.2.1', '1.2.2', '1.2.3', '1.2.4'): # noqa: E501 new_template_name += '_k8s-1.9_weave-2.3.0' elif cse_version in ( '1.2.5', '1.2.6', '1.2.7', ): # noqa: E501 new_template_name += '_k8s-1.10_weave-2.3.0' elif cse_version in ('2.0.0'): new_template_name += '_k8s-1.12_weave-2.3.0' elif 'ubuntu' in old_template_name: new_template_name = 'ubuntu-16.04' if cse_version in ('1.0.0'): new_template_name += '_k8s-1.9_weave-2.1.3' elif cse_version in ('1.1.0', '1.2.0', '1.2.1', '1.2.2', '1.2.3', '1.2.4', '1.2.5', '1.2.6', '1.2.7'): # noqa: E501 new_template_name += '_k8s-1.10_weave-2.3.0' elif cse_version in ('2.0.0'): new_template_name += '_k8s-1.13_weave-2.3.0' if new_template_name: console_message_printer.info("Updating metadata of cluster.") task = vapp.remove_metadata( ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME ) # noqa: E501 client.get_task_monitor().wait_for_success(task) new_metadata_to_add = { ClusterMetadataKey.TEMPLATE_NAME: new_template_name, ClusterMetadataKey.TEMPLATE_REVISION: 0 } task = vapp.set_multiple_metadata(new_metadata_to_add) client.get_task_monitor().wait_for_success(task) # this step uses hard-coded data from the newly updated # cse.template.name and cse.template.revision metadata fields as # well as github history to add [cse.os, cse.docker.version, # cse.kubernetes, cse.kubernetes.version, cse.cni, cse.cni.version] # to the clusters vapp.reload() metadata_dict = metadata_to_dict(vapp.get_metadata()) template_name = metadata_dict.get(ClusterMetadataKey.TEMPLATE_NAME) template_revision = str( metadata_dict.get(ClusterMetadataKey.TEMPLATE_REVISION, '0')) # noqa: E501 if template_name: k8s_version, docker_version = get_k8s_and_docker_versions( template_name, template_revision=template_revision, cse_version=cse_version) # noqa: E501 tokens = template_name.split('_') new_metadata = { ClusterMetadataKey.OS: tokens[0], ClusterMetadataKey.DOCKER_VERSION: docker_version, ClusterMetadataKey.KUBERNETES: 'upstream', ClusterMetadataKey.KUBERNETES_VERSION: k8s_version, ClusterMetadataKey.CNI: tokens[2].split('-')[0], ClusterMetadataKey.CNI_VERSION: tokens[2].split('-')[1], } task = vapp.set_multiple_metadata(new_metadata) client.get_task_monitor().wait_for_success(task) console_message_printer.general( "Finished processing metadata of cluster.") reset_admin_pw = False vm_resources = vapp.get_all_vms() for vm_resource in vm_resources: try: vapp.get_admin_password(vm_resource.get('name')) except EntityNotFoundException: reset_admin_pw = True break if reset_admin_pw: try: console_message_printer.info( f"Undeploying the vApp '{cluster['name']}'") task = vapp.undeploy() client.get_task_monitor().wait_for_success(task) console_message_printer.general( "Successfully undeployed the vApp.") except Exception as err: console_message_printer.error(str(err)) for vm_resource in vm_resources: console_message_printer.info( f"Processing vm '{vm_resource.get('name')}'.") vm = VM(client, href=vm_resource.get('href')) vms.append(vm) console_message_printer.info("Updating vm admin password") task = vm.update_guest_customization_section( enabled=True, admin_password_enabled=True, admin_password_auto=not admin_password, admin_password=admin_password, ) client.get_task_monitor().wait_for_success(task) console_message_printer.general("Successfully updated vm") console_message_printer.info("Deploying vm.") task = vm.power_on_and_force_recustomization() client.get_task_monitor().wait_for_success(task) console_message_printer.general("Successfully deployed vm") console_message_printer.info("Deploying cluster") task = vapp.deploy(power_on=True) client.get_task_monitor().wait_for_success(task) console_message_printer.general( "Successfully deployed cluster") # noqa: E501 console_message_printer.general( f"Successfully processed cluster '{cluster['name']}'") if skip_wait_for_gc: return while True: to_remove = [] for vm in vms: status = vm.get_guest_customization_status() if status != 'GC_PENDING': to_remove.append(vm) for vm in to_remove: vms.remove(vm) console_message_printer.info( f"Waiting on guest customization to finish on {len(vms)} vms.") if not len(vms) == 0: time.sleep(5) else: break except cryptography.fernet.InvalidToken: click.secho(CONFIG_DECRYPTION_ERROR_MSG, fg='red') except Exception as err: click.secho(str(err), fg='red') finally: if client: client.logout()
def install_template(template_name, template_revision, config_file_name, force_create, retain_temp_vapp, ssh_key, skip_config_decryption=False, decryption_password=None, msg_update_callback=utils.NullPrinter()): """Install a particular template in CSE. If template_name and revision are wild carded to *, all templates defined in remote template cookbook will be installed. :param str template_name: :param str template_revision: :param str config_file_name: config file name. :param bool force_create: if True and template already exists in vCD, overwrites existing template. :param str ssh_key: public ssh key to place into template vApp(s). :param bool retain_temp_vapp: if True, temporary vApp will not destroyed, so the user can ssh into and debug the vm. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. """ config = get_validated_config( config_file_name, skip_config_decryption=skip_config_decryption, decryption_password=decryption_password, log_wire_file=INSTALL_WIRELOG_FILEPATH, logger_debug=INSTALL_LOGGER, msg_update_callback=msg_update_callback) populate_vsphere_list(config['vcs']) msg = f"Installing template '{template_name}' at revision " \ f"'{template_revision}' on vCloud Director using config file " \ f"'{config_file_name}'" msg_update_callback.info(msg) INSTALL_LOGGER.info(msg) client = None try: # Telemetry data construction cse_params = { PayloadKey.TEMPLATE_NAME: template_name, PayloadKey.TEMPLATE_REVISION: template_revision, PayloadKey.WAS_DECRYPTION_SKIPPED: bool(skip_config_decryption), PayloadKey.WERE_TEMPLATES_FORCE_UPDATED: bool(force_create), PayloadKey.WAS_TEMP_VAPP_RETAINED: bool(retain_temp_vapp), PayloadKey.WAS_SSH_KEY_SPECIFIED: bool(ssh_key) } # Record telemetry data record_user_action_details( cse_operation=CseOperation.TEMPLATE_INSTALL, cse_params=cse_params, telemetry_settings=config['service']['telemetry']) log_filename = None log_wire = utils.str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = INSTALL_WIRELOG_FILEPATH client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], server_constants.SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" msg_update_callback.general(msg) INSTALL_LOGGER.info(msg) # read remote template cookbook rtm = RemoteTemplateManager( remote_template_cookbook_url=config['broker'] ['remote_template_cookbook_url'], # noqa: E501 logger=INSTALL_LOGGER, msg_update_callback=msg_update_callback) remote_template_cookbook = rtm.get_remote_template_cookbook() found_template = False for template in remote_template_cookbook['templates']: template_name_matched = template_name in ( template[server_constants.RemoteTemplateKey.NAME], '*' ) # noqa: E501 template_revision_matched = \ str(template_revision) in (str(template[server_constants.RemoteTemplateKey.REVISION]), '*') # noqa: E501 if template_name_matched and template_revision_matched: found_template = True _install_template( client=client, remote_template_manager=rtm, template=template, org_name=config['broker']['org'], vdc_name=config['broker']['vdc'], catalog_name=config['broker']['catalog'], network_name=config['broker']['network'], ip_allocation_mode=config['broker']['ip_allocation_mode'], storage_profile=config['broker']['storage_profile'], force_update=force_create, retain_temp_vapp=retain_temp_vapp, ssh_key=ssh_key, msg_update_callback=msg_update_callback) if not found_template: msg = f"Template '{template_name}' at revision " \ f"'{template_revision}' not found in remote template " \ "cookbook." msg_update_callback.error(msg) INSTALL_LOGGER.error(msg, exc_info=True) raise Exception(msg) # Record telemetry data on successful template install record_user_action( cse_operation=CseOperation.TEMPLATE_INSTALL, status=OperationStatus.SUCCESS, telemetry_settings=config['service']['telemetry']) # noqa: E501 except Exception: msg_update_callback.error( "Template Installation Error. Check CSE install logs") INSTALL_LOGGER.error("Template Installation Error", exc_info=True) # Record telemetry data on template install failure record_user_action(cse_operation=CseOperation.TEMPLATE_INSTALL, status=OperationStatus.FAILED, telemetry_settings=config['service']['telemetry']) finally: if client is not None: client.logout()
def list_template(ctx, config_file_name, skip_config_decryption, display_option): """List CSE k8s templates.""" if skip_config_decryption: password = None else: password = os.getenv('CSE_CONFIG_PASSWORD') or prompt_text( PASSWORD_FOR_CONFIG_DECRYPTION_MSG, color='green', hide_input=True) try: console_message_printer = ConsoleMessagePrinter() try: check_python_version() except Exception as err: click.secho(str(err), fg='red') sys.exit(1) # We don't want to validate config file, because server startup or # installation is not being perfomred. If values in config file are # missing or bad, appropriate exception will be raised while accessing # or using them. if skip_config_decryption: with open(config_file_name) as config_file: config_dict = yaml.safe_load(config_file) or {} else: console_message_printer.info(f"Decrypting '{config_file_name}'") config_dict = yaml.safe_load( get_decrypted_file_contents(config_file_name, password)) or {} local_templates = [] if display_option in (DISPLAY_ALL, DISPLAY_DIFF, DISPLAY_LOCAL): client = None try: # To suppress the warning message that pyvcloud prints if # ssl_cert verification is skipped. if not config_dict['vcd']['verify']: requests.packages.urllib3.disable_warnings() client = Client(config_dict['vcd']['host'], api_version=config_dict['vcd']['api_version'], verify_ssl_certs=config_dict['vcd']['verify']) credentials = BasicLoginCredentials( config_dict['vcd']['username'], SYSTEM_ORG_NAME, config_dict['vcd']['password']) client.set_credentials(credentials) org_name = config_dict['broker']['org'] catalog_name = config_dict['broker']['catalog'] local_template_definitions = \ get_all_k8s_local_template_definition( client=client, catalog_name=catalog_name, org_name=org_name) default_template_name = \ config_dict['broker']['default_template_name'] default_template_revision = \ str(config_dict['broker']['default_template_revision']) for definition in local_template_definitions: template = {} template['name'] = definition[LocalTemplateKey.NAME] template['revision'] = \ definition[LocalTemplateKey.REVISION] template['compute_policy'] = \ definition[LocalTemplateKey.COMPUTE_POLICY] template['local'] = True template['remote'] = False if str(definition[LocalTemplateKey.REVISION] ) == default_template_revision and definition[ LocalTemplateKey. NAME] == default_template_name: # noqa: E501 template['default'] = True else: template['default'] = False template['deprecated'] = \ str_to_bool(definition[LocalTemplateKey.DEPRECATED]) template['cpu'] = definition[LocalTemplateKey.CPU] template['memory'] = definition[LocalTemplateKey.MEMORY] template['description'] = \ definition[LocalTemplateKey.DESCRIPTION] local_templates.append(template) finally: if client: client.logout() remote_templates = [] if display_option in (DISPLAY_ALL, DISPLAY_DIFF, DISPLAY_REMOTE): rtm = RemoteTemplateManager( remote_template_cookbook_url=config_dict['broker'] ['remote_template_cookbook_url'], # noqa: E501 msg_update_callback=ConsoleMessagePrinter()) remote_template_cookbook = rtm.get_remote_template_cookbook() remote_template_definitions = remote_template_cookbook['templates'] for definition in remote_template_definitions: template = {} template['name'] = definition[RemoteTemplateKey.NAME] template['revision'] = definition[RemoteTemplateKey.REVISION] template['compute_policy'] = \ definition[RemoteTemplateKey.COMPUTE_POLICY] template['local'] = False template['remote'] = True template['default'] = False template['deprecated'] = \ str_to_bool(definition[RemoteTemplateKey.DEPRECATED]) template['cpu'] = definition[RemoteTemplateKey.CPU] template['memory'] = definition[RemoteTemplateKey.MEMORY] template['description'] = \ definition[RemoteTemplateKey.DESCRIPTION] remote_templates.append(template) result = [] if display_option is DISPLAY_ALL: result = remote_templates # If local copy of template exists, update the remote definition # with relevant values, else add the local definition to the result # list. for local_template in local_templates: found = False for remote_template in remote_templates: if str(local_template[LocalTemplateKey.REVISION]) == str( remote_template[RemoteTemplateKey.REVISION] ) and local_template[ LocalTemplateKey.NAME] == remote_template[ RemoteTemplateKey.NAME]: # noqa: E501 remote_template['compute_policy'] = \ local_template['compute_policy'] remote_template['local'] = local_template['local'] remote_template['default'] = local_template['default'] found = True break if not found: result.append(local_template) elif display_option in DISPLAY_DIFF: for remote_template in remote_templates: found = False for local_template in local_templates: if str(local_template[LocalTemplateKey.REVISION]) == str( remote_template[RemoteTemplateKey.REVISION] ) and local_template[ LocalTemplateKey.NAME] == remote_template[ RemoteTemplateKey.NAME]: # noqa: E501 found = True break if not found: result.append(remote_template) elif display_option in DISPLAY_LOCAL: result = local_templates elif display_option in DISPLAY_REMOTE: result = remote_templates stdout(result, ctx, sort_headers=False) except cryptography.fernet.InvalidToken: click.secho(CONFIG_DECRYPTION_ERROR_MSG, fg='red') except Exception as err: click.secho(str(err), fg='red')
def install_cse(config_file_name, skip_template_creation, force_update, ssh_key, retain_temp_vapp, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, msg_update_callback=None): """Handle logistics for CSE installation. Handles decision making for configuring AMQP exchange/settings, extension registration, catalog setup, and template creation. :param str config_file_name: config file name. :param bool skip_template_creation: If True, skip creating the templates. :param bool force_update: if True and templates already exist in vCD, overwrites existing templates. :param str ssh_key: public ssh key to place into template vApp(s). :param bool retain_temp_vapp: if True, temporary vApp will not destroyed, so the user can ssh into and debug the vm. :param str pks_config_file_name: pks config file name. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object that writes messages onto console. :raises AmqpError: if AMQP exchange could not be created. """ configure_install_logger() config = get_validated_config( config_file_name, pks_config_file_name=pks_config_file_name, skip_config_decryption=skip_config_decryption, decryption_password=decryption_password, msg_update_callback=msg_update_callback) populate_vsphere_list(config['vcs']) msg = f"Installing CSE on vCloud Director using config file " \ f"'{config_file_name}'" if msg_update_callback: msg_update_callback.info(msg) LOGGER.info(msg) client = None try: log_filename = None log_wire = str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = INSTALL_WIRELOG_FILEPATH client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" if msg_update_callback: msg_update_callback.general(msg) LOGGER.info(msg) # create amqp exchange if it doesn't exist amqp = config['amqp'] _create_amqp_exchange(amqp['exchange'], amqp['host'], amqp['port'], amqp['vhost'], amqp['ssl'], amqp['username'], amqp['password'], msg_update_callback=msg_update_callback) # register or update cse on vCD _register_cse(client, amqp['routing_key'], amqp['exchange'], msg_update_callback=msg_update_callback) # register rights to vCD # TODO() should also remove rights when unregistering CSE _register_right(client, right_name=CSE_NATIVE_DEPLOY_RIGHT_NAME, description=CSE_NATIVE_DEPLOY_RIGHT_DESCRIPTION, category=CSE_NATIVE_DEPLOY_RIGHT_CATEGORY, bundle_key=CSE_NATIVE_DEPLOY_RIGHT_BUNDLE_KEY, msg_update_callback=msg_update_callback) _register_right(client, right_name=CSE_PKS_DEPLOY_RIGHT_NAME, description=CSE_PKS_DEPLOY_RIGHT_DESCRIPTION, category=CSE_PKS_DEPLOY_RIGHT_CATEGORY, bundle_key=CSE_PKS_DEPLOY_RIGHT_BUNDLE_KEY, msg_update_callback=msg_update_callback) # set up cse catalog org = get_org(client, org_name=config['broker']['org']) create_and_share_catalog(org, config['broker']['catalog'], catalog_desc='CSE templates', msg_update_callback=msg_update_callback) if skip_template_creation: msg = "Skipping creation of templates." if msg_update_callback: msg_update_callback.info(msg) LOGGER.warning(msg) else: # read remote template cookbook, download all scripts rtm = RemoteTemplateManager( remote_template_cookbook_url=config['broker'] ['remote_template_cookbook_url'], # noqa: E501 logger=LOGGER, msg_update_callback=msg_update_callback) remote_template_cookbook = rtm.get_remote_template_cookbook() # create all templates defined in cookbook for template in remote_template_cookbook['templates']: _install_template( client=client, remote_template_manager=rtm, template=template, org_name=config['broker']['org'], vdc_name=config['broker']['vdc'], catalog_name=config['broker']['catalog'], network_name=config['broker']['network'], ip_allocation_mode=config['broker']['ip_allocation_mode'], storage_profile=config['broker']['storage_profile'], force_update=force_update, retain_temp_vapp=retain_temp_vapp, ssh_key=ssh_key, msg_update_callback=msg_update_callback) # if it's a PKS setup, setup NSX-T constructs if config.get('pks_config'): nsxt_servers = config.get('pks_config')['nsxt_servers'] for nsxt_server in nsxt_servers: msg = f"Configuring NSX-T server ({nsxt_server.get('name')})" \ " for CSE. Please check install logs for details." if msg_update_callback: msg_update_callback.general(msg) LOGGER.info(msg) nsxt_client = NSXTClient(host=nsxt_server.get('host'), username=nsxt_server.get('username'), password=nsxt_server.get('password'), http_proxy=nsxt_server.get('proxy'), https_proxy=nsxt_server.get('proxy'), verify_ssl=nsxt_server.get('verify'), logger_instance=LOGGER, log_requests=True, log_headers=True, log_body=True) setup_nsxt_constructs( nsxt_client=nsxt_client, nodes_ip_block_id=nsxt_server.get('nodes_ip_block_ids'), pods_ip_block_id=nsxt_server.get('pods_ip_block_ids'), ncp_boundary_firewall_section_anchor_id=nsxt_server.get( 'distributed_firewall_section_anchor_id') ) # noqa: E501 except Exception: if msg_update_callback: msg_update_callback.error( "CSE Installation Error. Check CSE install logs") LOGGER.error("CSE Installation Error", exc_info=True) raise # TODO() need installation relevant exceptions for rollback finally: if client is not None: client.logout()
def add_additional_details_to_config( config: Dict, vcd_host: str, vcd_username: str, vcd_password: str, verify_ssl: bool, is_legacy_mode: bool, is_mqtt_exchange: bool, log_wire: bool, log_wire_file: str ): """Update config dict with computed key-value pairs. :param dict config: :param str vcd_host: :param str vcd_username: :param str vcd_password: :param bool verify_ssl: :param bool is_legacy_mode: :param bool is_mqtt_exchange: :param bool log_wire: :param str log_wire_file: :return: the updated config file :rtype: dict """ # Compute common supported api versions by the CSE server and vCD sysadmin_client = None try: sysadmin_client = Client( vcd_host, verify_ssl_certs=verify_ssl, log_file=log_wire_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) sysadmin_client.set_credentials( BasicLoginCredentials( vcd_username, SYSTEM_ORG_NAME, vcd_password ) ) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS) common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() if is_legacy_mode: common_supported_api_versions = \ [x for x in common_supported_api_versions if VCDApiVersion(x) < VcdApiVersionObj.VERSION_35.value] else: common_supported_api_versions = \ [x for x in common_supported_api_versions if VCDApiVersion(x) >= VcdApiVersionObj.VERSION_35.value] config['service']['supported_api_versions'] = \ common_supported_api_versions finally: if sysadmin_client: sysadmin_client.logout() # Convert legacy_mode flag in service_section to corresponding # feature flags if 'feature_flags' not in config: config['feature_flags'] = {} config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode) config['feature_flags']['non_legacy_api'] = \ not str_to_bool(is_legacy_mode) # Compute the default api version as the max supported version # Also compute the RDE version in use max_vcd_api_version_supported: str = get_max_api_version(config['service']['supported_api_versions']) # noqa: E501 config['service']['default_api_version'] = max_vcd_api_version_supported config['service']['rde_version_in_use'] = semantic_version.Version( rde_utils.get_runtime_rde_version_by_vcd_api_version( max_vcd_api_version_supported ) ) # Update the config dict with telemetry specific key value pairs update_with_telemetry_settings( config_dict=config, vcd_host=vcd_host, vcd_username=vcd_username, vcd_password=vcd_password, verify_ssl=verify_ssl, is_mqtt_exchange=is_mqtt_exchange ) return config
def check_cse_installation(config, msg_update_callback=None): """Ensure that CSE is installed on vCD according to the config file. Checks, 1. AMQP exchange exists 2. CSE is registered with vCD, 3. CSE K8 catalog exists :param dict config: config yaml file as a dictionary :param utils.ConsoleMessagePrinter msg_update_callback: Callback object that writes messages onto console. :raises Exception: if CSE is not registered to vCD as an extension, or if specified catalog does not exist, or if specified template(s) do not exist. """ if msg_update_callback: msg_update_callback.info( "Validating CSE installation according to config file") err_msgs = [] client = None try: log_filename = None log_wire = str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = SERVER_DEBUG_WIRELOG_FILEPATH client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) # check that AMQP exchange exists amqp = config['amqp'] credentials = pika.PlainCredentials(amqp['username'], amqp['password']) parameters = pika.ConnectionParameters(amqp['host'], amqp['port'], amqp['vhost'], credentials, ssl=amqp['ssl'], connection_attempts=3, retry_delay=2, socket_timeout=5) connection = None try: connection = pika.BlockingConnection(parameters) channel = connection.channel() try: channel.exchange_declare(exchange=amqp['exchange'], exchange_type=EXCHANGE_TYPE, durable=True, passive=True, auto_delete=False) if msg_update_callback: msg_update_callback.general( f"AMQP exchange '{amqp['exchange']}' exists") except pika.exceptions.ChannelClosed: msg = f"AMQP exchange '{amqp['exchange']}' does not exist" if msg_update_callback: msg_update_callback.error(msg) err_msgs.append(msg) except Exception: # TODO() replace raw exception with specific msg = f"Could not connect to AMQP exchange '{amqp['exchange']}'" if msg_update_callback: msg_update_callback.error(msg) err_msgs.append(msg) finally: if connection is not None: connection.close() # check that CSE is registered to vCD correctly ext = APIExtension(client) try: cse_info = ext.get_extension(CSE_SERVICE_NAME, namespace=CSE_SERVICE_NAMESPACE) rkey_matches = cse_info['routingKey'] == amqp['routing_key'] exchange_matches = cse_info['exchange'] == amqp['exchange'] if not rkey_matches or not exchange_matches: msg = "CSE is registered as an extension, but the extension " \ "settings on vCD are not the same as config settings." if not rkey_matches: msg += f"\nvCD-CSE routing key: {cse_info['routingKey']}" \ f"\nCSE config routing key: {amqp['routing_key']}" if not exchange_matches: msg += f"\nvCD-CSE exchange: {cse_info['exchange']}" \ f"\nCSE config exchange: {amqp['exchange']}" if msg_update_callback: msg_update_callback.info(msg) err_msgs.append(msg) if cse_info['enabled'] == 'true': if msg_update_callback: msg_update_callback.general( "CSE on vCD is currently enabled") else: if msg_update_callback: msg_update_callback.info( "CSE on vCD is currently disabled") except MissingRecordException: msg = "CSE is not registered to vCD" if msg_update_callback: msg_update_callback.error(msg) err_msgs.append(msg) # check that catalog exists in vCD org_name = config['broker']['org'] org = get_org(client, org_name=org_name) catalog_name = config['broker']['catalog'] if catalog_exists(org, catalog_name): if msg_update_callback: msg_update_callback.general(f"Found catalog '{catalog_name}'") else: msg = f"Catalog '{catalog_name}' not found" if msg_update_callback: msg_update_callback.error(msg) err_msgs.append(msg) finally: if client: client.logout() if err_msgs: raise Exception(err_msgs) if msg_update_callback: msg_update_callback.general("CSE installation is valid")
name=network['network_name'], type=FenceMode.ISOLATED.value) if len(new_network_list) > 0: print("Isolated network is visible in VDC: {0}".format( network['network_name'])) network_exists = True else: print("Isolated network is not visible yet: {0}".format( network['network_name'])) time.sleep(3) # Check for vApps and create them if they don't exist. We have to # reload the VDC object so it has all the links to current vApps. vdc.reload() for vapp_cfg in cfg.vapps: try: vapp = vdc.get_vapp(vapp_cfg['name']) print("vApp exists: {0}".format(vapp_cfg['name'])) except Exception: print("vApp does not exist: name={0}".format(vapp_cfg['name'])) vapp_resource = vdc.instantiate_vapp(**vapp_cfg) print("vApp instantiated") # We don't track the task as instantiating a vApp takes a while. # Uncomment below if you want to ensure the vApps are available. # handle_task(client, vapp_resource.Tasks.Task[0]) # Log out. print("All done!") client.logout()
def _load_template_definition_from_catalog( self, msg_update_callback=utils.NullPrinter()): # NOTE: If `enable_tkg_plus` in the config file is set to false, # CSE server will skip loading the TKG+ template this will prevent # users from performing TKG+ related operations. msg = "Loading k8s template definition from catalog" logger.SERVER_LOGGER.info(msg) msg_update_callback.general_no_color(msg) client = None try: log_filename = None log_wire = \ utils.str_to_bool(self.config['service'].get('log_wire')) if log_wire: log_filename = logger.SERVER_DEBUG_WIRELOG_FILEPATH # Since the config param has been read from file by # get_validated_config method, we can safely use the # default_api_version key, it will be set to the highest api # version supported by VCD and CSE. client = Client( self.config['vcd']['host'], api_version=self.config['service']['default_api_version'], verify_ssl_certs=self.config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials( self.config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, # noqa: E501 self.config['vcd']['password']) client.set_credentials(credentials) is_tkg_plus_enabled = server_utils.is_tkg_plus_enabled(self.config) legacy_mode = self.config['service']['legacy_mode'] org_name = self.config['broker']['org'] catalog_name = self.config['broker']['catalog'] k8_templates = ltm.get_valid_k8s_local_template_definition( client=client, catalog_name=catalog_name, org_name=org_name, legacy_mode=legacy_mode, is_tkg_plus_enabled=is_tkg_plus_enabled, logger_debug=logger.SERVER_LOGGER, msg_update_callback=msg_update_callback) if not k8_templates: msg = "No valid K8 templates were found in catalog " \ f"'{catalog_name}'. Unable to start CSE server." msg_update_callback.error(msg) logger.SERVER_LOGGER.error(msg) sys.exit(1) # Check that default k8s template exists in vCD at the correct # revision default_template_name = \ self.config['broker']['default_template_name'] default_template_revision = \ str(self.config['broker']['default_template_revision']) found_default_template = False for template in k8_templates: if str(template[server_constants.LocalTemplateKey.REVISION]) == default_template_revision and \ template[server_constants.LocalTemplateKey.NAME] == default_template_name: # noqa: E501 found_default_template = True if not found_default_template: msg = f"Default template {default_template_name} with " \ f"revision {default_template_revision} not found." \ " Unable to start CSE server." msg_update_callback.error(msg) logger.SERVER_LOGGER.error(msg) sys.exit(1) self.config['broker']['templates'] = k8_templates finally: if client: client.logout()
def convert_cluster(ctx, config_file_name, cluster_name, password, org_name, vdc_name, skip_wait_for_gc): try: check_python_version() except Exception as err: click.secho(str(err), fg='red') sys.exit(1) client = None try: console_message_printer = ConsoleMessagePrinter() config = get_validated_config( config_file_name, msg_update_callback=console_message_printer) log_filename = None log_wire = str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = 'cluster_convert_wire.log' client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" console_message_printer.general(msg) cluster_records = get_all_clusters(client=client, cluster_name=cluster_name, org_name=org_name, ovdc_name=vdc_name) if len(cluster_records) == 0: console_message_printer.info(f"No clusters were found.") return vms = [] for cluster in cluster_records: console_message_printer.info( f"Processing cluster '{cluster['name']}'.") vapp_href = cluster['vapp_href'] vapp = VApp(client, href=vapp_href) console_message_printer.info("Processing metadata of cluster.") metadata = metadata_to_dict(vapp.get_metadata()) old_template_name = None new_template_name = None if ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME in metadata: # noqa: E501 old_template_name = metadata.pop(ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME) # noqa: E501 version = metadata.get(ClusterMetadataKey.CSE_VERSION) if old_template_name: console_message_printer.info( "Determining k8s version on cluster.") if 'photon' in old_template_name: new_template_name = 'photon-v2' if '1.0.0' in version: new_template_name += '_k8s-1.8_weave-2.0.5' elif any(ver in version for ver in ('1.1.0', '1.2.0', '1.2.1', '1.2.2', '1.2.3', '1.2.4',)): # noqa: E501 new_template_name += '_k8s-1.9_weave-2.3.0' elif any(ver in version for ver in ('1.2.5', '1.2.6', '1.2.7',)): # noqa: E501 new_template_name += '_k8s-1.10_weave-2.3.0' elif '2.0.0' in version: new_template_name += '_k8s-1.12_weave-2.3.0' elif 'ubuntu' in old_template_name: new_template_name = 'ubuntu-16.04' if '1.0.0' in version: new_template_name += '_k8s-1.9_weave-2.1.3' elif any(ver in version for ver in ('1.1.0', '1.2.0', '1.2.1', '1.2.2', '1.2.3', '1.2.4', '1.2.5', '1.2.6', '1.2.7')): # noqa: E501 new_template_name += '_k8s-1.10_weave-2.3.0' elif '2.0.0' in version: new_template_name += '_k8s-1.13_weave-2.3.0' if new_template_name: console_message_printer.info("Updating metadata of cluster.") task = vapp.remove_metadata(ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME) # noqa: E501 client.get_task_monitor().wait_for_success(task) new_metadata_to_add = { ClusterMetadataKey.TEMPLATE_NAME: new_template_name, ClusterMetadataKey.TEMPLATE_REVISION: 0 } task = vapp.set_multiple_metadata(new_metadata_to_add) client.get_task_monitor().wait_for_success(task) console_message_printer.general( "Finished processing metadata of cluster.") try: console_message_printer.info( f"Undeploying the vApp '{cluster['name']}'") task = vapp.undeploy() client.get_task_monitor().wait_for_success(task) console_message_printer.general( "Successfully undeployed the vApp.") except Exception as err: console_message_printer.error(str(err)) vm_resources = vapp.get_all_vms() for vm_resource in vm_resources: console_message_printer.info( f"Processing vm '{vm_resource.get('name')}'.") vm = VM(client, href=vm_resource.get('href')) vms.append(vm) console_message_printer.info("Updating vm admin password.") task = vm.update_guest_customization_section( enabled=True, admin_password_enabled=True, admin_password_auto=not password, admin_password=password, ) client.get_task_monitor().wait_for_success(task) console_message_printer.general("Successfully updated vm .") console_message_printer.info("Deploying vm.") task = vm.power_on_and_force_recustomization() client.get_task_monitor().wait_for_success(task) console_message_printer.general("Successfully deployed vm.") console_message_printer.info("Deploying cluster") task = vapp.deploy(power_on=True) client.get_task_monitor().wait_for_success(task) console_message_printer.general("Successfully deployed cluster.") console_message_printer.general( f"Successfully processed cluster '{cluster['name']}'.") if skip_wait_for_gc: return while True: for vm in vms: status = vm.get_guest_customization_status() if status != 'GC_PENDING': vms.remove(vm) console_message_printer.info( f"Waiting on guest customization to finish on {len(vms)} vms.") if not len(vms) == 0: time.sleep(5) else: break except Exception as err: click.secho(str(err), fg='red') finally: if client: client.logout()
def _load_template_definition_from_catalog( self, msg_update_callback=utils.NullPrinter()): # noqa: E501 msg = "Loading k8s template definition from catalog" logger.SERVER_LOGGER.info(msg) msg_update_callback.general_no_color(msg) client = None try: log_filename = None log_wire = \ utils.str_to_bool(self.config['service'].get('log_wire')) if log_wire: log_filename = logger.SERVER_DEBUG_WIRELOG_FILEPATH client = Client(self.config['vcd']['host'], api_version=self.config['vcd']['api_version'], verify_ssl_certs=self.config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(self.config['vcd']['username'], SYSTEM_ORG_NAME, self.config['vcd']['password']) client.set_credentials(credentials) org_name = self.config['broker']['org'] catalog_name = self.config['broker']['catalog'] k8_templates = ltm.get_all_k8s_local_template_definition( client=client, catalog_name=catalog_name, org_name=org_name) if not k8_templates: msg = "No valid K8 templates were found in catalog " \ f"'{catalog_name}'. Unable to start CSE server." msg_update_callback.error(msg) logger.SERVER_LOGGER.error(msg) sys.exit(1) # Check that default k8s template exists in vCD at the correct # revision default_template_name = \ self.config['broker']['default_template_name'] default_template_revision = \ str(self.config['broker']['default_template_revision']) found_default_template = False for template in k8_templates: if str(template[LocalTemplateKey.REVISION] ) == default_template_revision and template[ LocalTemplateKey. NAME] == default_template_name: # noqa: E501 found_default_template = True msg = f"Found K8 template '{template['name']}' at revision " \ f"{template['revision']} in catalog '{catalog_name}'" msg_update_callback.general(msg) logger.SERVER_LOGGER.info(msg) if not found_default_template: msg = f"Default template {default_template_name} with " \ f"revision {default_template_revision} not found." \ " Unable to start CSE server." msg_update_callback.error(msg) logger.SERVER_LOGGER.error(msg) sys.exit(1) self.config['broker']['templates'] = k8_templates finally: if client: client.logout()
def get_validated_config(config_file_name, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, log_wire_file=None, logger_debug=NULL_LOGGER, msg_update_callback=NullPrinter()): """Get the config file as a dictionary and check for validity. Ensures that all properties exist and all values are the expected type. Checks that AMQP connection is available, and vCD/VCs are valid. Does not guarantee that CSE has been installed according to this config file. :param str config_file_name: path to config file. :param str pks_config_file_name: path to PKS config file. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param str log_wire_file: log_wire_file to use if needed to wire log pyvcloud requests and responses :param logging.Logger logger_debug: logger to log with. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :return: CSE config :rtype: dict :raises KeyError: if config file has missing or extra properties. :raises TypeError: if the value type for a config file property is incorrect. :raises container_service_extension.exceptions.AmqpConnectionError: (when not using MQTT) if AMQP connection failed (host, password, port, username, vhost is invalid). :raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid. :raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or 'password' is invalid. :raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password' is invalid. """ check_file_permissions(config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(config_file_name) as config_file: config = yaml.safe_load(config_file) or {} else: msg_update_callback.info( f"Decrypting '{config_file_name}'") try: config = yaml.safe_load( get_decrypted_file_contents(config_file_name, decryption_password)) or {} except cryptography.fernet.InvalidToken: raise Exception(CONFIG_DECRYPTION_ERROR_MSG) msg_update_callback.info( f"Validating config file '{config_file_name}'") # This allows us to compare top-level config keys and value types use_mqtt = should_use_mqtt_protocol(config) sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \ else SAMPLE_MQTT_CONFIG sample_config = { **sample_message_queue_config, **SAMPLE_VCD_CONFIG, **SAMPLE_VCS_CONFIG, **SAMPLE_SERVICE_CONFIG, **SAMPLE_BROKER_CONFIG } log_wire = str_to_bool(config.get('service', {}).get('log_wire')) nsxt_wire_logger = NULL_LOGGER if not log_wire: log_wire_file = None nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER check_keys_and_value_types(config, sample_config, location='config file', msg_update_callback=msg_update_callback) # MQTT validation not required because no MQTT host, exchange, etc. # is needed in the config file since the server code creates and # registers the MQTT extension directly using server constants if not use_mqtt: _validate_amqp_config(config['amqp'], msg_update_callback) try: _validate_vcd_and_vcs_config(config['vcd'], config['vcs'], msg_update_callback, log_file=log_wire_file, log_wire=log_wire) except vim.fault.InvalidLogin: raise Exception(VCENTER_LOGIN_ERROR_MSG) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") except requests.exceptions.ConnectionError as err: raise Exception(f"Cannot connect to {err.request.url}.") _validate_broker_config(config['broker'], legacy_mode=config['service']['legacy_mode'], msg_update_callback=msg_update_callback, logger_debug=logger_debug) check_keys_and_value_types(config['service'], SAMPLE_SERVICE_CONFIG['service'], location="config file 'service' section", excluded_keys=['log_wire'], msg_update_callback=msg_update_callback) check_keys_and_value_types(config['service']['telemetry'], SAMPLE_SERVICE_CONFIG['service']['telemetry'], location="config file 'service->telemetry' " "section", msg_update_callback=msg_update_callback) msg_update_callback.general( f"Config file '{config_file_name}' is valid") if pks_config_file_name: check_file_permissions(pks_config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(pks_config_file_name) as f: pks_config = yaml.safe_load(f) or {} else: msg_update_callback.info( f"Decrypting '{pks_config_file_name}'") pks_config = yaml.safe_load( get_decrypted_file_contents(pks_config_file_name, decryption_password)) or {} msg_update_callback.info( f"Validating PKS config file '{pks_config_file_name}'") _validate_pks_config_structure(pks_config, msg_update_callback) try: _validate_pks_config_data_integrity(pks_config, msg_update_callback, logger_debug=logger_debug, logger_wire=nsxt_wire_logger) except requests.exceptions.SSLError as err: raise Exception(f"SSL verification failed: {str(err)}") msg_update_callback.general( f"PKS Config file '{pks_config_file_name}' is valid") config['pks_config'] = pks_config else: config['pks_config'] = None # Compute common supported api versions by the CSE server and vCD sysadmin_client = None try: sysadmin_client = Client( config['vcd']['host'], verify_ssl_certs=config['vcd']['verify'], log_file=log_wire_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) sysadmin_client.set_credentials(BasicLoginCredentials( config['vcd']['username'], SYSTEM_ORG_NAME, config['vcd']['password'])) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS) common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() config['service']['supported_api_versions'] = \ common_supported_api_versions finally: if sysadmin_client: sysadmin_client.logout() # Convert legacy_mode flag in service_section to corresponding # feature flags is_legacy_mode = config['service']['legacy_mode'] if 'feature_flags' not in config: config['feature_flags'] = {} config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode) config['feature_flags']['non_legacy_api'] = \ not str_to_bool(is_legacy_mode) # Temporary work around before api version is completely removed from # config if is_legacy_mode: supported_api_versions_float = \ [float(x) for x in config['service']['supported_api_versions'] if float(x) < 35.0] else: supported_api_versions_float = \ [float(x) for x in config['service']['supported_api_versions'] if float(x) >= 35.0] config['vcd']['api_version'] = str(max(supported_api_versions_float)) # Store telemetry instance id, url and collector id in config # This steps needs to be done after api_version has been computed # and stored in the config store_telemetry_settings(config) return config
class VmwareCloudSystem(System): """Client to VMware vCloud API""" def __init__(self, hostname, username, organization, password, api_port, api_version, **kwargs): super(VmwareCloudSystem, self).__init__(**kwargs) self.endpoint = 'https://{}:{}'.format(hostname, api_port) self.username = username self.organization = organization self.password = password self.api_version = api_version self._client = None def info(self): return 'VmwareCloudSystem endpoint={}, api_version={}'.format( self.endpoint, self.api_version) @property def client(self): if self._client is None: self._client = Client( self.endpoint, api_version=self.api_version, verify_ssl_certs=False, ) self.client.set_credentials( BasicLoginCredentials(self.username, self.organization, self.password)) return self._client def stats(self, *requested_stats): stats = self.count_vcloud(self.client) return {stat: stats[stat] for stat in requested_stats} def count_vcloud(self, client): """ Obtain counts via vCloud API. Multiple dependent requests are needed therefore we collect them all in one pass to avoid repeating previous requests e.g. to fetch VMs, one must first fetch vApps and vdcs. :param client: :return: """ org_resource = client.get_org() org = Org(client, resource=org_resource) stats = { 'num_availability_zone': 0, 'num_orchestration_stack': 0, 'num_vm': 0 } for vdc_info in org.list_vdcs(): stats['num_availability_zone'] += 1 vdc = VDC(client, resource=org.get_vdc(vdc_info['name'])) for vapp_info in vdc.list_resources(): try: vapp_resource = vdc.get_vapp(vapp_info.get('name')) except Exception: continue # not a vapp (probably vapp template or something) vapp = VApp(client, resource=vapp_resource) stats['num_orchestration_stack'] += 1 stats['num_vm'] += len(vapp.get_all_vms()) return stats def disconnect(self): if self._client is not None: self._client.logout() self._client = None
def _validate_vcd_and_vcs_config( vcd_dict, vcs, msg_update_callback=NullPrinter(), log_file=None, log_wire=False ): """Ensure that 'vcd' and vcs' section of config are correct. Checks that * 'vcd' and 'vcs' section of config have correct keys and value types. * vCD and all registered VCs in vCD are accessible. * api version specified for vcd is supported by CSE. :param dict vcd_dict: 'vcd' section of config file as a dict. :param list vcs: 'vcs' section of config file as a list of dicts. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :param str log_file: log_file for pyvcloud wire log. :param bool log_wire: If pyvcloud requests should be logged. :raises KeyError: if @vcd_dict or a vc in @vcs has missing or extra properties. :raises TypeError: if the value type for a @vcd_dict or vc property is incorrect. :raises ValueError: if vCD has a VC that is not listed in the config file. """ check_keys_and_value_types(vcd_dict, SAMPLE_VCD_CONFIG['vcd'], location="config file 'vcd' section", msg_update_callback=msg_update_callback) if not vcd_dict['verify']: msg_update_callback.general( 'InsecureRequestWarning: Unverified HTTPS request is ' 'being made. Adding certificate verification is ' 'strongly advised.' ) requests.packages.urllib3.disable_warnings() client = None try: _validate_vcd_url_scheme(vcd_dict['host']) client = Client( vcd_dict['host'], verify_ssl_certs=vcd_dict['verify'], log_file=log_file, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire ) client.set_credentials( BasicLoginCredentials( vcd_dict['username'], SYSTEM_ORG_NAME, vcd_dict['password'] ) ) msg_update_callback.general( "Connected to vCloud Director " f"({vcd_dict['host']}:{vcd_dict['port']})" ) for index, vc in enumerate(vcs, 1): check_keys_and_value_types( vc, SAMPLE_VCS_CONFIG['vcs'][0], location=f"config file 'vcs' section, vc #{index}", msg_update_callback=msg_update_callback ) # Check that all registered VCs in vCD are listed in config file platform = Platform(client) config_vc_names = [vc['name'] for vc in vcs] for platform_vc in platform.list_vcenters(): platform_vc_name = platform_vc.get('name') if platform_vc_name not in config_vc_names: raise ValueError( f"vCenter '{platform_vc_name}' registered in " "vCD but not found in config file" ) # Check that all VCs listed in config file are registered in vCD for vc in vcs: vcenter = platform.get_vcenter(vc['name']) if not (hasattr(vcenter, 'IsConnected') and vcenter.IsConnected): msg = f"vCenter Server '{vc['name']}' not available" msg_update_callback.info(msg) continue vsphere_url = urlparse(vcenter.Url.text) vsphere_url_port = vsphere_url.port if vsphere_url_port: v = VSphere( vsphere_url.hostname, vc['username'], vc['password'], vsphere_url.port ) else: v = VSphere( vsphere_url.hostname, vc['username'], vc['password'] ) v.connect() msg = f"Connected to vCenter Server '{vc['name']}' as " \ f"'{vc['username']}' ({vsphere_url.hostname}" if vsphere_url_port: msg += f":{vsphere_url.port}" msg += ")" msg_update_callback.general(msg) finally: if client is not None: client.logout()