def install_cse(config_file_name, skip_template_creation, force_update, ssh_key, retain_temp_vapp, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, msg_update_callback=utils.NullPrinter()): """Handle logistics for CSE installation. Handles decision making for configuring AMQP exchange/settings, defined entity schema registration for vCD api version >= 35, extension registration, catalog setup and template creation. Also records telemetry data on installation details. :param str config_file_name: config file name. :param bool skip_template_creation: If True, skip creating the templates. :param bool force_update: if True and templates already exist in vCD, overwrites existing templates. :param str ssh_key: public ssh key to place into template vApp(s). :param bool retain_temp_vapp: if True, temporary vApp will not destroyed, so the user can ssh into and debug the vm. :param str pks_config_file_name: pks config file name. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :raises cse_exception.AmqpError: if AMQP exchange could not be created. """ config = get_validated_config( config_file_name, pks_config_file_name=pks_config_file_name, skip_config_decryption=skip_config_decryption, decryption_password=decryption_password, log_wire_file=INSTALL_WIRELOG_FILEPATH, logger_debug=INSTALL_LOGGER, msg_update_callback=msg_update_callback) populate_vsphere_list(config['vcs']) msg = f"Installing CSE on vCloud Director using config file " \ f"'{config_file_name}'" msg_update_callback.info(msg) INSTALL_LOGGER.info(msg) client = None try: # Telemetry - Construct telemetry data telemetry_data = { PayloadKey.WAS_DECRYPTION_SKIPPED: bool(skip_config_decryption), # noqa: E501 PayloadKey.WAS_PKS_CONFIG_FILE_PROVIDED: bool(pks_config_file_name), # noqa: E501 PayloadKey.WERE_TEMPLATES_SKIPPED: bool(skip_template_creation), # noqa: E501 PayloadKey.WERE_TEMPLATES_FORCE_UPDATED: bool(force_update), # noqa: E501 PayloadKey.WAS_TEMP_VAPP_RETAINED: bool(retain_temp_vapp), # noqa: E501 PayloadKey.WAS_SSH_KEY_SPECIFIED: bool(ssh_key) # noqa: E501 } # Telemetry - Record detailed telemetry data on install record_user_action_details( CseOperation.SERVICE_INSTALL, telemetry_data, telemetry_settings=config['service']['telemetry']) # noqa: E501 log_filename = None log_wire = utils.str_to_bool(config['service'].get('log_wire')) if log_wire: log_filename = INSTALL_WIRELOG_FILEPATH client = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = BasicLoginCredentials(config['vcd']['username'], server_constants.SYSTEM_ORG_NAME, config['vcd']['password']) client.set_credentials(credentials) msg = f"Connected to vCD as system administrator: " \ f"{config['vcd']['host']}:{config['vcd']['port']}" msg_update_callback.general(msg) INSTALL_LOGGER.info(msg) # create amqp exchange if it doesn't exist amqp = config['amqp'] _create_amqp_exchange(amqp['exchange'], amqp['host'], amqp['port'], amqp['vhost'], amqp['ssl'], amqp['username'], amqp['password'], msg_update_callback=msg_update_callback) # register or update cse on vCD _register_cse(client, amqp['routing_key'], amqp['exchange'], msg_update_callback=msg_update_callback) # register cse def schema on VCD # schema should be located at # ~/.cse-schema/api-v<API VERSION>/schema.json _register_def_schema(client, msg_update_callback=msg_update_callback, log_wire=log_wire) # Since we use CSE extension id as our telemetry instance_id, the # validated config won't have the instance_id yet. Now that CSE has # been registered as an extension, we should update the telemetry # config with the correct instance_id if config['service']['telemetry']['enable']: store_telemetry_settings(config) # register rights to vCD # TODO() should also remove rights when unregistering CSE _register_right( client, right_name=server_constants. CSE_NATIVE_DEPLOY_RIGHT_NAME, # noqa: E501 description=server_constants. CSE_NATIVE_DEPLOY_RIGHT_DESCRIPTION, # noqa: E501 category=server_constants. CSE_NATIVE_DEPLOY_RIGHT_CATEGORY, # noqa: E501 bundle_key=server_constants. CSE_NATIVE_DEPLOY_RIGHT_BUNDLE_KEY, # noqa: E501 msg_update_callback=msg_update_callback) _register_right( client, right_name=server_constants. CSE_PKS_DEPLOY_RIGHT_NAME, # noqa: E501 description=server_constants. CSE_PKS_DEPLOY_RIGHT_DESCRIPTION, # noqa: E501 category=server_constants. CSE_PKS_DEPLOY_RIGHT_CATEGORY, # noqa: E501 bundle_key=server_constants. CSE_PKS_DEPLOY_RIGHT_BUNDLE_KEY, # noqa: E501 msg_update_callback=msg_update_callback) # set up placement policies for all types of clusters _setup_placement_policies( client, policy_list=server_constants. CLUSTER_PLACEMENT_POLICIES, # noqa: E501 msg_update_callback=msg_update_callback, log_wire=log_wire) # set up cse catalog org = vcd_utils.get_org(client, org_name=config['broker']['org']) vcd_utils.create_and_share_catalog( org, config['broker']['catalog'], catalog_desc='CSE templates', logger=INSTALL_LOGGER, msg_update_callback=msg_update_callback) if skip_template_creation: msg = "Skipping creation of templates." msg_update_callback.info(msg) INSTALL_LOGGER.warning(msg) else: # read remote template cookbook, download all scripts rtm = RemoteTemplateManager( remote_template_cookbook_url=config['broker'] ['remote_template_cookbook_url'], # noqa: E501 logger=INSTALL_LOGGER, msg_update_callback=msg_update_callback) remote_template_cookbook = rtm.get_remote_template_cookbook() # create all templates defined in cookbook for template in remote_template_cookbook['templates']: # TODO tag created templates with placement policies _install_template( client=client, remote_template_manager=rtm, template=template, org_name=config['broker']['org'], vdc_name=config['broker']['vdc'], catalog_name=config['broker']['catalog'], network_name=config['broker']['network'], ip_allocation_mode=config['broker']['ip_allocation_mode'], storage_profile=config['broker']['storage_profile'], force_update=force_update, retain_temp_vapp=retain_temp_vapp, ssh_key=ssh_key, msg_update_callback=msg_update_callback) # if it's a PKS setup, setup NSX-T constructs if config.get('pks_config'): nsxt_servers = config['pks_config']['nsxt_servers'] wire_logger = NULL_LOGGER if log_wire: wire_logger = SERVER_NSXT_WIRE_LOGGER for nsxt_server in nsxt_servers: msg = f"Configuring NSX-T server ({nsxt_server.get('name')})" \ " for CSE. Please check install logs for details." msg_update_callback.general(msg) INSTALL_LOGGER.info(msg) nsxt_client = NSXTClient(host=nsxt_server.get('host'), username=nsxt_server.get('username'), password=nsxt_server.get('password'), logger_debug=INSTALL_LOGGER, logger_wire=wire_logger, http_proxy=nsxt_server.get('proxy'), https_proxy=nsxt_server.get('proxy'), verify_ssl=nsxt_server.get('verify')) setup_nsxt_constructs( nsxt_client=nsxt_client, nodes_ip_block_id=nsxt_server.get('nodes_ip_block_ids'), pods_ip_block_id=nsxt_server.get('pods_ip_block_ids'), ncp_boundary_firewall_section_anchor_id=nsxt_server.get( 'distributed_firewall_section_anchor_id') ) # noqa: E501 # Telemetry - Record successful install action record_user_action(CseOperation.SERVICE_INSTALL, telemetry_settings=config['service']['telemetry']) except Exception: msg_update_callback.error( "CSE Installation Error. Check CSE install logs") INSTALL_LOGGER.error("CSE Installation Error", exc_info=True) # Telemetry - Record failed install action record_user_action(CseOperation.SERVICE_INSTALL, status=OperationStatus.FAILED, telemetry_settings=config['service']['telemetry']) raise # TODO() need installation relevant exceptions for rollback finally: if client is not None: client.logout()
def get_validated_config(config_file_name, pks_config_file_name=None, skip_config_decryption=False, decryption_password=None, log_wire_file=None, logger_debug=NULL_LOGGER, msg_update_callback=NullPrinter()): """Get the config file as a dictionary and check for validity. Ensures that all properties exist and all values are the expected type. Checks that AMQP connection is available, and vCD/VCs are valid. Does not guarantee that CSE has been installed according to this config file. :param str config_file_name: path to config file. :param str pks_config_file_name: path to PKS config file. :param bool skip_config_decryption: do not decrypt the config file. :param str decryption_password: password to decrypt the config file. :param str log_wire_file: log_wire_file to use if needed to wire log pyvcloud requests and responses :param logging.Logger logger: logger to log with. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. :return: CSE config :rtype: dict :raises KeyError: if config file has missing or extra properties. :raises TypeError: if the value type for a config file property is incorrect. :raises container_service_extension.exceptions.AmqpConnectionError: if AMQP connection failed (host, password, port, username, vhost is invalid). :raises pyvcloud.vcd.exceptions.NotAcceptableException: if 'vcd' 'api_version' is unsupported. :raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid. :raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or 'password' is invalid. :raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password' is invalid. """ check_file_permissions(config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(config_file_name) as config_file: config = yaml.safe_load(config_file) or {} else: msg_update_callback.info( f"Decrypting '{config_file_name}'") config = yaml.safe_load( get_decrypted_file_contents(config_file_name, decryption_password)) or {} msg_update_callback.info( f"Validating config file '{config_file_name}'") # This allows us to compare top-level config keys and value types sample_config = { **SAMPLE_AMQP_CONFIG, **SAMPLE_VCD_CONFIG, **SAMPLE_VCS_CONFIG, **SAMPLE_SERVICE_CONFIG, **SAMPLE_BROKER_CONFIG } log_wire = str_to_bool(config.get('service', {}).get('log_wire')) nsxt_wire_logger = NULL_LOGGER if not log_wire: log_wire_file = None nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER check_keys_and_value_types(config, sample_config, location='config file', msg_update_callback=msg_update_callback) _validate_amqp_config(config['amqp'], msg_update_callback) _validate_vcd_and_vcs_config(config['vcd'], config['vcs'], msg_update_callback, log_file=log_wire_file, log_wire=log_wire) _validate_broker_config(config['broker'], msg_update_callback, logger_debug) check_keys_and_value_types(config['service'], SAMPLE_SERVICE_CONFIG['service'], location="config file 'service' section", excluded_keys=['log_wire'], msg_update_callback=msg_update_callback) check_keys_and_value_types(config['service']['telemetry'], SAMPLE_SERVICE_CONFIG['service']['telemetry'], location="config file 'service->telemetry' " "section", msg_update_callback=msg_update_callback) msg_update_callback.general( f"Config file '{config_file_name}' is valid") if pks_config_file_name: check_file_permissions(pks_config_file_name, msg_update_callback=msg_update_callback) if skip_config_decryption: with open(pks_config_file_name) as f: pks_config = yaml.safe_load(f) or {} else: msg_update_callback.info( f"Decrypting '{pks_config_file_name}'") pks_config = yaml.safe_load( get_decrypted_file_contents(pks_config_file_name, decryption_password)) or {} msg_update_callback.info( f"Validating PKS config file '{pks_config_file_name}'") _validate_pks_config_structure(pks_config, msg_update_callback) _validate_pks_config_data_integrity(pks_config, msg_update_callback, logger_debug=logger_debug, logger_wire=nsxt_wire_logger) msg_update_callback.general( f"PKS Config file '{pks_config_file_name}' is valid") config['pks_config'] = pks_config else: config['pks_config'] = None # Store telemetry instance id, url and collector id in config store_telemetry_settings(config) return config