def assign_placement_policy_to_template(
client,
cse_placement_policy,
catalog_name,
catalog_item_name,
org_name,
logger=NULL_LOGGER,
log_wire=False,
msg_update_callback=NullPrinter()): # noqa: E501
cpm = compute_policy_manager.ComputePolicyManager(client,
log_wire=log_wire)
try:
policy = compute_policy_manager.get_cse_vdc_compute_policy(
cpm, cse_placement_policy, is_placement_policy=True)
task = cpm.assign_vdc_placement_policy_to_vapp_template_vms(
policy['href'], org_name, catalog_name, catalog_item_name)
if task is not None:
client.get_task_monitor().wait_for_success(task)
msg = "Successfully tagged template " \
f"{catalog_item_name} with placement policy " \
f"{cse_placement_policy}."
else:
msg = f"{catalog_item_name} already tagged with" \
f" placement policy {cse_placement_policy}."
msg_update_callback.general(msg)
logger.info(msg)
except Exception as err:
msg = f"Failed to tag template {catalog_item_name} with " \
f"placement policy {cse_placement_policy}. Error: {err}"
msg_update_callback.error(msg)
logger.error(msg)
raise
def _validate_vcd_config(
vcd_dict,
msg_update_callback=NullPrinter(),
log_file=None,
log_wire=False
):
"""Ensure that 'vcd' section of config is correct.
Checks that
* 'vcd' section of config has correct keys and value types.
* vCD is accessible.
:param dict vcd_dict: 'vcd' section of config file as a dict.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:param str log_file: log_file for pyvcloud wire log.
:param bool log_wire: If pyvcloud requests should be logged.
:raises KeyError: if @vcd_dict has missing
:raises TypeError: if the value type for a @vcd_dict is incorrect.
"""
check_keys_and_value_types(
vcd_dict,
SAMPLE_VCD_CONFIG['vcd'],
location="config file 'vcd' section",
msg_update_callback=msg_update_callback
)
if not vcd_dict['verify']:
msg_update_callback.general(
'InsecureRequestWarning: Unverified HTTPS request is '
'being made. Adding certificate verification is '
'strongly advised.'
)
requests.packages.urllib3.disable_warnings()
client = None
try:
_validate_vcd_url_scheme(vcd_dict['host'])
client = Client(
vcd_dict['host'],
verify_ssl_certs=vcd_dict['verify'],
log_file=log_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire
)
client.set_credentials(
BasicLoginCredentials(
vcd_dict['username'],
SYSTEM_ORG_NAME,
vcd_dict['password']
)
)
msg_update_callback.general(
"Connected to vCloud Director "
f"({vcd_dict['host']}:{vcd_dict['port']})"
)
finally:
if client is not None:
client.logout()
def _validate_pks_config_structure(pks_config,
msg_update_callback=NullPrinter()):
sample_config = {
**SAMPLE_PKS_SERVERS_SECTION,
**SAMPLE_PKS_ACCOUNTS_SECTION,
**SAMPLE_PKS_ORGS_SECTION,
**SAMPLE_PKS_PVDCS_SECTION,
**SAMPLE_PKS_NSXT_SERVERS_SECTION
}
check_keys_and_value_types(pks_config,
sample_config,
location='pks config file',
excluded_keys=[PKS_ORGS_SECTION_KEY],
msg_update_callback=msg_update_callback)
pks_servers = pks_config[PKS_SERVERS_SECTION_KEY]
for index, pks_server in enumerate(pks_servers, 1):
check_keys_and_value_types(
pks_server,
SAMPLE_PKS_SERVERS_SECTION[PKS_SERVERS_SECTION_KEY][0],
location=f"pks config file '{PKS_SERVERS_SECTION_KEY}' "
f"section, pks server #{index}",
excluded_keys=['proxy'],
msg_update_callback=msg_update_callback)
pks_accounts = pks_config[PKS_ACCOUNTS_SECTION_KEY]
for index, pks_account in enumerate(pks_accounts, 1):
check_keys_and_value_types(
pks_account,
SAMPLE_PKS_ACCOUNTS_SECTION[PKS_ACCOUNTS_SECTION_KEY][0],
location=f"pks config file '{PKS_ACCOUNTS_SECTION_KEY}' "
f"section, pks account #{index}",
msg_update_callback=msg_update_callback)
if PKS_ORGS_SECTION_KEY in pks_config.keys():
orgs = pks_config[PKS_ORGS_SECTION_KEY]
for index, org in enumerate(orgs, 1):
check_keys_and_value_types(
org,
SAMPLE_PKS_ORGS_SECTION[PKS_ORGS_SECTION_KEY][0],
location=f"pks config file '{PKS_ORGS_SECTION_KEY}' "
f"section, org #{index}",
msg_update_callback=msg_update_callback)
pvdcs = pks_config[PKS_PVDCS_SECTION_KEY]
for index, pvdc in enumerate(pvdcs, 1):
check_keys_and_value_types(
pvdc,
SAMPLE_PKS_PVDCS_SECTION[PKS_PVDCS_SECTION_KEY][0],
location=f"pks config file '{PKS_PVDCS_SECTION_KEY}' "
f"section, pvdc #{index}",
msg_update_callback=msg_update_callback)
nsxt_servers = pks_config[PKS_NSXT_SERVERS_SECTION_KEY]
for index, nsxt_server in enumerate(nsxt_servers, 1):
check_keys_and_value_types(
nsxt_server,
SAMPLE_PKS_NSXT_SERVERS_SECTION[PKS_NSXT_SERVERS_SECTION_KEY][0],
location=f"pks config file '{PKS_NSXT_SERVERS_SECTION_KEY}' "
f"section, nsxt server #{index}",
excluded_keys=['proxy'],
msg_update_callback=msg_update_callback)
def get_telemetry_instance_id(vcd_host: str,
vcd_username: str,
vcd_password: str,
verify_ssl: bool,
is_mqtt_exchange: bool,
logger_debug=NULL_LOGGER,
msg_update_callback=NullPrinter()):
"""Get CSE AMQP or MQTT extension id which is used as instance id.
Any exception is logged as error. No exception is leaked out
of this method and does not affect the server startup.
:param str vcd_host:
:param str vcd_username:
:param str vcd_password:
:param bool verify_ssl:
:param bool is_mqtt_exchange:
:param logging.logger logger_debug: logger instance to log any error
in retrieving CSE extension id.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return instance id to use for sending data to Vmware telemetry server
:rtype str (unless no instance id found)
"""
client = None
try:
client = Client(vcd_host, verify_ssl_certs=verify_ssl)
client.set_credentials(
BasicLoginCredentials(vcd_username, SYSTEM_ORG_NAME, vcd_password))
if is_mqtt_exchange:
# Get MQTT extension uuid
mqtt_ext_manager = MQTTExtensionManager(client)
ext_info = mqtt_ext_manager.get_extension_info(
ext_name=CSE_SERVICE_NAME,
ext_version=MQTT_EXTENSION_VERSION,
ext_vendor=MQTT_EXTENSION_VENDOR)
if not ext_info:
logger_debug.debug("Failed to retrieve telemetry instance id")
return None
logger_debug.debug("Retrieved telemetry instance id")
return mqtt_ext_manager.get_extension_uuid(
ext_info[MQTTExtKey.EXT_URN_ID])
else:
# Get AMQP extension id
ext = APIExtension(client)
cse_info = ext.get_extension_info(CSE_SERVICE_NAME,
namespace=CSE_SERVICE_NAMESPACE)
logger_debug.debug("Retrieved telemetry instance id")
return cse_info.get('id')
except Exception as err:
msg = f"Cannot retrieve telemetry instance id:{err}"
msg_update_callback.general(msg)
logger_debug.error(msg, exc_info=True)
finally:
if client is not None:
client.logout()
def __init__(self, remote_template_cookbook_url, logger=NULL_LOGGER,
msg_update_callback=NullPrinter()):
""".
:param str remote_template_cookbook_url:
:param logging.Logger logger: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback:
Callback object.
"""
self.url = remote_template_cookbook_url
self.logger = logger
self.msg_update_callback = msg_update_callback
self.cookbook = None
def _validate_broker_config(
broker_dict,
legacy_mode=False,
msg_update_callback=NullPrinter(),
logger_debug=NULL_LOGGER
):
"""Ensure that 'broker' section of config is correct.
Checks that 'broker' section of config has correct keys and value
types. Also checks that 'default_broker' property is a valid template.
:param dict broker_dict: 'broker' section of config file as a dict.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:raises KeyError: if @broker_dict has missing properties.
:raises TypeError: if the value type for a @broker_dict property is
incorrect.
:raises ValueError: if 'ip_allocation_mode' is not 'dhcp' or 'pool'. Or
if remote_template_cookbook_url is invalid.
"""
check_keys_and_value_types(
broker_dict,
SAMPLE_BROKER_CONFIG['broker'],
location="config file 'broker' section",
msg_update_callback=msg_update_callback
)
valid_ip_allocation_modes = [
'dhcp',
'pool'
]
if broker_dict['ip_allocation_mode'] not in valid_ip_allocation_modes:
raise ValueError(
"IP allocation mode is "
f"'{broker_dict['ip_allocation_mode']}' when it "
"should be either 'dhcp' or 'pool'"
)
rtm = RemoteTemplateManager(
remote_template_cookbook_url=broker_dict['remote_template_cookbook_url'], # noqa: E501
legacy_mode=legacy_mode,
logger=logger_debug
)
remote_template_cookbook = rtm.get_filtered_remote_template_cookbook()
if not remote_template_cookbook:
raise Exception("Remote template cookbook is invalid.")
def __init__(self, name, target, action, logger=NULL_LOGGER,
msg_update_callback=NullPrinter()):
"""Initialize TemplateRule object.
:param str name: name of the rule.
:param dict target: target template of the rule. The keys 'name' and
'revision' should be present in the dictionary.
:param dict action: attributes of the target template to update,
accepted keys are 'compute_policy', 'cpu' and 'memory'.
:param logging.Logger logger: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback.
"""
self.name = name
self.target = target
self.action = action
self.logger = logger
self.msg_update_callback = msg_update_callback
def create_and_share_catalog(org,
catalog_name,
catalog_desc='',
logger=NULL_LOGGER,
msg_update_callback=NullPrinter()):
"""Create and share specified catalog.
If catalog does not exist in vCD, create it. Share the specified catalog
to all orgs.
:param pyvcloud.vcd.org.Org org:
:param str catalog_name:
:param str catalog_desc:
:param logging.Logger logger: optional logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return: XML representation of specified catalog.
:rtype: lxml.objectify.ObjectifiedElement
:raises pyvcloud.vcd.exceptions.EntityNotFoundException: if catalog sharing
fails due to catalog creation failing.
"""
if catalog_exists(org, catalog_name):
msg = f"Found catalog '{catalog_name}'"
msg_update_callback.general(msg)
logger.info(msg)
else:
msg = f"Creating catalog '{catalog_name}'"
msg_update_callback.info(msg)
logger.info(msg)
org.create_catalog(catalog_name, catalog_desc)
msg = f"Created catalog '{catalog_name}'"
msg_update_callback.general(msg)
logger.info(msg)
org.reload()
org.share_catalog(catalog_name)
org.reload()
# DEV NOTE: With api v33.0 and onwards, get_catalog operation will fail for
# non admin users of an org which is not hosting the catalog, even if the
# catalog is explicitly shared with the org in question. Please use this
# method only for org admin and sys admins.
return org.get_catalog(catalog_name)
def __init__(self,
remote_template_cookbook_url,
legacy_mode: bool = False,
logger=NULL_LOGGER,
msg_update_callback=NullPrinter()):
""".
:param str remote_template_cookbook_url:
:param bool legacy_mode:
:param logging.Logger logger: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback:
Callback object.
"""
self.legacy_mode = legacy_mode
self.url = remote_template_cookbook_url
self.logger = logger
self.msg_update_callback = msg_update_callback
self.filtered_cookbook = None
self.unfiltered_cookbook = None
self.cookbook_version: semantic_version.Version = None
self.scripts_directory_path: str = None
def _validate_amqp_config(amqp_dict, msg_update_callback=NullPrinter()):
"""Ensure that 'amqp' section of config is correct.
Checks that 'amqp' section of config has correct keys and value types.
Also ensures that connection to AMQP server is valid.
:param dict amqp_dict: 'amqp' section of config file as a dict.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:raises KeyError: if @amqp_dict has missing or extra properties.
:raises TypeError: if the value type for an @amqp_dict property
is incorrect.
:raises AmqpConnectionError: if AMQP connection failed.
"""
check_keys_and_value_types(amqp_dict,
SAMPLE_AMQP_CONFIG['amqp'],
location="config file 'amqp' section",
msg_update_callback=msg_update_callback)
credentials = pika.PlainCredentials(amqp_dict['username'],
amqp_dict['password'])
parameters = pika.ConnectionParameters(amqp_dict['host'],
amqp_dict['port'],
amqp_dict['vhost'],
credentials,
connection_attempts=3,
retry_delay=2,
socket_timeout=5)
connection = None
try:
connection = pika.BlockingConnection(parameters)
msg_update_callback.general(
"Connected to AMQP server "
f"({amqp_dict['host']}:{amqp_dict['port']})")
except Exception as err:
raise AmqpConnectionError(f"AMQP server error : {str(err)}")
finally:
if connection is not None:
connection.close()
def vgr_callback(prepend_msg='',
logger=NULL_LOGGER, msg_update_callback=NullPrinter()):
"""Create a callback function to use for vsphere-guest-run functions.
:param str prepend_msg: string to prepend to all messages received from
vsphere-guest-run function.
:param logging.Logger logger: logger to use in case of error.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return: callback function to print messages received
from vsphere-guest-run
:rtype: function
"""
def callback(message, exception=None):
msg = f"{prepend_msg}{message}"
msg_update_callback.general_no_color(msg)
logger.info(msg)
if exception is not None:
msg_update_callback.error(
f"vsphere-guest-run error: {exception}")
logger.error("vsphere-guest-run error", exc_info=True)
return callback
def __init__(self, remote_template_cookbook_url, legacy_mode: bool = False,
cookbook_version=None, logger=NULL_LOGGER,
msg_update_callback=NullPrinter()):
""".
:param str remote_template_cookbook_url:
:param bool legacy_mode:
:param semantic_version.Version cookbook_version: Use this parameter
to optionally set the value for cookbook_version. This value is
automatically filled by get_filtered_cookbook() or
get_unfiltered_cookbook()
:param logging.Logger logger: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback:
Callback object.
"""
self.legacy_mode = legacy_mode
self.url = remote_template_cookbook_url
self.logger = logger
self.msg_update_callback = msg_update_callback
self.filtered_cookbook = None
self.unfiltered_cookbook = None
self.cookbook_version: semantic_version.Version = cookbook_version
self.scripts_directory_path: Optional[str] = None
def _validate_vcd_and_vcs_config(
vcd_dict,
vcs,
msg_update_callback=NullPrinter(),
log_file=None,
log_wire=False
):
"""Ensure that 'vcd' and vcs' section of config are correct.
Checks that
* 'vcd' and 'vcs' section of config have correct keys and value types.
* vCD and all registered VCs in vCD are accessible.
* api version specified for vcd is supported by CSE.
:param dict vcd_dict: 'vcd' section of config file as a dict.
:param list vcs: 'vcs' section of config file as a list of dicts.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:param str log_file: log_file for pyvcloud wire log.
:param bool log_wire: If pyvcloud requests should be logged.
:raises KeyError: if @vcd_dict or a vc in @vcs has missing or
extra properties.
:raises TypeError: if the value type for a @vcd_dict or vc property
is incorrect.
:raises ValueError: if vCD has a VC that is not listed in the config file.
"""
check_keys_and_value_types(vcd_dict, SAMPLE_VCD_CONFIG['vcd'],
location="config file 'vcd' section",
msg_update_callback=msg_update_callback)
if not vcd_dict['verify']:
msg_update_callback.general(
'InsecureRequestWarning: Unverified HTTPS request is '
'being made. Adding certificate verification is '
'strongly advised.'
)
requests.packages.urllib3.disable_warnings()
client = None
try:
_validate_vcd_url_scheme(vcd_dict['host'])
client = Client(
vcd_dict['host'],
verify_ssl_certs=vcd_dict['verify'],
log_file=log_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire
)
client.set_credentials(
BasicLoginCredentials(
vcd_dict['username'],
SYSTEM_ORG_NAME,
vcd_dict['password']
)
)
msg_update_callback.general(
"Connected to vCloud Director "
f"({vcd_dict['host']}:{vcd_dict['port']})"
)
for index, vc in enumerate(vcs, 1):
check_keys_and_value_types(
vc,
SAMPLE_VCS_CONFIG['vcs'][0],
location=f"config file 'vcs' section, vc #{index}",
msg_update_callback=msg_update_callback
)
# Check that all registered VCs in vCD are listed in config file
platform = Platform(client)
config_vc_names = [vc['name'] for vc in vcs]
for platform_vc in platform.list_vcenters():
platform_vc_name = platform_vc.get('name')
if platform_vc_name not in config_vc_names:
raise ValueError(
f"vCenter '{platform_vc_name}' registered in "
"vCD but not found in config file"
)
# Check that all VCs listed in config file are registered in vCD
for vc in vcs:
vcenter = platform.get_vcenter(vc['name'])
if not (hasattr(vcenter, 'IsConnected') and vcenter.IsConnected):
msg = f"vCenter Server '{vc['name']}' not available"
msg_update_callback.info(msg)
continue
vsphere_url = urlparse(vcenter.Url.text)
vsphere_url_port = vsphere_url.port
if vsphere_url_port:
v = VSphere(
vsphere_url.hostname,
vc['username'],
vc['password'],
vsphere_url.port
)
else:
v = VSphere(
vsphere_url.hostname,
vc['username'],
vc['password']
)
v.connect()
msg = f"Connected to vCenter Server '{vc['name']}' as " \
f"'{vc['username']}' ({vsphere_url.hostname}"
if vsphere_url_port:
msg += f":{vsphere_url.port}"
msg += ")"
msg_update_callback.general(msg)
finally:
if client is not None:
client.logout()
def upload_ova_to_catalog(client,
catalog_name,
filepath,
update=False,
org=None,
org_name=None,
logger=NULL_LOGGER,
msg_update_callback=NullPrinter()):
"""Upload local ova file to vCD catalog.
:param pyvcloud.vcd.client.Client client:
:param str filepath: file path to the .ova file.
:param str catalog_name: name of catalog.
:param bool update: signals whether to overwrite an existing catalog
item with this new one.
:param pyvcloud.vcd.org.Org org: specific org to use.
:param str org_name: specific org to use if @org is not given.
If None, uses currently logged-in org from @client.
:param logging.Logger logger: optional logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:raises pyvcloud.vcd.exceptions.EntityNotFoundException if catalog
does not exist.
:raises pyvcloud.vcd.exceptions.UploadException if upload fails.
"""
if org is None:
org = get_org(client, org_name=org_name)
catalog_item_name = pathlib.Path(filepath).name
if update:
try:
msg = f"Update flag set. Checking catalog '{catalog_name}' for " \
f"'{catalog_item_name}'"
msg_update_callback.info(msg)
logger.info(msg)
org.delete_catalog_item(catalog_name, catalog_item_name)
org.reload()
wait_for_catalog_item_to_resolve(client,
catalog_name,
catalog_item_name,
org=org)
msg = f"Update flag set. Checking catalog '{catalog_name}' for " \
f"'{catalog_item_name}'"
msg_update_callback.info(msg)
logger.info(msg)
except EntityNotFoundException:
pass
else:
try:
# DEV NOTE: With api v33.0 and onwards, get_catalog_item operation
# will fail for non admin users of an org which is not hosting the
# catalog, even if the catalog is explicitly shared with the org in
# question. Please use this method only for org admin and
# sys admins.
org.get_catalog_item(catalog_name, catalog_item_name)
msg = f"'{catalog_item_name}' already exists in catalog " \
f"'{catalog_name}'"
msg_update_callback.general(msg)
logger.info(msg)
return
except EntityNotFoundException:
pass
msg = f"Uploading '{catalog_item_name}' to catalog '{catalog_name}'"
msg_update_callback.info(msg)
logger.info(msg)
org.upload_ovf(catalog_name, filepath)
org.reload()
wait_for_catalog_item_to_resolve(client,
catalog_name,
catalog_item_name,
org=org)
msg = f"Uploaded '{catalog_item_name}' to catalog '{catalog_name}'"
msg_update_callback.general(msg)
logger.info(msg)
def __init__(self,
client,
sys_admin_client,
build_params,
org=None,
vdc=None,
ssh_key=None,
logger=NULL_LOGGER,
msg_update_callback=NullPrinter(),
log_wire=False):
""".
:param pyvcloud.vcd.Client client:
:param pyvcloud.vcd.Client sys_admin_client:
:param dict build_params:
:param pyvcloud.vcd.org.Org org: specific org to use. Will override the
org_name specified in build_params, can be used to save few vCD
calls to create the Org object.
:param pyvcloud.vcd.vdc.VDC vdc: specific vdc to use. Will override the
vdc_name specified in build_params, can be used to save few vCD
calls to create the Vdc object.
:param str ssh_key: public ssh key to place into the template vApp(s).
:param logging.Logger logger: logger object.
:param utils.ConsoleMessagePrinter msg_update_callback:
Callback object.
"""
self._is_valid = False
self.client = client
self.sys_admin_client = sys_admin_client
self.ssh_key = ssh_key
self.logger = logger
self.msg_update_callback = msg_update_callback
if self.client is None or self.sys_admin_client is None:
return
# validate and populate required fields
self.template_name = build_params.get(
TemplateBuildKey.TEMPLATE_NAME) # noqa: E501
self.template_revision = build_params.get(
TemplateBuildKey.TEMPLATE_REVISION) # noqa: E501
self.ova_name = build_params.get(
TemplateBuildKey.SOURCE_OVA_NAME) # noqa: E501
self.ova_href = build_params.get(
TemplateBuildKey.SOURCE_OVA_HREF) # noqa: E501
self.ova_sha256 = build_params.get(
TemplateBuildKey.SOURCE_OVA_SHA256) # noqa: E501
if org:
self.org = org
self.org_name = org.get_name()
else:
self.org_name = build_params.get(
TemplateBuildKey.ORG_NAME) # noqa: E501
self.org = get_org(self.client, org_name=self.org_name)
if vdc:
self.vdc = vdc
self.vdc.get_resource() # to make sure vdc.resource is populated
self.vdc_name = vdc.name
else:
self.vdc_name = build_params.get(
TemplateBuildKey.VDC_NAME) # noqa: E501
self.vdc = get_vdc(self.client,
vdc_name=self.vdc_name,
org=self.org)
self.catalog_name = build_params.get(
TemplateBuildKey.CATALOG_NAME) # noqa: E501
self.catalog_item_name = build_params.get(
TemplateBuildKey.CATALOG_ITEM_NAME) # noqa: E501
self.catalog_item_description = \
build_params.get(TemplateBuildKey.CATALOG_ITEM_DESCRIPTION) # noqa: E501
self.temp_vapp_name = build_params.get(
TemplateBuildKey.TEMP_VAPP_NAME) # noqa: E501
self.temp_vm_name = build_params.get(
TemplateBuildKey.TEMP_VM_NAME) # noqa: E501
self.cpu = build_params.get(TemplateBuildKey.CPU)
self.memory = build_params.get(TemplateBuildKey.MEMORY)
self.network_name = build_params.get(
TemplateBuildKey.NETWORK_NAME) # noqa: E501
self.ip_allocation_mode = build_params.get(
TemplateBuildKey.IP_ALLOCATION_MODE) # noqa: E501
self.storage_profile = build_params.get(
TemplateBuildKey.STORAGE_PROFILE) # noqa: E501
self.cse_placement_policy = build_params.get(
TemplateBuildKey.CSE_PLACEMENT_POLICY) # noqa: E501
self.remote_cookbook_version = build_params.get(
TemplateBuildKey.REMOTE_COOKBOOK_VERSION) # noqa: E501
self.log_wire = log_wire
if self.template_name and self.template_revision and \
self.ova_name and self.ova_href and self.ova_sha256 and \
self.org and self.org_name and self.vdc and self.vdc_name and \
self.catalog_name and self.catalog_item_name and \
self.catalog_item_description and self.temp_vapp_name and \
self.temp_vm_name and self.cpu and self.memory and \
self.network_name and self.ip_allocation_mode and \
self.storage_profile:
self._is_valid = True
def _validate_pks_config_data_integrity(
pks_config,
msg_update_callback=NullPrinter(),
logger_debug=NULL_LOGGER,
logger_wire=NULL_LOGGER
):
all_pks_servers = \
[entry['name'] for entry in pks_config[PKS_SERVERS_SECTION_KEY]]
all_pks_accounts = \
[entry['name'] for entry in pks_config[PKS_ACCOUNTS_SECTION_KEY]]
# Create a cache with pks_account to Credentials mapping
pks_account_info_table = {}
for pks_account in pks_config[PKS_ACCOUNTS_SECTION_KEY]:
pks_account_name = pks_account['pks_api_server']
credentials = Credentials(
pks_account['username'],
pks_account['secret']
)
pks_account_info_table[pks_account_name] = credentials
# Check for duplicate pks api server names
duplicate_pks_server_names = get_duplicate_items_in_list(all_pks_servers)
if len(duplicate_pks_server_names) != 0:
raise ValueError(
f"Duplicate PKS api server(s) : {duplicate_pks_server_names} found"
f" in Section : {PKS_SERVERS_SECTION_KEY}"
)
# Check for duplicate pks account names
duplicate_pks_account_names = get_duplicate_items_in_list(all_pks_accounts)
if len(duplicate_pks_account_names) != 0:
raise ValueError(
f"Duplicate PKS account(s) : {duplicate_pks_account_names} found"
f" in Section : {PKS_ACCOUNTS_SECTION_KEY}"
)
# Check validity of all PKS api servers referenced in PKS accounts section
for pks_account in pks_config[PKS_ACCOUNTS_SECTION_KEY]:
pks_server_name = pks_account.get('pks_api_server')
if pks_server_name not in all_pks_servers:
raise ValueError(
f"Unknown PKS api server : {pks_server_name} referenced by "
f"PKS account : {pks_account.get('name')} in Section : "
f"{PKS_ACCOUNTS_SECTION_KEY}"
)
# Check validity of all PKS accounts referenced in Orgs section
if PKS_ORGS_SECTION_KEY in pks_config.keys():
for org in pks_config[PKS_ORGS_SECTION_KEY]:
referenced_accounts = org.get('pks_accounts')
if not referenced_accounts:
continue
for account in referenced_accounts:
if account not in all_pks_accounts:
raise ValueError(
f"Unknown PKS account : {account} "
f"referenced by Org : {org.get('name')} "
f"in Section : {PKS_ORGS_SECTION_KEY}"
)
# Check validity of all PKS api servers referenced in PVDC section
for pvdc in pks_config[PKS_PVDCS_SECTION_KEY]:
pks_server_name = pvdc.get('pks_api_server')
if pks_server_name not in all_pks_servers:
raise ValueError(
f"Unknown PKS api server : {pks_server_name} "
f"referenced by PVDC : {pvdc.get('name')} in "
f"Section : {PKS_PVDCS_SECTION_KEY}"
)
# Check validity of all PKS api servers referenced in the pks_api_servers
# section
for pks_server in pks_config[PKS_SERVERS_SECTION_KEY]:
pks_account = pks_account_info_table.get(pks_server.get('name'))
pks_configuration = Configuration()
pks_configuration.proxy = f"http://{pks_server['proxy']}:80" \
if pks_server.get('proxy') else None
pks_configuration.host = \
f"https://{pks_server['host']}:{pks_server['port']}/" \
f"{VERSION_V1}"
pks_configuration.access_token = None
pks_configuration.username = pks_account.username
pks_configuration.verify_ssl = pks_server['verify']
pks_configuration.secret = pks_account.secret
pks_configuration.uaac_uri = \
f"https://{pks_server['host']}:{pks_server['uaac_port']}"
uaa_client = UaaClient(
pks_configuration.uaac_uri,
pks_configuration.username,
pks_configuration.secret,
proxy_uri=pks_configuration.proxy
)
token = uaa_client.getToken()
if not token:
raise ValueError(
"Unable to connect to PKS server : "
f"{pks_server.get('name')} ({pks_server.get('host')})"
)
pks_configuration.token = token
client = ApiClient(configuration=pks_configuration)
if client:
msg_update_callback.general(
"Connected to PKS server ("
f"{pks_server.get('name')} : {pks_server.get('host')})"
)
# Check validity of all PKS api servers referenced in NSX-T section
for nsxt_server in pks_config[PKS_NSXT_SERVERS_SECTION_KEY]:
pks_server_name = nsxt_server.get('pks_api_server')
if pks_server_name not in all_pks_servers:
raise ValueError(
f"Unknown PKS api server : {pks_server_name} referenced by "
f"NSX-T server : {nsxt_server.get('name')} in Section : "
f"{PKS_NSXT_SERVERS_SECTION_KEY}"
)
# Create a NSX-T client and verify connection
# server
nsxt_client = NSXTClient(
host=nsxt_server.get('host'),
username=nsxt_server.get('username'),
password=nsxt_server.get('password'),
logger_debug=logger_debug,
logger_wire=logger_wire,
http_proxy=nsxt_server.get('proxy'),
https_proxy=nsxt_server.get('proxy'),
verify_ssl=nsxt_server.get('verify')
)
if not nsxt_client.test_connectivity():
raise ValueError(
"Unable to connect to NSX-T server : "
f"{nsxt_server.get('name')} ({nsxt_server.get('host')})"
)
msg_update_callback.general(
f"Connected to NSX-T server ({nsxt_server.get('host')})"
)
ipset_manager = IPSetManager(nsxt_client)
if nsxt_server.get('nodes_ip_block_ids'):
block_not_found = False
ip_block_id = ''
try:
for ip_block_id in nsxt_server.get('nodes_ip_block_ids'):
if not ipset_manager.get_ip_block_by_id(ip_block_id):
block_not_found = True
except HTTPError:
block_not_found = True
if block_not_found:
raise ValueError(
f"Unknown Node IP Block : {ip_block_id} referenced by "
f"NSX-T server : {nsxt_server.get('name')}.")
if nsxt_server.get('pods_ip_block_ids'):
block_not_found = False
ip_block_id = ''
try:
for ip_block_id in nsxt_server.get('pods_ip_block_ids'):
if not ipset_manager.get_ip_block_by_id(ip_block_id):
block_not_found = True
except HTTPError:
block_not_found = True
if block_not_found:
raise ValueError(
f"Unknown Pod IP Block : {ip_block_id} referenced by "
f"NSX-T server : {nsxt_server.get('name')}.")
dfw_manager = DFWManager(nsxt_client)
fw_section_id = \
nsxt_server.get('distributed_firewall_section_anchor_id')
section = dfw_manager.get_firewall_section(id=fw_section_id)
if not section:
raise ValueError(
f"Unknown Firewall section : {fw_section_id} referenced by "
f"NSX-T server : {nsxt_server.get('name')}.")
def get_validated_config(
config_file_name,
pks_config_file_name=None,
skip_config_decryption=False,
decryption_password=None,
log_wire_file=None,
logger_debug=NULL_LOGGER,
msg_update_callback=NullPrinter()
):
"""Get the config file as a dictionary and check for validity.
Ensures that all properties exist and all values are the expected type.
Checks that AMQP connection is available, and vCD/VCs are valid.
Does not guarantee that CSE has been installed according to this
config file. Additionally populates certain key-value pairs in the
config dict to avoid repeated computation of those e.g.
supported api versions, feature flags, RDE version in use etc.
:param str config_file_name: path to config file.
:param str pks_config_file_name: path to PKS config file.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param str log_wire_file: log_wire_file to use if needed to wire log
pyvcloud requests and responses
:param logging.Logger logger_debug: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return: CSE config
:rtype: dict
:raises KeyError: if config file has missing or extra properties.
:raises TypeError: if the value type for a config file property
is incorrect.
:raises container_service_extension.exceptions.AmqpConnectionError:
(when not using MQTT) if AMQP connection failed (host, password, port,
username, vhost is invalid).
:raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid.
:raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or
'password' is invalid.
:raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password'
is invalid.
"""
check_file_permissions(config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(config_file_name) as config_file:
config = yaml.safe_load(config_file) or {}
else:
msg_update_callback.info(
f"Decrypting '{config_file_name}'")
try:
config = yaml.safe_load(
get_decrypted_file_contents(
config_file_name,
decryption_password
)
) or {}
except cryptography.fernet.InvalidToken:
raise Exception(CONFIG_DECRYPTION_ERROR_MSG)
msg_update_callback.info(
f"Validating config file '{config_file_name}'"
)
is_no_vc_communication_mode = \
server_utils.is_no_vc_communication_mode(ServerConfig(config))
use_mqtt = server_utils.should_use_mqtt_protocol(ServerConfig(config))
sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \
else SAMPLE_MQTT_CONFIG
# This allows us to compare top-level config keys and value types
sample_config = {
**sample_message_queue_config,
**SAMPLE_VCD_CONFIG,
**SAMPLE_SERVICE_CONFIG,
**SAMPLE_BROKER_CONFIG
}
if not is_no_vc_communication_mode:
sample_config.update(SAMPLE_VCS_CONFIG)
else:
if 'vcs' in config:
del config['vcs']
log_wire = str_to_bool(config.get('service', {}).get('log_wire'))
nsxt_wire_logger = NULL_LOGGER
if not log_wire:
log_wire_file = None
nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER
check_keys_and_value_types(
config,
sample_config,
location='config file',
msg_update_callback=msg_update_callback
)
# MQTT validation not required because no MQTT host, exchange, etc.
# is needed in the config file since the server code creates and
# registers the MQTT extension directly using server constants
if not use_mqtt:
_validate_amqp_config(config['amqp'], msg_update_callback)
# Validation of service properties is done first as those properties are
# used in broker validation.
check_keys_and_value_types(
config['service'],
SAMPLE_SERVICE_CONFIG['service'],
location="config file 'service' section",
excluded_keys=['log_wire'],
msg_update_callback=msg_update_callback
)
try:
if is_no_vc_communication_mode:
_validate_vcd_config(
config['vcd'],
msg_update_callback,
log_file=log_wire_file,
log_wire=log_wire
)
else:
_validate_vcd_and_vcs_config(
config['vcd'],
config['vcs'],
msg_update_callback,
log_file=log_wire_file,
log_wire=log_wire
)
except vim.fault.InvalidLogin:
raise Exception(VCENTER_LOGIN_ERROR_MSG)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
except requests.exceptions.ConnectionError as err:
raise Exception(f"Cannot connect to {err.request.url}.")
check_keys_and_value_types(
config['service']['telemetry'],
SAMPLE_SERVICE_CONFIG['service']['telemetry'],
location="config file 'service->telemetry' section",
msg_update_callback=msg_update_callback
)
_validate_broker_config(
config['broker'],
legacy_mode=config['service']['legacy_mode'],
msg_update_callback=msg_update_callback,
logger_debug=logger_debug
)
config = add_additional_details_to_config(
config=config,
vcd_host=config['vcd']['host'],
vcd_username=config['vcd']['username'],
vcd_password=config['vcd']['password'],
verify_ssl=config['vcd']['verify'],
is_legacy_mode=config['service']['legacy_mode'],
is_mqtt_exchange=server_utils.should_use_mqtt_protocol(
ServerConfig(config)
),
log_wire=log_wire,
log_wire_file=log_wire_file
)
_raise_error_if_amqp_not_supported(
use_mqtt,
config['service']['default_api_version'],
logger=logger_debug
)
msg_update_callback.general(
f"Config file '{config_file_name}' is valid"
)
if pks_config_file_name:
check_file_permissions(pks_config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(pks_config_file_name) as f:
pks_config = yaml.safe_load(f) or {}
else:
msg_update_callback.info(
f"Decrypting '{pks_config_file_name}'")
pks_config = yaml.safe_load(
get_decrypted_file_contents(pks_config_file_name,
decryption_password)) or {}
msg_update_callback.info(
f"Validating PKS config file '{pks_config_file_name}'")
_validate_pks_config_structure(pks_config, msg_update_callback)
try:
_validate_pks_config_data_integrity(pks_config,
msg_update_callback,
logger_debug=logger_debug,
logger_wire=nsxt_wire_logger)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
msg_update_callback.general(
f"PKS Config file '{pks_config_file_name}' is valid")
config['pks_config'] = pks_config
else:
config['pks_config'] = None
return config
def get_validated_config(config_file_name,
pks_config_file_name=None,
skip_config_decryption=False,
decryption_password=None,
log_wire_file=None,
logger_debug=NULL_LOGGER,
msg_update_callback=NullPrinter()):
"""Get the config file as a dictionary and check for validity.
Ensures that all properties exist and all values are the expected type.
Checks that AMQP connection is available, and vCD/VCs are valid.
Does not guarantee that CSE has been installed according to this
config file.
:param str config_file_name: path to config file.
:param str pks_config_file_name: path to PKS config file.
:param bool skip_config_decryption: do not decrypt the config file.
:param str decryption_password: password to decrypt the config file.
:param str log_wire_file: log_wire_file to use if needed to wire log
pyvcloud requests and responses
:param logging.Logger logger_debug: logger to log with.
:param utils.ConsoleMessagePrinter msg_update_callback: Callback object.
:return: CSE config
:rtype: dict
:raises KeyError: if config file has missing or extra properties.
:raises TypeError: if the value type for a config file property
is incorrect.
:raises container_service_extension.exceptions.AmqpConnectionError:
(when not using MQTT) if AMQP connection failed (host, password, port,
username, vhost is invalid).
:raises requests.exceptions.ConnectionError: if 'vcd' 'host' is invalid.
:raises pyvcloud.vcd.exceptions.VcdException: if 'vcd' 'username' or
'password' is invalid.
:raises pyVmomi.vim.fault.InvalidLogin: if 'vcs' 'username' or 'password'
is invalid.
"""
check_file_permissions(config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(config_file_name) as config_file:
config = yaml.safe_load(config_file) or {}
else:
msg_update_callback.info(
f"Decrypting '{config_file_name}'")
try:
config = yaml.safe_load(
get_decrypted_file_contents(config_file_name,
decryption_password)) or {}
except cryptography.fernet.InvalidToken:
raise Exception(CONFIG_DECRYPTION_ERROR_MSG)
msg_update_callback.info(
f"Validating config file '{config_file_name}'")
# This allows us to compare top-level config keys and value types
use_mqtt = should_use_mqtt_protocol(config)
sample_message_queue_config = SAMPLE_AMQP_CONFIG if not use_mqtt \
else SAMPLE_MQTT_CONFIG
sample_config = {
**sample_message_queue_config, **SAMPLE_VCD_CONFIG,
**SAMPLE_VCS_CONFIG, **SAMPLE_SERVICE_CONFIG,
**SAMPLE_BROKER_CONFIG
}
log_wire = str_to_bool(config.get('service', {}).get('log_wire'))
nsxt_wire_logger = NULL_LOGGER
if not log_wire:
log_wire_file = None
nsxt_wire_logger = SERVER_NSXT_WIRE_LOGGER
check_keys_and_value_types(config, sample_config, location='config file',
msg_update_callback=msg_update_callback)
# MQTT validation not required because no MQTT host, exchange, etc.
# is needed in the config file since the server code creates and
# registers the MQTT extension directly using server constants
if not use_mqtt:
_validate_amqp_config(config['amqp'], msg_update_callback)
try:
_validate_vcd_and_vcs_config(config['vcd'], config['vcs'],
msg_update_callback,
log_file=log_wire_file,
log_wire=log_wire)
except vim.fault.InvalidLogin:
raise Exception(VCENTER_LOGIN_ERROR_MSG)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
except requests.exceptions.ConnectionError as err:
raise Exception(f"Cannot connect to {err.request.url}.")
_validate_broker_config(config['broker'],
legacy_mode=config['service']['legacy_mode'],
msg_update_callback=msg_update_callback,
logger_debug=logger_debug)
check_keys_and_value_types(config['service'],
SAMPLE_SERVICE_CONFIG['service'],
location="config file 'service' section",
excluded_keys=['log_wire'],
msg_update_callback=msg_update_callback)
check_keys_and_value_types(config['service']['telemetry'],
SAMPLE_SERVICE_CONFIG['service']['telemetry'],
location="config file 'service->telemetry' "
"section",
msg_update_callback=msg_update_callback)
msg_update_callback.general(
f"Config file '{config_file_name}' is valid")
if pks_config_file_name:
check_file_permissions(pks_config_file_name,
msg_update_callback=msg_update_callback)
if skip_config_decryption:
with open(pks_config_file_name) as f:
pks_config = yaml.safe_load(f) or {}
else:
msg_update_callback.info(
f"Decrypting '{pks_config_file_name}'")
pks_config = yaml.safe_load(
get_decrypted_file_contents(pks_config_file_name,
decryption_password)) or {}
msg_update_callback.info(
f"Validating PKS config file '{pks_config_file_name}'")
_validate_pks_config_structure(pks_config, msg_update_callback)
try:
_validate_pks_config_data_integrity(pks_config,
msg_update_callback,
logger_debug=logger_debug,
logger_wire=nsxt_wire_logger)
except requests.exceptions.SSLError as err:
raise Exception(f"SSL verification failed: {str(err)}")
msg_update_callback.general(
f"PKS Config file '{pks_config_file_name}' is valid")
config['pks_config'] = pks_config
else:
config['pks_config'] = None
# Compute common supported api versions by the CSE server and vCD
sysadmin_client = None
try:
sysadmin_client = Client(
config['vcd']['host'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_wire_file,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
sysadmin_client.set_credentials(BasicLoginCredentials(
config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password']))
vcd_supported_api_versions = \
set(sysadmin_client.get_supported_versions_list())
cse_supported_api_versions = set(SUPPORTED_VCD_API_VERSIONS)
common_supported_api_versions = \
list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501
common_supported_api_versions.sort()
config['service']['supported_api_versions'] = \
common_supported_api_versions
finally:
if sysadmin_client:
sysadmin_client.logout()
# Convert legacy_mode flag in service_section to corresponding
# feature flags
is_legacy_mode = config['service']['legacy_mode']
if 'feature_flags' not in config:
config['feature_flags'] = {}
config['feature_flags']['legacy_api'] = str_to_bool(is_legacy_mode)
config['feature_flags']['non_legacy_api'] = \
not str_to_bool(is_legacy_mode)
# Temporary work around before api version is completely removed from
# config
if is_legacy_mode:
supported_api_versions_float = \
[float(x) for x in config['service']['supported_api_versions']
if float(x) < 35.0]
else:
supported_api_versions_float = \
[float(x) for x in config['service']['supported_api_versions']
if float(x) >= 35.0]
config['vcd']['api_version'] = str(max(supported_api_versions_float))
# Store telemetry instance id, url and collector id in config
# This steps needs to be done after api_version has been computed
# and stored in the config
store_telemetry_settings(config)
return config