def upload_tkgm_template(client, ova_file_path, catalog_name, catalog_item_name, org_name, force, logger=NULL_LOGGER, msg_update_callback=utils.NullPrinter()): """.""" org = vcd_utils.get_org(client, org_name=org_name) if vcd_utils.catalog_item_exists(org=org, catalog_name=catalog_name, catalog_item_name=catalog_item_name): if not force: msg = f"Catalog item '{catalog_item_name}' already exists " \ f"in catalog '{catalog_name}'" msg_update_callback.info(msg) logger.info(msg) return False else: msg = f"Deleting catalog item '{catalog_item_name}' " \ f"from catalog '{catalog_name}'" msg_update_callback.general(msg) logger.info(msg) org.delete_catalog_item(catalog_name, catalog_item_name) org.reload() vcd_utils.upload_ova_to_catalog(client=client, source_filepath=ova_file_path, catalog_name=catalog_name, catalog_item_name=catalog_item_name, org=org, logger=logger, msg_update_callback=msg_update_callback) return True
def get_template_property(client, org_name, catalog_name, catalog_item_name): org = vcd_utils.get_org(client, org_name=org_name) item_resource = org.get_catalog_item(name=catalog_name, item_name=catalog_item_name) entity_resource = client.get_resource(item_resource.Entity.get('href')) property_map = {} if hasattr(entity_resource, "Children") and hasattr( entity_resource.Children, "Vm"): # noqa: E501 vm = entity_resource.Children.Vm if hasattr(vm, "{http://schemas.dmtf.org/ovf/envelope/1}ProductSection" ): # noqa: E501 product_section = getattr( vm, "{http://schemas.dmtf.org/ovf/envelope/1}ProductSection" ) # noqa: E501 for item in product_section.getchildren(): if item.tag == "{http://schemas.dmtf.org/ovf/envelope/1}Property": # noqa: E501 key = item.get( '{http://schemas.dmtf.org/ovf/envelope/1}key' ) # noqa: E501 value = item.get( '{http://schemas.dmtf.org/ovf/envelope/1}value' ) # noqa: E501 property_map[key] = value return property_map
def reload_templates(request_data, op_ctx): """.""" user_context = op_ctx.get_user_context(api_version=None) user_client = user_context.client if not user_client.is_sysadmin: raise e.UnauthorizedRequestError( error_message= 'Unauthorized to reload CSE native and TKG templates.' # noqa: E501 ) org = vcd_utils.get_org(user_client, user_context.org_name) user_href = org.get_user(user_context.name).get('href') task = Task(user_client) task_resource = task.update( status=TaskStatus.RUNNING.value, namespace='vcloud.cse', operation="Reloading native templates.", operation_name='template operation', details='', progress=None, owner_href=user_context.org_href, owner_name=user_context.org_name, owner_type='application/vnd.vmware.vcloud.org+xml', user_href=user_href, user_name=user_context.name, org_href=user_context.org_href) task_href = task_resource.get('href') op_ctx.is_async = True _reload_templates_async(op_ctx, task_href) return {"task_href": task_href}
def _follow_task(op_ctx: ctx.OperationContext, task_href: str, ovdc_id: str): try: task = vcd_task.Task(client=op_ctx.sysadmin_client) session = op_ctx.sysadmin_client.get_vcloud_session() vdc = vcd_utils.get_vdc(op_ctx.sysadmin_client, vdc_id=ovdc_id) org = vcd_utils.get_org(op_ctx.sysadmin_client) user_name = session.get('user') user_href = org.get_user(user_name).get('href') msg = "Remove ovdc compute policy" # TODO(pyvcloud): Add method to retireve task from task href t = task.update(status=vcd_task.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=msg, operation_name=msg, details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, org_href=op_ctx.user.org_href, task_href=task_href) op_ctx.sysadmin_client.get_task_monitor().wait_for_status(t) except Exception as err: logger.SERVER_LOGGER.error(f"{err}") finally: if op_ctx.sysadmin_client: op_ctx.end()
def update_ovdc(operation_context: ctx.OperationContext, ovdc_id: str, ovdc_spec: common_models.Ovdc) -> dict: # noqa: 501 """Update ovdc with the updated k8s runtimes list. :param ctx.OperationContext operation_context: context for the request :param common_models.Ovdc ovdc_spec: Ovdc object having the updated k8s runtime list :return: dictionary containing the task href for the update operation :rtype: dict """ # NOTE: For CSE 3.0, if `enable_tkg_plus` flag in config is set to false, # Prevent enable/disable of OVDC for TKG+ k8s runtime by throwing an # exception msg = "Updating OVDC placement policies" task = vcd_task.Task(operation_context.sysadmin_client) org = vcd_utils.get_org(operation_context.client) user_href = org.get_user(operation_context.user.name).get('href') vdc = vcd_utils.get_vdc(operation_context.sysadmin_client, vdc_id=ovdc_id, # noqa: E501 is_admin_operation=True) logger.SERVER_LOGGER.debug(msg) task_resource = task.update( status=vcd_client.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=msg, operation_name='OVDC Update', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=operation_context.user.name, org_href=operation_context.user.org_href, task_href=None, error_message=None, stack_trace=None) task_href = task_resource.get('href') operation_context.is_async = True # NOTE: Telemetry is currently handled in the async function as it is not # possible to know the operation (enable/disable) without comparing it to # current k8s runtimes. if ClusterEntityKind.TKG_PLUS.value in ovdc_spec.k8s_runtime and \ not server_utils.is_tkg_plus_enabled(): msg = "TKG+ is not enabled on CSE server. Please enable TKG+ in the " \ "server and try again." logger.SERVER_LOGGER.debug(msg) raise Exception(msg) policy_list = [RUNTIME_DISPLAY_NAME_TO_INTERNAL_NAME_MAP[p] for p in ovdc_spec.k8s_runtime] # noqa: E501 _update_ovdc_using_placement_policy_async(operation_context=operation_context, # noqa:E501 task=task, task_href=task_href, user_href=user_href, policy_list=policy_list, # noqa:E501 ovdc_id=ovdc_id, vdc=vdc, org_name=ovdc_spec.org_name, remove_cp_from_vms_on_disable=ovdc_spec.remove_cp_from_vms_on_disable) # noqa:E501 return {'task_href': task_href}
def cluster_create(data: dict, op_ctx: ctx.OperationContext): """Request handler for cluster create operation. :return: Defined entity of the native cluster :rtype: container_service_extension.def_.models.DefEntity """ input_entity: dict = data[RequestKey.INPUT_SPEC] payload_version = input_entity.get( rde_constants.PayloadKey.PAYLOAD_VERSION) # noqa: E501 rde_utils.raise_error_if_unsupported_payload_version(payload_version) # Validate the Input payload based on the (Operation, payload_version). # Get the validator based on the payload_version # ToDo : Don't use default cloudapi_client. Use the specific versioned one rde_validator_factory.get_validator( rde_version=rde_constants. MAP_INPUT_PAYLOAD_VERSION_TO_RDE_VERSION[payload_version]).validate( cloudapi_client=op_ctx.cloudapi_client, sysadmin_client=op_ctx.sysadmin_client, entity=input_entity) def_entity_service = entity_service.DefEntityService( op_ctx.cloudapi_client) # noqa: E501 entity_type = server_utils.get_registered_def_entity_type() converted_entity: AbstractNativeEntity = rde_utils.convert_input_rde_to_runtime_rde_format( input_entity) # noqa: E501 def_entity = common_models.DefEntity( entity=converted_entity, entityType=entity_type.id) # noqa: E501 # No need to set org context for non sysadmin users org_context = None if op_ctx.client.is_sysadmin(): org_resource = pyvcloud_utils.get_org( op_ctx.client, org_name=converted_entity.metadata.org_name) # noqa: E501 org_context = org_resource.href.split('/')[-1] _, task_href = def_entity_service.create_entity( entity_type_id=entity_type.id, entity=def_entity, tenant_org_context=org_context, is_request_async=True) task_resource = op_ctx.sysadmin_client.get_resource(task_href) entity_id = task_resource.Owner.get('id') def_entity = def_entity_service.get_entity(entity_id) def_entity.entity.status.task_href = task_href telemetry_handler.record_user_action_details( cse_operation=telemetry_constants.CseOperation.V36_CLUSTER_APPLY, cse_params={ server_constants.CLUSTER_ENTITY: def_entity, telemetry_constants.PayloadKey.SOURCE_DESCRIPTION: thread_local_data.get_thread_local_data( ThreadLocalData.USER_AGENT) # noqa: E501 }) return def_entity.to_dict()
def set_tenant_org_context(self, org_name=None): """Set tenant org context if not set in the client. :param str org_name: Name of the org. If not set, makes use of the logged in org name """ if cli_constants.TKGRequestHeaderKey.X_VMWARE_VCLOUD_TENANT_CONTEXT not in self._tkg_client.default_headers: # noqa: E501 logger.CLIENT_LOGGER.debug(f"Setting client org context with org name {org_name}") # noqa: E501 org_resource = vcd_utils.get_org(self._client, org_name=org_name) org_id = org_resource.href.split('/')[-1] self._tkg_client.set_default_header(cli_constants.TKGRequestHeaderKey.X_VMWARE_VCLOUD_TENANT_CONTEXT, # noqa: E501 org_id)
def save_metadata(client, org_name, catalog_name, catalog_item_name, data, keys_to_write=None): keys_to_write = keys_to_write or [] org = vcd_utils.get_org(client, org_name=org_name) org.set_multiple_metadata_on_catalog_item( catalog_name=catalog_name, item_name=catalog_item_name, key_value_dict={k: data.get(k, '') for k in keys_to_write}, # noqa: E501 domain=MetadataDomain.SYSTEM, visibility=MetadataVisibility.PRIVATE)
def remove_vdc_compute_policy_from_vdc( self, # noqa: E501 ovdc_id, compute_policy_href, force=False): # noqa: E501 """Delete the compute policy from the specified vdc. :param str ovdc_id: id of the vdc to assign the policy :param compute_policy_href: policy href to remove :param bool force: If True, will set affected VMs' compute policy to 'System Default' :return: dictionary containing 'task_href'. """ vdc = vcd_utils.get_vdc(self._sysadmin_client, vdc_id=ovdc_id) # TODO the following org will be associated with 'System' org. # task created should be associated with the corresponding org of the # vdc object. org = vcd_utils.get_org(self._sysadmin_client) org.reload() user_name = self._session.get('user') user_href = org.get_user(user_name).get('href') task = Task(self._sysadmin_client) task_resource = task.update( status=vcd_client.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=f"Removing compute policy (href: {compute_policy_href})" f" from org VDC (vdc id: {ovdc_id})", operation_name='Remove org VDC compute policy', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, org_href=org.href) task_href = task_resource.get('href') self._remove_compute_policy_from_vdc_async( ovdc_id=ovdc_id, compute_policy_href=compute_policy_href, task_resource=task_resource, force=force) return {'task_href': task_href}
def save_metadata(client, org_name, catalog_name, catalog_item_name, template_data, metadata_key_list=None): metadata_key_list = metadata_key_list or [] org = vcd_utils.get_org(client, org_name=org_name) org.set_multiple_metadata_on_catalog_item( catalog_name=catalog_name, item_name=catalog_item_name, key_value_dict={ k: template_data.get(k, '') for k in metadata_key_list }, # noqa: E501 domain=MetadataDomain.SYSTEM, visibility=MetadataVisibility.PRIVATE)
def remove_all_vdc_compute_policies_from_vapp_template_vms( self, org_name, catalog_name, catalog_item_name): # noqa: E501 """Remove all compute policies from vms of given vapp template. :param str org_name: name of the organization that has the catalog. :param str catalog_name: name of the catalog. :param str catalog_item_name: name of the catalog item that has vms. :return: an object of type EntityType.TASK XML which represents the asynchronous task that is updating virtual application template. :rtype: lxml.objectify.ObjectifiedElement """ self._raise_error_if_not_supported() org = vcd_utils.get_org(self._sysadmin_client, org_name=org_name) return org.remove_all_compute_policies_from_vapp_template_vms( catalog_name, catalog_item_name)
def read_all_tkgm_template(client, org_name, catalog_name, logger=NULL_LOGGER, msg_update_callback=utils.NullPrinter()): """.""" org = vcd_utils.get_org(client, org_name=org_name) catalog_item_names = [ entry['name'] for entry in org.list_catalog_items(catalog_name) ] templates = [] for item_name in catalog_item_names: md = org.get_all_metadata_from_catalog_item(catalog_name=catalog_name, item_name=item_name) metadata_dict = metadata_to_dict(md) filtered_dict = {} kind = metadata_dict.get(server_constants.TKGmTemplateKey.KIND) if kind == shared_constants.ClusterEntityKind.TKG_M.value: msg = f"Found catalog item `{item_name}`." logger.debug(msg) msg_update_callback.general(msg) for entry in server_constants.TKGmTemplateKey: key = entry.value value = metadata_dict.get(key, '') # If catalog item has been renamed directly in VCD, # update the `name` metadata field. if entry == server_constants.TKGmTemplateKey.NAME and value != item_name: # noqa: E501 msg = f"Template `{value}` has been " \ "renamed outside of CSE. Attempting to fix." logger.debug(msg) msg_update_callback.general(msg) org.set_metadata_on_catalog_item(catalog_name=catalog_name, item_name=item_name, key=key, value=item_name) value = item_name filtered_dict[key] = value msg = f"Template `{item_name}` successfully loaded." logger.debug(msg) msg_update_callback.general(msg) templates.append(filtered_dict) return templates
def assign_vdc_placement_policy_to_vapp_template_vms( self, compute_policy_href, org_name, catalog_name, catalog_item_name): """Assign the compute policy to vms of given vapp template. :param str compute_policy_href: compute policy to be removed :param str org_name: name of the organization that has the catalog :param str catalog_name: name of the catalog :param str catalog_item_name: name of the catalog item that has vms :return: an object of type vcd_client.TASK XML which represents the asynchronous task that is updating virtual application template. :rtype: lxml.objectify.ObjectifiedElement """ self._raise_error_if_not_supported() org = vcd_utils.get_org(self._sysadmin_client, org_name=org_name) return org.assign_placement_policy_to_vapp_template_vms( catalog_name=catalog_name, catalog_item_name=catalog_item_name, placement_policy_href=compute_policy_href, placement_policy_final=True)
def get_all_k8s_local_template_definition(client, catalog_name, org=None, org_name=None, logger_debug=logger.NULL_LOGGER): """Fetch all CSE k8s templates in a catalog. A CSE k8s template is a catalog item that has all the necessary metadata stamped onto it. If only partial metadata is present on a catalog item, that catalog item will be disqualified from the result. :param pyvcloud.vcd.Client client: A sys admin client to be used to retrieve metadata off the catalog items. :param str catalog_name: Name of the catalog where the template resides. :param pyvcloud.vcd.Org org: Org object which hosts the catalog. :param str org_name: Name of the org that is hosting the catalog. Can be provided in lieu of param org, however param org takes precedence. :return: list of dictionaries containing template data :rtype: list of dicts """ if not org: org = get_org(client, org_name=org_name) catalog_item_names = [ entry['name'] for entry in org.list_catalog_items(catalog_name) ] templates = [] for item_name in catalog_item_names: md = org.get_all_metadata_from_catalog_item(catalog_name=catalog_name, item_name=item_name) metadata_dict = metadata_to_dict(md) # if catalog item doesn't have all the required metadata keys, # CSE should not recognize it as a template expected_metadata_keys = \ set([entry.value for entry in LocalTemplateKey]) missing_metadata_keys = expected_metadata_keys - metadata_dict.keys() num_missing_metadata_keys = len(missing_metadata_keys) if num_missing_metadata_keys == len(expected_metadata_keys): # This catalog item has no CSE related metadata, so skip it. continue if num_missing_metadata_keys > 0: # This catalog item has partial CSE metadata, so skip it but also # log relevant information. msg = f"Catalog item '{item_name}' missing " \ f"{num_missing_metadata_keys} metadata: " \ f"{missing_metadata_keys}" # noqa: F841 logger_debug.debug(msg) continue # non-string metadata is written to the dictionary as a string # when 'upgrade_from' metadata is empty, vcd returns it as: "['']" # when 'upgrade_from' metadata is not empty, vcd returns it as an array # coerce "['']" to the more usable empty array [] if isinstance(metadata_dict[LocalTemplateKey.UPGRADE_FROM], str): metadata_dict[LocalTemplateKey.UPGRADE_FROM] = ast.literal_eval( metadata_dict[LocalTemplateKey.UPGRADE_FROM]) # noqa: E501 if metadata_dict[LocalTemplateKey.UPGRADE_FROM] == ['']: metadata_dict[LocalTemplateKey.UPGRADE_FROM] = [] templates.append(metadata_dict) return templates
def _reload_templates_async(op_ctx, task_href): user_context = None task = None user_href = None try: user_context = op_ctx.get_user_context(api_version=None) user_client = user_context.client org = vcd_utils.get_org(user_client, user_context.org_name) user_href = org.get_user(user_context.name).get('href') task = Task(user_client) server_config = server_utils.get_server_runtime_config() if not server_utils.is_no_vc_communication_mode(): native_templates = \ template_reader.read_native_template_definition_from_catalog( config=server_config ) server_config.set_value_at('broker.templates', native_templates) task.update(status=TaskStatus.RUNNING.value, namespace='vcloud.cse', operation="Finished reloading native templates.", operation_name='template reload', details='', progress=None, owner_href=user_context.org_href, owner_name=user_context.org_name, owner_type='application/vnd.vmware.vcloud.org+xml', user_href=user_href, user_name=user_context.name, org_href=user_context.org_href, task_href=task_href) else: msg = "Skipping loading k8s template definition from catalog " \ "since `No communication with VCenter` mode is on." logger.SERVER_LOGGER.info(msg) server_config.set_value_at('broker.templates', []) task.update(status=TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=msg, operation_name='template reload', details='', progress=None, owner_href=user_context.org_href, owner_name=user_context.org_name, owner_type='application/vnd.vmware.vcloud.org+xml', user_href=user_href, user_name=user_context.name, org_href=user_context.org_href, task_href=task_href) task.update(status=TaskStatus.RUNNING.value, namespace='vcloud.cse', operation="Reloading TKG templates.", operation_name='template reload', details='', progress=None, owner_href=user_context.org_href, owner_name=user_context.org_name, owner_type='application/vnd.vmware.vcloud.org+xml', user_href=user_href, user_name=user_context.name, org_href=user_context.org_href, task_href=task_href) tkgm_templates = \ template_reader.read_tkgm_template_definition_from_catalog( config=server_config ) server_config.set_value_at('broker.tkgm_templates', tkgm_templates) task.update(status=TaskStatus.SUCCESS.value, namespace='vcloud.cse', operation="Finished reloading all templates.", operation_name='template reload', details='', progress=None, owner_href=user_context.org_href, owner_name=user_context.org_name, owner_type='application/vnd.vmware.vcloud.org+xml', user_href=user_href, user_name=user_context.name, org_href=user_context.org_href, task_href=task_href) except Exception: msg = "Error reloading templates." logger.SERVER_LOGGER.error(msg, exc_info=True) if task and user_context and user_href: task.update(status=TaskStatus.ERROR.value, namespace='vcloud.cse', operation=msg, operation_name='template reload', details='', progress=None, owner_href=user_context.org_href, owner_name=user_context.org_name, owner_type='application/vnd.vmware.vcloud.org+xml', user_href=user_href, user_name=user_context.name, org_href=user_context.org_href, task_href=task_href) finally: op_ctx.end()
def init_rde_environment(config_filepath=BASE_CONFIG_FILEPATH, logger=NULL_LOGGER): # noqa: E501 """Set up module variables according to config dict. :param str config_filepath: """ global CLIENT, ORG_HREF, VDC_HREF, \ CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \ TEMPLATE_DEFINITIONS, TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \ CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_LOGIN_CMD, \ USERNAME_TO_LOGIN_CMD, USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, \ TEST_VDC_HREF, VCD_API_VERSION_TO_USE logger.debug("Setting RDE environement") config = testutils.yaml_to_dict(config_filepath) logger.debug(f"Config file used: {config}") # download all remote template scripts rtm = RemoteTemplateManager( config['broker']['remote_template_cookbook_url'], legacy_mode=config['service']['legacy_mode']) template_cookbook = rtm.get_filtered_remote_template_cookbook() TEMPLATE_DEFINITIONS = template_cookbook['templates'] rtm.download_all_template_scripts(force_overwrite=True) sysadmin_client = Client(config['vcd']['host'], verify_ssl_certs=config['vcd']['verify']) sysadmin_client.set_credentials( BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password'])) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set( shared_constants.SUPPORTED_VCD_API_VERSIONS) # noqa: E501 common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() max_api_version = get_max_api_version(common_supported_api_versions) CLIENT = Client(config['vcd']['host'], api_version=max_api_version, verify_ssl_certs=config['vcd']['verify']) credentials = BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password']) CLIENT.set_credentials(credentials) VCD_API_VERSION_TO_USE = max_api_version logger.debug(f"Using VCD api version: {VCD_API_VERSION_TO_USE}") CATALOG_NAME = config['broker']['catalog'] SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \ f"{config['vcd']['username']} " \ f"-iwp {config['vcd']['password']} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_ADMIN_NAME} " \ f"-iwp {CLUSTER_ADMIN_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_AUTHOR_NAME} " \ f"-iwp {CLUSTER_AUTHOR_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" USERNAME_TO_LOGIN_CMD = { SYS_ADMIN_NAME: SYS_ADMIN_LOGIN_CMD, CLUSTER_ADMIN_NAME: CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_NAME: CLUSTER_AUTHOR_LOGIN_CMD } # hrefs for Org and VDC that hosts the catalog org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org']) vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=config['broker']['vdc'], org=org) ORG_HREF = org.href VDC_HREF = vdc.href logger.debug(f"Using template org {org.get_name()} with href {ORG_HREF}") logger.debug(f"Using template vdc {vdc.name} with href {VDC_HREF}") # hrefs for Org and VDC that tests cluster operations test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG) test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org) TEST_ORG_HREF = test_org.href TEST_VDC_HREF = test_vdc.href logger.debug(f"Using test org {test_org.get_name()} " f"with href {TEST_ORG_HREF}") logger.debug(f"Using test vdc {test_vdc.name} with href {TEST_VDC_HREF}") create_cluster_admin_role(config['vcd'], logger=logger) create_cluster_author_role(config['vcd'], logger=logger)
def init_rde_environment(config_filepath=BASE_CONFIG_FILEPATH, logger=NULL_LOGGER): # noqa: E501 """Set up module variables according to config dict. :param str config_filepath: :param logging.Logger logger: """ global CLIENT, ORG_HREF, VDC_HREF, \ CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \ TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \ CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_LOGIN_CMD, \ USERNAME_TO_LOGIN_CMD, USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, \ TEST_VDC_HREF, VCD_API_VERSION_TO_USE, VCD_SITE logger.debug("Setting RDE environement") config = testutils.yaml_to_dict(config_filepath) logger.debug(f"Config file used: {config}") sysadmin_client = Client(config['vcd']['host'], verify_ssl_certs=config['vcd']['verify']) sysadmin_client.set_credentials( BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password'])) vcd_supported_api_versions = \ set(sysadmin_client.get_supported_versions_list()) cse_supported_api_versions = set( shared_constants.SUPPORTED_VCD_API_VERSIONS) # noqa: E501 common_supported_api_versions = \ list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 common_supported_api_versions.sort() max_api_version = get_max_api_version(common_supported_api_versions) CLIENT = Client(config['vcd']['host'], api_version=max_api_version, verify_ssl_certs=config['vcd']['verify']) credentials = BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password']) CLIENT.set_credentials(credentials) VCD_API_VERSION_TO_USE = max_api_version logger.debug(f"Using VCD api version: {VCD_API_VERSION_TO_USE}") CATALOG_NAME = config['broker']['catalog'] VCD_SITE = f"https://{config['vcd']['host']}" SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \ f"{config['vcd']['username']} " \ f"-iwp {config['vcd']['password']} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_ADMIN_NAME} " \ f"-iwp {CLUSTER_ADMIN_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" CLUSTER_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {CLUSTER_AUTHOR_NAME} " \ f"-iwp {CLUSTER_AUTHOR_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" USERNAME_TO_LOGIN_CMD = { SYS_ADMIN_NAME: SYS_ADMIN_LOGIN_CMD, CLUSTER_ADMIN_NAME: CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_NAME: CLUSTER_AUTHOR_LOGIN_CMD } # hrefs for Org and VDC that hosts the catalog org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org']) vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=config['broker']['vdc'], org=org) ORG_HREF = org.href VDC_HREF = vdc.href logger.debug(f"Using template org {org.get_name()} with href {ORG_HREF}") logger.debug(f"Using template vdc {vdc.name} with href {VDC_HREF}") # hrefs for Org and VDC that tests cluster operations test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG) test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org) TEST_ORG_HREF = test_org.href TEST_VDC_HREF = test_vdc.href logger.debug(f"Using test org {test_org.get_name()} " f"with href {TEST_ORG_HREF}") logger.debug(f"Using test vdc {test_vdc.name} with href {TEST_VDC_HREF}") if SHOULD_INSTALL_PREREQUISITES: create_cluster_admin_role(config['vcd'], logger=logger) create_cluster_author_role(config['vcd'], logger=logger) # create and publish sizing class sc1 to TEST_VDC cpm = ComputePolicyManager(sysadmin_client=sysadmin_client, log_wire=True) created_policy = None try: created_policy = cpm.add_vdc_compute_policy( SIZING_CLASS_NAME, description=SIZING_CLASS_DESCRIPTION, cpu_count=2, memory_mb=2048) except HTTPError as err: if 'already exists' in err.response.text: logger.debug( f"Compute policy {SIZING_CLASS_NAME} already exists" ) # noqa: E501 created_policy = cpm.get_vdc_compute_policy(SIZING_CLASS_NAME) else: logger.error( f"Request to create sizing policy {SIZING_CLASS_NAME} failed." ) # noqa: E501 raise try: cpm.add_compute_policy_to_vdc( pyvcloud_utils.extract_id( test_vdc.get_resource_admin().get('id')), # noqa: E501 created_policy['id']) except Exception as err: logger.error( f"Error publishing sizing policy {SIZING_CLASS_NAME} to vdc {TEST_VDC}: {err}" ) # noqa: E501 create_cluster_admin_role(config['vcd'], logger=logger) create_cluster_author_role(config['vcd'], logger=logger)
def __init__(self, client, sys_admin_client, build_params, org=None, vdc=None, ssh_key=None, logger=NULL_LOGGER, msg_update_callback=NullPrinter(), log_wire=False): """. :param pyvcloud.vcd.Client client: :param pyvcloud.vcd.Client sys_admin_client: :param dict build_params: :param pyvcloud.vcd.org.Org org: specific org to use. Will override the org_name specified in build_params, can be used to save few vCD calls to create the Org object. :param pyvcloud.vcd.vdc.VDC vdc: specific vdc to use. Will override the vdc_name specified in build_params, can be used to save few vCD calls to create the Vdc object. :param str ssh_key: public ssh key to place into the template vApp(s). :param logging.Logger logger: logger object. :param utils.ConsoleMessagePrinter msg_update_callback: Callback object. """ self._is_valid = False self.client = client self.sys_admin_client = sys_admin_client self.ssh_key = ssh_key self.logger = logger self.msg_update_callback = msg_update_callback if self.client is None or self.sys_admin_client is None: return # validate and populate required fields self.template_name = build_params.get( TemplateBuildKey.TEMPLATE_NAME) # noqa: E501 self.template_revision = build_params.get( TemplateBuildKey.TEMPLATE_REVISION) # noqa: E501 self.ova_name = build_params.get( TemplateBuildKey.SOURCE_OVA_NAME) # noqa: E501 self.ova_href = build_params.get( TemplateBuildKey.SOURCE_OVA_HREF) # noqa: E501 self.ova_sha256 = build_params.get( TemplateBuildKey.SOURCE_OVA_SHA256) # noqa: E501 if org: self.org = org self.org_name = org.get_name() else: self.org_name = build_params.get( TemplateBuildKey.ORG_NAME) # noqa: E501 self.org = get_org(self.client, org_name=self.org_name) if vdc: self.vdc = vdc self.vdc.get_resource() # to make sure vdc.resource is populated self.vdc_name = vdc.name else: self.vdc_name = build_params.get( TemplateBuildKey.VDC_NAME) # noqa: E501 self.vdc = get_vdc(self.client, vdc_name=self.vdc_name, org=self.org) self.catalog_name = build_params.get( TemplateBuildKey.CATALOG_NAME) # noqa: E501 self.catalog_item_name = build_params.get( TemplateBuildKey.CATALOG_ITEM_NAME) # noqa: E501 self.catalog_item_description = \ build_params.get(TemplateBuildKey.CATALOG_ITEM_DESCRIPTION) # noqa: E501 self.temp_vapp_name = build_params.get( TemplateBuildKey.TEMP_VAPP_NAME) # noqa: E501 self.temp_vm_name = build_params.get( TemplateBuildKey.TEMP_VM_NAME) # noqa: E501 self.cpu = build_params.get(TemplateBuildKey.CPU) self.memory = build_params.get(TemplateBuildKey.MEMORY) self.network_name = build_params.get( TemplateBuildKey.NETWORK_NAME) # noqa: E501 self.ip_allocation_mode = build_params.get( TemplateBuildKey.IP_ALLOCATION_MODE) # noqa: E501 self.storage_profile = build_params.get( TemplateBuildKey.STORAGE_PROFILE) # noqa: E501 self.cse_placement_policy = build_params.get( TemplateBuildKey.CSE_PLACEMENT_POLICY) # noqa: E501 self.remote_cookbook_version = build_params.get( TemplateBuildKey.REMOTE_COOKBOOK_VERSION) # noqa: E501 self.log_wire = log_wire if self.template_name and self.template_revision and \ self.ova_name and self.ova_href and self.ova_sha256 and \ self.org and self.org_name and self.vdc and self.vdc_name and \ self.catalog_name and self.catalog_item_name and \ self.catalog_item_description and self.temp_vapp_name and \ self.temp_vm_name and self.cpu and self.memory and \ self.network_name and self.ip_allocation_mode and \ self.storage_profile: self._is_valid = True
def remove_compute_policy_from_vdc_sync(self, vdc, compute_policy_href, force=False, is_placement_policy=False, task_resource=None): """Remove compute policy from vdc. This method makes use of an umbrella task which can be used for tracking progress. If the umbrella task is not specified, it is created. :param pyvcloud.vcd.vdc.VDC vdc: VDC object :param str compute_policy_href: href of the compute policy to remove :param bool force: Force remove compute policy from vms in the VDC as well :param lxml.objectify.Element task_resource: Task resource for the umbrella task """ user_name = self._session.get('user') task = Task(self._sysadmin_client) task_href = None is_umbrella_task = task_resource is not None # Create a task if not umbrella task if not is_umbrella_task: # TODO the following org will be associated with 'System' org. # task created should be associated with the corresponding org of # the vdc object. org = vcd_utils.get_org(self._sysadmin_client) org.reload() user_href = org.get_user(user_name).get('href') org_href = org.href task_resource = task.update( status=vcd_client.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation= f"Removing compute policy (href: {compute_policy_href})" # noqa: E501 f" from org VDC (vdc id: {vdc.name})", operation_name='Remove org VDC compute policy', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, org_href=org.href) else: user_href = task_resource.User.get('href') org_href = task_resource.Organization.get('href') task_href = task_resource.get('href') try: # remove the compute policy from VMs if force is True if force: compute_policy_id = retrieve_compute_policy_id_from_href( compute_policy_href) # noqa: E501 vdc_id = vcd_utils.extract_id(vdc.get_resource().get('id')) vapps = vcd_utils.get_all_vapps_in_ovdc( client=self._sysadmin_client, ovdc_id=vdc_id) target_vms = [] system_default_href = None operation_msg = None for cp_dict in self.list_compute_policies_on_vdc(vdc_id): if cp_dict['name'] == _SYSTEM_DEFAULT_COMPUTE_POLICY: system_default_href = cp_dict['href'] break if is_placement_policy: for vapp in vapps: target_vms += \ [vm for vm in vapp.get_all_vms() if self._get_vm_placement_policy_id(vm) == compute_policy_id] # noqa: E501 vm_names = [vm.get('name') for vm in target_vms] operation_msg = f"Removing placement policy from " \ f"{len(vm_names)} VMs. " \ f"Affected VMs: {vm_names}" else: for vapp in vapps: target_vms += \ [vm for vm in vapp.get_all_vms() if self._get_vm_sizing_policy_id(vm) == compute_policy_id] # noqa: E501 vm_names = [vm.get('name') for vm in target_vms] operation_msg = "Setting sizing policy to " \ f"'{_SYSTEM_DEFAULT_COMPUTE_POLICY}' on " \ f"{len(vm_names)} VMs. " \ f"Affected VMs: {vm_names}" task.update(status=vcd_client.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=operation_msg, operation_name='Remove org VDC compute policy', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, task_href=task_href, org_href=org_href) task_monitor = self._sysadmin_client.get_task_monitor() for vm_resource in target_vms: vm = VM(self._sysadmin_client, href=vm_resource.get('href')) _task = None operation_msg = None if is_placement_policy: if hasattr(vm_resource, 'ComputePolicy') and \ not hasattr(vm_resource.ComputePolicy, 'VmSizingPolicy'): # noqa: E501 # Updating sizing policy for the VM _task = vm.update_compute_policy( compute_policy_href=system_default_href) operation_msg = \ "Setting compute policy to " \ f"'{_SYSTEM_DEFAULT_COMPUTE_POLICY}' "\ f"on VM '{vm_resource.get('name')}'" task.update( status=vcd_client.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=operation_msg, operation_name= f'Setting sizing policy to {_SYSTEM_DEFAULT_COMPUTE_POLICY}', # noqa: E501 details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, task_href=task_href, org_href=org_href) task_monitor.wait_for_success(_task) _task = vm.remove_placement_policy() operation_msg = "Removing placement policy on VM " \ f"'{vm_resource.get('name')}'" task.update( status=vcd_client.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=operation_msg, operation_name='Remove org VDC compute policy', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, task_href=task_href, org_href=org_href) task_monitor.wait_for_success(_task) else: _task = vm.update_compute_policy( compute_policy_href=system_default_href) operation_msg = "Setting sizing policy to " \ f"'{_SYSTEM_DEFAULT_COMPUTE_POLICY}' "\ f"on VM '{vm_resource.get('name')}'" task.update( status=vcd_client.TaskStatus.RUNNING.value, namespace='vcloud.cse', operation=operation_msg, operation_name='Remove org VDC compute policy', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, task_href=task_href, org_href=org_href) task_monitor.wait_for_success(_task) final_status = vcd_client.TaskStatus.RUNNING.value \ if is_umbrella_task else vcd_client.TaskStatus.SUCCESS.value task.update(status=final_status, namespace='vcloud.cse', operation=f"Removing compute policy (href:" f"{compute_policy_href}) from org VDC '{vdc.name}'", operation_name='Remove org VDC compute policy', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=user_name, task_href=task_href, org_href=org_href) vdc.remove_compute_policy(compute_policy_href) except Exception as err: logger.SERVER_LOGGER.error(err, exc_info=True) # Set task to error if not an umbrella task if not is_umbrella_task: msg = 'Failed to remove compute policy: ' \ f'{compute_policy_href} from the OVDC: {vdc.name}' task.update(status=vcd_client.TaskStatus.ERROR.value, namespace='vcloud.cse', operation=msg, operation_name='Remove org VDC compute policy', details='', progress=None, owner_href=vdc.href, owner_name=vdc.name, owner_type=vcd_client.EntityType.VDC.value, user_href=user_href, user_name=self._session.get('user'), task_href=task_href, org_href=org_href, error_message=f"{err}", stack_trace='') raise err
def init_environment(config_filepath=BASE_CONFIG_FILEPATH): """Set up module variables according to config dict. :param str config_filepath: """ global AMQP_USERNAME, AMQP_PASSWORD, CLIENT, ORG_HREF, VDC_HREF, \ CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \ TEMPLATE_DEFINITIONS, TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \ ORG_ADMIN_LOGIN_CMD, K8_AUTHOR_LOGIN_CMD, USERNAME_TO_LOGIN_CMD, \ USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, TEST_VDC_HREF, \ VCD_API_VERSION_TO_USE, TEMPLATE_COOKBOOK_VERSION config = testutils.yaml_to_dict(config_filepath) rtm = \ RemoteTemplateManager(config['broker']['remote_template_cookbook_url'], legacy_mode=config['service']['legacy_mode']) template_cookbook = rtm.get_filtered_remote_template_cookbook() TEMPLATE_COOKBOOK_VERSION = rtm.cookbook_version TEMPLATE_DEFINITIONS = template_cookbook['templates'] rtm.download_all_template_scripts(force_overwrite=True) CLIENT = Client(config['vcd']['host'], api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify']) credentials = BasicLoginCredentials(config['vcd']['username'], shared_constants.SYSTEM_ORG_NAME, config['vcd']['password']) CLIENT.set_credentials(credentials) VCD_API_VERSION_TO_USE = config['vcd']['api_version'] CATALOG_NAME = config['broker']['catalog'] AMQP_USERNAME = config['amqp']['username'] AMQP_PASSWORD = config['amqp']['password'] SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \ f"{config['vcd']['username']} " \ f"-iwp {config['vcd']['password']} " \ f"-V {VCD_API_VERSION_TO_USE}" ORG_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {ORG_ADMIN_NAME} -iwp {ORG_ADMIN_PASSWORD} " \ f"-V {VCD_API_VERSION_TO_USE}" K8_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG} " \ f"{K8_AUTHOR_NAME} -iwp {K8_AUTHOR_PASSWORD}" \ f" -V {VCD_API_VERSION_TO_USE}" USERNAME_TO_LOGIN_CMD = { 'sys_admin': SYS_ADMIN_LOGIN_CMD, 'org_admin': ORG_ADMIN_LOGIN_CMD, 'k8_author': K8_AUTHOR_LOGIN_CMD } USERNAME_TO_CLUSTER_NAME = { 'sys_admin': SYS_ADMIN_TEST_CLUSTER_NAME, 'org_admin': ORG_ADMIN_TEST_CLUSTER_NAME, 'k8_author': K8_AUTHOR_TEST_CLUSTER_NAME } # hrefs for Org and VDC that hosts the catalog org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org']) vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=config['broker']['vdc'], org=org) ORG_HREF = org.href VDC_HREF = vdc.href # hrefs for Org and VDC that tests cluster operations test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG) test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org) TEST_ORG_HREF = test_org.href TEST_VDC_HREF = test_vdc.href create_k8_author_role(config['vcd'])
def get_all_k8s_local_template_definition( client, catalog_name, org=None, org_name=None, legacy_mode=False, logger_debug=logger.NULL_LOGGER, msg_update_callback=utils.NullPrinter()): # noqa: E501 """Fetch all CSE k8s templates in a catalog. A CSE k8s template is a catalog item that has all the necessary metadata stamped onto it. If only partial metadata is present on a catalog item, that catalog item will be disqualified from the result. :param pyvcloud.vcd.Client client: A sys admin client to be used to retrieve metadata off the catalog items. :param str catalog_name: Name of the catalog where the template resides. :param pyvcloud.vcd.Org org: Org object which hosts the catalog. :param str org_name: Name of the org that is hosting the catalog. Can be provided in lieu of param org, however param org takes precedence. :param bool legacy_mode: True, if CSE is running in legacy mode :param logging.Logger logger_debug: :param utils.NullPrinter msg_update_callback: :return: list of dictionaries containing template data :rtype: list of dicts """ if not org: org = get_org(client, org_name=org_name) catalog_item_names = [ entry['name'] for entry in org.list_catalog_items(catalog_name) ] templates = [] # Select the right Key enum based on legacy_mode flag localTemplateKey = LocalTemplateKey if legacy_mode: # if template is loaded in legacy mode, make sure to avoid the keys # min_cse_version and max_cse_version localTemplateKey = LegacyLocalTemplateKey for item_name in catalog_item_names: md = org.get_all_metadata_from_catalog_item(catalog_name=catalog_name, item_name=item_name) metadata_dict = metadata_to_dict(md) # if catalog item doesn't have all the required metadata keys, # CSE should not recognize it as a template expected_metadata_keys = \ set([entry.value for entry in localTemplateKey]) missing_metadata_keys = expected_metadata_keys - metadata_dict.keys() num_missing_metadata_keys = len(missing_metadata_keys) if num_missing_metadata_keys == len(expected_metadata_keys): # This catalog item has no CSE related metadata, so skip it. continue if num_missing_metadata_keys > 0: # This catalog item has partial CSE metadata, so skip it but also # log relevant information. msg = f"Catalog item '{item_name}' missing " \ f"{num_missing_metadata_keys} metadata: " \ f"{missing_metadata_keys}" # noqa: F841 logger_debug.debug(msg) msg_update_callback.info(msg) continue if not legacy_mode: # Do not load the template in non-legacy_mode if # min_cse_version and max_cse_version are not present # in the metadata_dict curr_cse_version = server_utils.get_installed_cse_version() valid_cse_versions = semantic_version.SimpleSpec( f">={metadata_dict[localTemplateKey.MIN_CSE_VERSION]}," f"<={metadata_dict[localTemplateKey.MAX_CSE_VERSION]}") if not valid_cse_versions.match(curr_cse_version): template_name = \ metadata_dict.get(localTemplateKey.NAME, "Unknown") template_revision = \ metadata_dict.get(localTemplateKey.REVISION, "Unknown") msg = f"Template '{template_name}' at " \ f"revision '{template_revision}' exists but is " \ f"not valid for CSE {curr_cse_version}" logger_debug.debug(msg) msg_update_callback.info(msg) continue # non-string metadata is written to the dictionary as a string # when 'upgrade_from' metadata is empty, vcd returns it as: "['']" # when 'upgrade_from' metadata is not empty, vcd returns it as an array # coerce "['']" to the more usable empty array [] if isinstance(metadata_dict[localTemplateKey.UPGRADE_FROM], str): metadata_dict[localTemplateKey.UPGRADE_FROM] = \ ast.literal_eval(metadata_dict[localTemplateKey.UPGRADE_FROM]) if metadata_dict[localTemplateKey.UPGRADE_FROM] == ['']: metadata_dict[localTemplateKey.UPGRADE_FROM] = [] templates.append(metadata_dict) return templates