def delete_nodes_thread(self): LOGGER.debug(f"About to delete nodes from cluster with name: " f"{self.cluster_name}") try: vapp = VApp(self.tenant_client, href=self.cluster['vapp_href']) template = self._get_template() self._update_task( TaskStatus.RUNNING, message=f"Deleting " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" node(s) from " f"{self.cluster_name}({self.cluster_id})") try: server_config = get_server_runtime_config() delete_nodes_from_cluster( server_config, vapp, template, self.req_spec.get(RequestKey.NODE_NAMES_LIST), self.req_spec.get(RequestKey.FORCE_DELETE)) except Exception: LOGGER.error(f"Couldn't delete node " f"{self.req_spec.get(RequestKey.NODE_NAMES_LIST)}" f" from cluster:{self.cluster_name}") self._update_task( TaskStatus.RUNNING, message=f"Undeploying " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" node(s) for {self.cluster_name}({self.cluster_id})") for vm_name in self.req_spec.get(RequestKey.NODE_NAMES_LIST): vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name)) try: task = vm.undeploy() self.tenant_client.get_task_monitor().wait_for_status(task) except Exception: LOGGER.warning(f"Couldn't undeploy VM {vm_name}") self._update_task( TaskStatus.RUNNING, message=f"Deleting " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" VM(s) for {self.cluster_name}({self.cluster_id})") task = vapp.delete_vms(self.req_spec.get(RequestKey.NODE_NAMES_LIST)) # noqa: E501 self.tenant_client.get_task_monitor().wait_for_status(task) self._update_task( TaskStatus.SUCCESS, message=f"Deleted " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" node(s) to cluster " f"{self.cluster_name}({self.cluster_id})") except Exception as e: LOGGER.error(traceback.format_exc()) error_obj = error_to_json(e) stack_trace = \ ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) finally: self._disconnect_sys_admin()
def _delete_nodes_async(self, *args, cluster_name, cluster_vapp_href, node_names_list): try: self._update_task( TaskStatus.RUNNING, message=f"Deleting {len(node_names_list)} node(s) " f"from cluster {cluster_name}") self._delete_nodes(cluster_name=cluster_name, cluster_vapp_href=cluster_vapp_href, node_names_list=node_names_list) self._update_task( TaskStatus.SUCCESS, message=f"Deleted {len(node_names_list)} node(s)" f" to cluster {cluster_name}") except Exception as e: LOGGER.error(f"Unexpected error while deleting nodes " f"{node_names_list}: {e}", exc_info=True) error_obj = error_to_json(e) stack_trace = ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) # noqa: E501 self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) finally: self.logout_sys_admin_client()
def create_nodes_thread(self): LOGGER.debug(f"About to add nodes to cluster with name: " f"{self.cluster_name}") try: server_config = get_server_runtime_config() org_resource = self.tenant_client.get_org() org = Org(self.tenant_client, resource=org_resource) vdc = VDC(self.tenant_client, href=self.cluster['vdc_href']) vapp = VApp(self.tenant_client, href=self.cluster['vapp_href']) template = self._get_template() self._update_task( TaskStatus.RUNNING, message=f"Creating {self.req_spec.get(RequestKey.NUM_WORKERS)}" f" node(s) for {self.cluster_name}({self.cluster_id})") node_type = NodeType.WORKER if self.req_spec.get(RequestKey.ENABLE_NFS): node_type = NodeType.NFS new_nodes = add_nodes(self.req_spec.get(RequestKey.NUM_WORKERS), template, node_type, server_config, self.tenant_client, org, vdc, vapp, self.req_spec) if node_type == NodeType.NFS: self._update_task( TaskStatus.SUCCESS, message=f"Created " f"{self.req_spec.get(RequestKey.NUM_WORKERS)} " f"node(s) for " f"{self.cluster_name}({self.cluster_id})") elif node_type == NodeType.WORKER: self._update_task( TaskStatus.RUNNING, message=f"Adding " f"{self.req_spec.get(RequestKey.NUM_WORKERS)} " f"node(s) to cluster " f"{self.cluster_name}({self.cluster_id})") target_nodes = [] for spec in new_nodes['specs']: target_nodes.append(spec['target_vm_name']) vapp.reload() join_cluster(server_config, vapp, template, target_nodes) self._update_task( TaskStatus.SUCCESS, message=f"Added " f"{self.req_spec.get(RequestKey.NUM_WORKERS)} " f"node(s) to cluster " f"{self.cluster_name}({self.cluster_id})") except NodeCreationError as e: error_obj = error_to_json(e) LOGGER.error(traceback.format_exc()) stack_trace = \ ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) raise except Exception as e: error_obj = error_to_json(e) LOGGER.error(traceback.format_exc()) stack_trace = \ ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) finally: self._disconnect_sys_admin()
def create_cluster_thread(self): network_name = self.req_spec.get(RequestKey.NETWORK_NAME) try: clusters = load_from_metadata(self.tenant_client, name=self.cluster_name) if len(clusters) != 0: raise ClusterAlreadyExistsError(f"Cluster {self.cluster_name} " "already exists.") org_resource = self.tenant_client.get_org_by_name( self.req_spec.get(RequestKey.ORG_NAME)) org = Org(self.tenant_client, resource=org_resource) vdc_resource = org.get_vdc(self.req_spec.get(RequestKey.OVDC_NAME)) vdc = VDC(self.tenant_client, resource=vdc_resource) template = self._get_template() self._update_task( TaskStatus.RUNNING, message=f"Creating cluster vApp {self.cluster_name}" f"({self.cluster_id})") try: vapp_resource = vdc.create_vapp( self.cluster_name, description=f"cluster {self.cluster_name}", network=network_name, fence_mode='bridged') except Exception as e: raise ClusterOperationError( "Error while creating vApp:", str(e)) self.tenant_client.get_task_monitor().wait_for_status( vapp_resource.Tasks.Task[0]) tags = {} tags['cse.cluster.id'] = self.cluster_id tags['cse.version'] = pkg_resources.require( 'container-service-extension')[0].version tags['cse.template'] = template['name'] vapp = VApp(self.tenant_client, href=vapp_resource.get('href')) for k, v in tags.items(): task = vapp.set_metadata('GENERAL', 'READWRITE', k, v) self.tenant_client.get_task_monitor().wait_for_status(task) self._update_task( TaskStatus.RUNNING, message=f"Creating master node for {self.cluster_name}" f"({self.cluster_id})") vapp.reload() server_config = get_server_runtime_config() try: add_nodes(1, template, NodeType.MASTER, server_config, self.tenant_client, org, vdc, vapp, self.req_spec) except Exception as e: raise MasterNodeCreationError( "Error while adding master node:", str(e)) self._update_task( TaskStatus.RUNNING, message=f"Initializing cluster {self.cluster_name}" f"({self.cluster_id})") vapp.reload() init_cluster(server_config, vapp, template) master_ip = get_master_ip(server_config, vapp, template) task = vapp.set_metadata('GENERAL', 'READWRITE', 'cse.master.ip', master_ip) self.tenant_client.get_task_monitor().wait_for_status(task) if self.req_spec.get(RequestKey.NUM_WORKERS) > 0: self._update_task( TaskStatus.RUNNING, message=f"Creating " f"{self.req_spec.get(RequestKey.NUM_WORKERS)} " f"node(s) for " f"{self.cluster_name}({self.cluster_id})") try: add_nodes(self.req_spec.get(RequestKey.NUM_WORKERS), template, NodeType.WORKER, server_config, self.tenant_client, org, vdc, vapp, self.req_spec) except Exception as e: raise WorkerNodeCreationError( "Error while creating worker node:", str(e)) self._update_task( TaskStatus.RUNNING, message=f"Adding " f"{self.req_spec.get(RequestKey.NUM_WORKERS)} " f"node(s) to " f"{self.cluster_name}({self.cluster_id})") vapp.reload() join_cluster(server_config, vapp, template) if self.req_spec.get(RequestKey.ENABLE_NFS): self._update_task( TaskStatus.RUNNING, message=f"Creating NFS node for {self.cluster_name}" f"({self.cluster_id})") try: add_nodes(1, template, NodeType.NFS, server_config, self.tenant_client, org, vdc, vapp, self.req_spec) except Exception as e: raise NFSNodeCreationError( "Error while creating NFS node:", str(e)) self._update_task( TaskStatus.SUCCESS, message=f"Created cluster {self.cluster_name}" f"({self.cluster_id})") except (MasterNodeCreationError, WorkerNodeCreationError, NFSNodeCreationError, ClusterJoiningError, ClusterInitializationError, ClusterOperationError) as e: LOGGER.error(traceback.format_exc()) error_obj = error_to_json(e) stack_trace = \ ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY] [ERROR_DESCRIPTION_KEY], stack_trace=stack_trace) raise e except Exception as e: LOGGER.error(traceback.format_exc()) error_obj = error_to_json(e) stack_trace = \ ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) finally: self._disconnect_sys_admin()
def _create_nodes_async(self, *args, cluster_name, cluster_vdc_href, cluster_vapp_href, cluster_id, template_name, template_revision, num_workers, network_name, num_cpu, mb_memory, storage_profile_name, ssh_key_filepath, enable_nfs, rollback): org = vcd_utils.get_org(self.tenant_client) vdc = VDC(self.tenant_client, href=cluster_vdc_href) vapp = VApp(self.tenant_client, href=cluster_vapp_href) template = get_template(name=template_name, revision=template_revision) msg = f"Creating {num_workers} node(s) from template " \ f"'{template_name}' (revision {template_revision}) and " \ f"adding to {cluster_name} ({cluster_id})" LOGGER.debug(msg) try: self._update_task(TaskStatus.RUNNING, message=msg) node_type = NodeType.WORKER if enable_nfs: node_type = NodeType.NFS server_config = utils.get_server_runtime_config() catalog_name = server_config['broker']['catalog'] new_nodes = add_nodes(client=self.tenant_client, num_nodes=num_workers, node_type=node_type, org=org, vdc=vdc, vapp=vapp, catalog_name=catalog_name, template=template, network_name=network_name, num_cpu=num_cpu, memory_in_mb=mb_memory, storage_profile=storage_profile_name, ssh_key_filepath=ssh_key_filepath) if node_type == NodeType.NFS: self._update_task( TaskStatus.SUCCESS, message=f"Created {num_workers} node(s) for " f"{cluster_name}({cluster_id})") elif node_type == NodeType.WORKER: self._update_task( TaskStatus.RUNNING, message=f"Adding {num_workers} node(s) to cluster " f"{cluster_name}({cluster_id})") target_nodes = [] for spec in new_nodes['specs']: target_nodes.append(spec['target_vm_name']) vapp.reload() join_cluster(vapp, template[LocalTemplateKey.NAME], template[LocalTemplateKey.REVISION], target_nodes) self._update_task( TaskStatus.SUCCESS, message=f"Added {num_workers} node(s) to cluster " f"{cluster_name}({cluster_id})") except NodeCreationError as e: if rollback: msg = f"Error adding nodes to {cluster_name} {cluster_id}." \ f" Deleting nodes: {e.node_names} (rollback=True)" self._update_task(TaskStatus.RUNNING, message=msg) LOGGER.info(msg) try: self._delete_nodes(cluster_name=cluster_name, cluster_vapp_href=cluster_vapp_href, node_names_list=e.node_names) except Exception: LOGGER.error(f"Failed to delete nodes {e.node_names} " f"from cluster {cluster_name}", exc_info=True) LOGGER.error(f"Error adding nodes to {cluster_name}", exc_info=True) error_obj = error_to_json(e) LOGGER.error(str(e), exc_info=True) stack_trace = ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) # noqa: E501 self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) # raising an exception here prints a stacktrace to server console except Exception as e: error_obj = error_to_json(e) LOGGER.error(str(e), exc_info=True) stack_trace = ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) # noqa: E501 self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) finally: self.logout_sys_admin_client()
def _create_cluster_async(self, *args, org_name, ovdc_name, cluster_name, cluster_id, template_name, template_revision, num_workers, network_name, num_cpu, mb_memory, storage_profile_name, ssh_key_filepath, enable_nfs, rollback): org = vcd_utils.get_org(self.tenant_client, org_name=org_name) vdc = vcd_utils.get_vdc( self.tenant_client, vdc_name=ovdc_name, org=org) LOGGER.debug(f"About to create cluster {cluster_name} on {ovdc_name}" f" with {num_workers} worker nodes, " f"storage profile={storage_profile_name}") try: self._update_task( TaskStatus.RUNNING, message=f"Creating cluster vApp {cluster_name}({cluster_id})") try: vapp_resource = \ vdc.create_vapp(cluster_name, description=f"cluster {cluster_name}", network=network_name, fence_mode='bridged') except Exception as e: msg = f"Error while creating vApp: {e}" LOGGER.debug(str(e)) raise ClusterOperationError(msg) self.tenant_client.get_task_monitor().wait_for_status(vapp_resource.Tasks.Task[0]) # noqa: E501 template = get_template(template_name, template_revision) tags = { ClusterMetadataKey.CLUSTER_ID: cluster_id, ClusterMetadataKey.CSE_VERSION: pkg_resources.require('container-service-extension')[0].version, # noqa: E501 ClusterMetadataKey.TEMPLATE_NAME: template[LocalTemplateKey.NAME], # noqa: E501 ClusterMetadataKey.TEMPLATE_REVISION: template[LocalTemplateKey.REVISION] # noqa: E501 } vapp = VApp(self.tenant_client, href=vapp_resource.get('href')) task = vapp.set_multiple_metadata(tags) self.tenant_client.get_task_monitor().wait_for_status(task) self._update_task( TaskStatus.RUNNING, message=f"Creating master node for " f"{cluster_name} ({cluster_id})") vapp.reload() server_config = utils.get_server_runtime_config() catalog_name = server_config['broker']['catalog'] try: add_nodes(client=self.tenant_client, num_nodes=1, node_type=NodeType.MASTER, org=org, vdc=vdc, vapp=vapp, catalog_name=catalog_name, template=template, network_name=network_name, num_cpu=num_cpu, memory_in_mb=mb_memory, storage_profile=storage_profile_name, ssh_key_filepath=ssh_key_filepath) except Exception as e: raise MasterNodeCreationError("Error adding master node:", str(e)) self._update_task( TaskStatus.RUNNING, message=f"Initializing cluster {cluster_name} ({cluster_id})") vapp.reload() init_cluster(vapp, template[LocalTemplateKey.NAME], template[LocalTemplateKey.REVISION]) master_ip = get_master_ip(vapp) task = vapp.set_metadata('GENERAL', 'READWRITE', 'cse.master.ip', master_ip) self.tenant_client.get_task_monitor().wait_for_status(task) self._update_task( TaskStatus.RUNNING, message=f"Creating {num_workers} node(s) for " f"{cluster_name}({cluster_id})") try: add_nodes(client=self.tenant_client, num_nodes=num_workers, node_type=NodeType.WORKER, org=org, vdc=vdc, vapp=vapp, catalog_name=catalog_name, template=template, network_name=network_name, num_cpu=num_cpu, memory_in_mb=mb_memory, storage_profile=storage_profile_name, ssh_key_filepath=ssh_key_filepath) except Exception as e: raise WorkerNodeCreationError("Error creating worker node:", str(e)) self._update_task( TaskStatus.RUNNING, message=f"Adding {num_workers} node(s) to " f"{cluster_name}({cluster_id})") vapp.reload() join_cluster(vapp, template[LocalTemplateKey.NAME], template[LocalTemplateKey.REVISION]) if enable_nfs: self._update_task( TaskStatus.RUNNING, message=f"Creating NFS node for " f"{cluster_name} ({cluster_id})") try: add_nodes(client=self.tenant_client, num_nodes=1, node_type=NodeType.NFS, org=org, vdc=vdc, vapp=vapp, catalog_name=catalog_name, template=template, network_name=network_name, num_cpu=num_cpu, memory_in_mb=mb_memory, storage_profile=storage_profile_name, ssh_key_filepath=ssh_key_filepath) except Exception as e: raise NFSNodeCreationError("Error creating NFS node:", str(e)) self._update_task( TaskStatus.SUCCESS, message=f"Created cluster {cluster_name} ({cluster_id})") except (MasterNodeCreationError, WorkerNodeCreationError, NFSNodeCreationError, ClusterJoiningError, ClusterInitializationError, ClusterOperationError) as e: if rollback: msg = f"Error creating cluster {cluster_name}. " \ f"Deleting cluster (rollback=True)" self._update_task(TaskStatus.RUNNING, message=msg) LOGGER.info(msg) try: cluster = get_cluster(self.tenant_client, cluster_name, cluster_id=cluster_id, org_name=org_name, ovdc_name=ovdc_name) self._delete_cluster(cluster_name=cluster_name, cluster_vdc_href=cluster['vdc_href']) except Exception: LOGGER.error(f"Failed to delete cluster {cluster_name}", exc_info=True) LOGGER.error(f"Error creating cluster {cluster_name}", exc_info=True) error_obj = error_to_json(e) stack_trace = ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) # noqa: E501 self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) # raising an exception here prints a stacktrace to server console except Exception as e: LOGGER.error(f"Unknown error creating cluster {cluster_name}", exc_info=True) error_obj = error_to_json(e) stack_trace = ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) # noqa: E501 self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) finally: self.logout_sys_admin_client()