def node_rollback(self, node_list): """Rollback for node creation failure. :param list node_list: faulty nodes to be deleted """ LOGGER.info(f"About to rollback nodes from cluster with name: " "{self.cluster_name}") LOGGER.info(f"Node list to be deleted:{node_list}") vapp = VApp(self.tenant_client, href=self.cluster['vapp_href']) template = self._get_template() try: server_config = get_server_runtime_config() delete_nodes_from_cluster(server_config, vapp, template, node_list, force=True) except Exception: LOGGER.warning("Couldn't delete node {node_list} from cluster:" "{self.cluster_name}") for vm_name in node_list: vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name)) try: vm.undeploy() except Exception: LOGGER.warning(f"Couldn't undeploy VM {vm_name}") vapp.delete_vms(node_list) LOGGER.info(f"Successfully deleted nodes: {node_list}")
def delete_nodes_thread(self): LOGGER.debug(f"About to delete nodes from cluster with name: " f"{self.cluster_name}") try: vapp = VApp(self.tenant_client, href=self.cluster['vapp_href']) template = self._get_template() self._update_task( TaskStatus.RUNNING, message=f"Deleting " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" node(s) from " f"{self.cluster_name}({self.cluster_id})") try: server_config = get_server_runtime_config() delete_nodes_from_cluster( server_config, vapp, template, self.req_spec.get(RequestKey.NODE_NAMES_LIST), self.req_spec.get(RequestKey.FORCE_DELETE)) except Exception: LOGGER.error(f"Couldn't delete node " f"{self.req_spec.get(RequestKey.NODE_NAMES_LIST)}" f" from cluster:{self.cluster_name}") self._update_task( TaskStatus.RUNNING, message=f"Undeploying " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" node(s) for {self.cluster_name}({self.cluster_id})") for vm_name in self.req_spec.get(RequestKey.NODE_NAMES_LIST): vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name)) try: task = vm.undeploy() self.tenant_client.get_task_monitor().wait_for_status(task) except Exception: LOGGER.warning(f"Couldn't undeploy VM {vm_name}") self._update_task( TaskStatus.RUNNING, message=f"Deleting " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" VM(s) for {self.cluster_name}({self.cluster_id})") task = vapp.delete_vms(self.req_spec.get(RequestKey.NODE_NAMES_LIST)) # noqa: E501 self.tenant_client.get_task_monitor().wait_for_status(task) self._update_task( TaskStatus.SUCCESS, message=f"Deleted " f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}" f" node(s) to cluster " f"{self.cluster_name}({self.cluster_id})") except Exception as e: LOGGER.error(traceback.format_exc()) error_obj = error_to_json(e) stack_trace = \ ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) self._update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501 stack_trace=stack_trace) finally: self._disconnect_sys_admin()
def get_sys_admin_client(): server_config = get_server_runtime_config() if not server_config['vcd']['verify']: SERVER_LOGGER.warning("InsecureRequestWarning: Unverified HTTPS " "request is being made. Adding certificate " "verification is strongly advised.") requests.packages.urllib3.disable_warnings() log_filename = None log_wire = str_to_bool(server_config['service'].get('log_wire')) if log_wire: log_filename = SERVER_DEBUG_WIRELOG_FILEPATH client = vcd_client.Client( uri=server_config['vcd']['host'], api_version=server_config['vcd']['api_version'], verify_ssl_certs=server_config['vcd']['verify'], log_file=log_filename, log_requests=log_wire, log_headers=log_wire, log_bodies=log_wire) credentials = vcd_client.BasicLoginCredentials( server_config['vcd']['username'], SYSTEM_ORG_NAME, server_config['vcd']['password']) client.set_credentials(credentials) return client
def record_user_action(cse_operation, status=OperationStatus.SUCCESS, message=None, telemetry_settings=None): """Record CSE user action information in telemetry server. No exceptions should be leaked. Catch all exceptions and log them. :param CseOperation cse_operation: :param OperationStatus status: SUCCESS/FAILURE of the user action :param str message: any information about failure or custom message :param dict telemetry_settings: telemetry section CSE config->service """ try: if not telemetry_settings: server_config = get_server_runtime_config() telemetry_settings = None if not server_config else \ server_config.get('service', {}).get('telemetry') if telemetry_settings: if telemetry_settings.get('enable'): payload = get_payload_for_user_action(cse_operation, status, message) # noqa: E501 _send_data_to_telemetry_server(payload, telemetry_settings) else: LOGGER.debug('No telemetry settings found.') except Exception as err: LOGGER.warning(f"Error in recording user action information:{str(err)}" ) # noqa: E501
def node_rollback(self, node_list): """Implements rollback for node creation failure :param list node_list: faulty nodes to be deleted """ LOGGER.info('About to rollback nodes from cluster with name: %s' % self.cluster_name) LOGGER.info('Node list to be deleted:%s' % node_list) vapp = VApp(self.client_tenant, href=self.cluster['vapp_href']) template = self.get_template() try: delete_nodes_from_cluster(self.config, vapp, template, node_list, force=True) except Exception: LOGGER.warning("Couldn't delete node %s from cluster:%s" % (node_list, self.cluster_name)) for vm_name in node_list: vm = VM(self.client_tenant, resource=vapp.get_vm(vm_name)) try: vm.undeploy() except Exception: LOGGER.warning("Couldn't undeploy VM %s" % vm_name) vapp.delete_vms(node_list) LOGGER.info('Successfully deleted nodes: %s' % node_list)
def on_connection_closed(self, connection, reply_code, reply_text): self._channel = None if self._closing: self._connection.ioloop.stop() else: LOGGER.warning(f"Connection closed, reopening in 5 seconds: " f"({reply_code}) {reply_text}") self._connection.add_timeout(5, self.reconnect)
def delete_nodes_thread(self): LOGGER.debug(f"About to delete nodes from cluster with name: " f"{self.cluster_name}") try: vapp = VApp(self.tenant_client, href=self.cluster['vapp_href']) template = self.get_template() self.update_task( TaskStatus.RUNNING, message=f"Deleting {len(self.req_spec['nodes'])} node(s) from " f"{self.cluster_name}({self.cluster_id})") try: server_config = get_server_runtime_config() delete_nodes_from_cluster(server_config, vapp, template, self.req_spec['nodes'], self.req_spec['force']) except Exception: LOGGER.error(f"Couldn't delete node {self.req_spec['nodes']} " f"from cluster:{self.cluster_name}") self.update_task( TaskStatus.RUNNING, message=f"Undeploying {len(self.req_spec['nodes'])} node(s) " f"for {self.cluster_name}({self.cluster_id})") for vm_name in self.req_spec['nodes']: vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name)) try: task = vm.undeploy() self.tenant_client.get_task_monitor().wait_for_status(task) except Exception: LOGGER.warning(f"Couldn't undeploy VM {vm_name}") self.update_task( TaskStatus.RUNNING, message=f"Deleting {len(self.req_spec['nodes'])} VM(s) for " f"{self.cluster_name}({self.cluster_id})") task = vapp.delete_vms(self.req_spec['nodes']) self.tenant_client.get_task_monitor().wait_for_status(task) self.update_task( TaskStatus.SUCCESS, message=f"Deleted {len(self.req_spec['nodes'])} node(s) to " f"cluster {self.cluster_name}({self.cluster_id})") except Exception as e: LOGGER.error(traceback.format_exc()) error_obj = error_to_json(e) stack_trace = ''.join(error_obj[ERROR_MESSAGE][ERROR_STACKTRACE]) self.update_task( TaskStatus.ERROR, error_message=error_obj[ERROR_MESSAGE][ERROR_DESCRIPTION], stack_trace=stack_trace) finally: self._disconnect_sys_admin()
def _connect_sysadmin(self): if not self.verify: LOGGER.warning('InsecureRequestWarning: ' 'Unverified HTTPS request is being made. ' 'Adding certificate verification is strongly ' 'advised.') requests.packages.urllib3.disable_warnings() self.client_sysadmin = Client(uri=self.host, api_version=self.version, verify_ssl_certs=self.verify, log_headers=True, log_bodies=True) credentials = BasicLoginCredentials(self.username, SYSTEM_ORG_NAME, self.password) self.client_sysadmin.set_credentials(credentials)
def _delete_nodes(self, *args, cluster_name, cluster_vapp_href, node_names_list): LOGGER.debug(f"About to delete nodes {node_names_list} " f"from cluster {cluster_name}") vapp = VApp(self.tenant_client, href=cluster_vapp_href) try: delete_nodes_from_cluster(vapp, node_names_list) except Exception: LOGGER.error(f"Couldn't delete node {node_names_list} " f"from cluster:{cluster_name}") for vm_name in node_names_list: vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name)) try: task = vm.undeploy() self.tenant_client.get_task_monitor().wait_for_status(task) except Exception: LOGGER.warning(f"Couldn't undeploy VM {vm_name}") task = vapp.delete_vms(node_names_list) self.tenant_client.get_task_monitor().wait_for_status(task)
def get_sys_admin_client(self): if self.config is not None: if not self.config['vcd']['verify']: LOGGER.warning('InsecureRequestWarning: Unverified HTTPS ' 'request is being made. Adding certificate ' 'verification is strongly advised.') requests.packages.urllib3.disable_warnings() client = Client(uri=self.config['vcd']['host'], api_version=self.config['vcd']['api_version'], verify_ssl_certs=self.config['vcd']['verify'], log_file=SERVER_DEBUG_WIRELOG_FILEPATH, log_requests=True, log_headers=True, log_bodies=True) credentials = BasicLoginCredentials(self.config['vcd']['username'], SYSTEM_ORG_NAME, self.config['vcd']['password']) client.set_credentials(credentials) return client return None
def delete_nodes_thread(self): LOGGER.debug('about to delete nodes from cluster with name: %s', self.cluster_name) try: vapp = VApp(self.client_tenant, href=self.cluster['vapp_href']) template = self.get_template() self.update_task( TaskStatus.RUNNING, message='Deleting %s node(s) from %s(%s)' % (len(self.body['nodes']), self.cluster_name, self.cluster_id)) try: delete_nodes_from_cluster(self.config, vapp, template, self.body['nodes'], self.body['force']) except Exception: LOGGER.error("Couldn't delete node %s from cluster:%s" % (self.body['nodes'], self.cluster_name)) self.update_task( TaskStatus.RUNNING, message='Undeploying %s node(s) for %s(%s)' % (len(self.body['nodes']), self.cluster_name, self.cluster_id)) for vm_name in self.body['nodes']: vm = VM(self.client_tenant, resource=vapp.get_vm(vm_name)) try: task = vm.undeploy() self.client_tenant.get_task_monitor().wait_for_status(task) except Exception as e: LOGGER.warning('couldn\'t undeploy VM %s' % vm_name) self.update_task( TaskStatus.RUNNING, message='Deleting %s VM(s) for %s(%s)' % (len(self.body['nodes']), self.cluster_name, self.cluster_id)) task = vapp.delete_vms(self.body['nodes']) self.client_tenant.get_task_monitor().wait_for_status(task) self.update_task( TaskStatus.SUCCESS, message='Deleted %s node(s) to cluster %s(%s)' % (len(self.body['nodes']), self.cluster_name, self.cluster_id)) except Exception as e: LOGGER.error(traceback.format_exc()) self.update_task(TaskStatus.ERROR, error_message=str(e))
def connect_vcd_user_via_token(vcd_uri, headers, verify_ssl_certs=True): if not verify_ssl_certs: LOGGER.warning("InsecureRequestWarning: Unverified HTTPS request is " "being made. Adding certificate verification is " "strongly advised.") requests.packages.urllib3.disable_warnings() token = headers.get('x-vcloud-authorization') accept_header = headers.get('Accept') version = accept_header.split('version=')[1] client_tenant = Client(uri=vcd_uri, api_version=version, verify_ssl_certs=verify_ssl_certs, log_file=SERVER_DEBUG_WIRELOG_FILEPATH, log_requests=True, log_headers=True, log_bodies=True) session = client_tenant.rehydrate_from_token(token) return ( client_tenant, session, )
def record_user_action_details(cse_operation, cse_params, telemetry_settings=None): """Record CSE user operation details in telemetry server. No exception should be leaked. Catch all exceptions and log them. :param CseOperation cse_operation: CSE operation information :param dict cse_params: CSE operation parameters :param dict telemetry_settings: telemetry section of config->service """ try: if not telemetry_settings: telemetry_settings = get_server_runtime_config()['service'][ 'telemetry'] # noqa: E501 if telemetry_settings['enable']: payload = OPERATION_TO_PAYLOAD_GENERATOR[cse_operation](cse_params) _send_data_to_telemetry_server(payload, telemetry_settings) except Exception as err: LOGGER.warning(f"Error in recording CSE operation details :{str(err)}" ) # noqa: E501
def on_channel_closed(self, channel, reply_code, reply_text): LOGGER.warning('Channel %i was closed: (%s) %s', channel, reply_code, reply_text) self._connection.close()
def on_channel_closed(self, channel, reply_code, reply_text): LOGGER.warning(f"Channel {channel} was closed: ({reply_code}) " f"{reply_text}") self._connection.close()