def node_rollback(self, node_list):
"""Rollback for node creation failure.
:param list node_list: faulty nodes to be deleted
"""
LOGGER.info(f"About to rollback nodes from cluster with name: "
"{self.cluster_name}")
LOGGER.info(f"Node list to be deleted:{node_list}")
vapp = VApp(self.tenant_client, href=self.cluster['vapp_href'])
template = self._get_template()
try:
server_config = get_server_runtime_config()
delete_nodes_from_cluster(server_config, vapp, template,
node_list, force=True)
except Exception:
LOGGER.warning("Couldn't delete node {node_list} from cluster:"
"{self.cluster_name}")
for vm_name in node_list:
vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name))
try:
vm.undeploy()
except Exception:
LOGGER.warning(f"Couldn't undeploy VM {vm_name}")
vapp.delete_vms(node_list)
LOGGER.info(f"Successfully deleted nodes: {node_list}")
def delete_nodes_thread(self):
LOGGER.debug(f"About to delete nodes from cluster with name: "
f"{self.cluster_name}")
try:
vapp = VApp(self.tenant_client, href=self.cluster['vapp_href'])
template = self._get_template()
self._update_task(
TaskStatus.RUNNING,
message=f"Deleting "
f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}"
f" node(s) from "
f"{self.cluster_name}({self.cluster_id})")
try:
server_config = get_server_runtime_config()
delete_nodes_from_cluster(
server_config,
vapp,
template,
self.req_spec.get(RequestKey.NODE_NAMES_LIST),
self.req_spec.get(RequestKey.FORCE_DELETE))
except Exception:
LOGGER.error(f"Couldn't delete node "
f"{self.req_spec.get(RequestKey.NODE_NAMES_LIST)}"
f" from cluster:{self.cluster_name}")
self._update_task(
TaskStatus.RUNNING,
message=f"Undeploying "
f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}"
f" node(s) for {self.cluster_name}({self.cluster_id})")
for vm_name in self.req_spec.get(RequestKey.NODE_NAMES_LIST):
vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name))
try:
task = vm.undeploy()
self.tenant_client.get_task_monitor().wait_for_status(task)
except Exception:
LOGGER.warning(f"Couldn't undeploy VM {vm_name}")
self._update_task(
TaskStatus.RUNNING,
message=f"Deleting "
f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}"
f" VM(s) for {self.cluster_name}({self.cluster_id})")
task = vapp.delete_vms(self.req_spec.get(RequestKey.NODE_NAMES_LIST)) # noqa: E501
self.tenant_client.get_task_monitor().wait_for_status(task)
self._update_task(
TaskStatus.SUCCESS,
message=f"Deleted "
f"{len(self.req_spec.get(RequestKey.NODE_NAMES_LIST))}"
f" node(s) to cluster "
f"{self.cluster_name}({self.cluster_id})")
except Exception as e:
LOGGER.error(traceback.format_exc())
error_obj = error_to_json(e)
stack_trace = \
''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY])
self._update_task(
TaskStatus.ERROR,
error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501
stack_trace=stack_trace)
finally:
self._disconnect_sys_admin()
def get_sys_admin_client():
server_config = get_server_runtime_config()
if not server_config['vcd']['verify']:
SERVER_LOGGER.warning("InsecureRequestWarning: Unverified HTTPS "
"request is being made. Adding certificate "
"verification is strongly advised.")
requests.packages.urllib3.disable_warnings()
log_filename = None
log_wire = str_to_bool(server_config['service'].get('log_wire'))
if log_wire:
log_filename = SERVER_DEBUG_WIRELOG_FILEPATH
client = vcd_client.Client(
uri=server_config['vcd']['host'],
api_version=server_config['vcd']['api_version'],
verify_ssl_certs=server_config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = vcd_client.BasicLoginCredentials(
server_config['vcd']['username'],
SYSTEM_ORG_NAME,
server_config['vcd']['password'])
client.set_credentials(credentials)
return client
def record_user_action(cse_operation,
status=OperationStatus.SUCCESS,
message=None,
telemetry_settings=None):
"""Record CSE user action information in telemetry server.
No exceptions should be leaked. Catch all exceptions and log them.
:param CseOperation cse_operation:
:param OperationStatus status: SUCCESS/FAILURE of the user action
:param str message: any information about failure or custom message
:param dict telemetry_settings: telemetry section CSE config->service
"""
try:
if not telemetry_settings:
server_config = get_server_runtime_config()
telemetry_settings = None if not server_config else \
server_config.get('service', {}).get('telemetry')
if telemetry_settings:
if telemetry_settings.get('enable'):
payload = get_payload_for_user_action(cse_operation, status,
message) # noqa: E501
_send_data_to_telemetry_server(payload, telemetry_settings)
else:
LOGGER.debug('No telemetry settings found.')
except Exception as err:
LOGGER.warning(f"Error in recording user action information:{str(err)}"
) # noqa: E501
def node_rollback(self, node_list):
"""Implements rollback for node creation failure
:param list node_list: faulty nodes to be deleted
"""
LOGGER.info('About to rollback nodes from cluster with name: %s' %
self.cluster_name)
LOGGER.info('Node list to be deleted:%s' % node_list)
vapp = VApp(self.client_tenant, href=self.cluster['vapp_href'])
template = self.get_template()
try:
delete_nodes_from_cluster(self.config,
vapp,
template,
node_list,
force=True)
except Exception:
LOGGER.warning("Couldn't delete node %s from cluster:%s" %
(node_list, self.cluster_name))
for vm_name in node_list:
vm = VM(self.client_tenant, resource=vapp.get_vm(vm_name))
try:
vm.undeploy()
except Exception:
LOGGER.warning("Couldn't undeploy VM %s" % vm_name)
vapp.delete_vms(node_list)
LOGGER.info('Successfully deleted nodes: %s' % node_list)
def on_connection_closed(self, connection, reply_code, reply_text):
self._channel = None
if self._closing:
self._connection.ioloop.stop()
else:
LOGGER.warning(f"Connection closed, reopening in 5 seconds: "
f"({reply_code}) {reply_text}")
self._connection.add_timeout(5, self.reconnect)
def delete_nodes_thread(self):
LOGGER.debug(f"About to delete nodes from cluster with name: "
f"{self.cluster_name}")
try:
vapp = VApp(self.tenant_client, href=self.cluster['vapp_href'])
template = self.get_template()
self.update_task(
TaskStatus.RUNNING,
message=f"Deleting {len(self.req_spec['nodes'])} node(s) from "
f"{self.cluster_name}({self.cluster_id})")
try:
server_config = get_server_runtime_config()
delete_nodes_from_cluster(server_config, vapp, template,
self.req_spec['nodes'],
self.req_spec['force'])
except Exception:
LOGGER.error(f"Couldn't delete node {self.req_spec['nodes']} "
f"from cluster:{self.cluster_name}")
self.update_task(
TaskStatus.RUNNING,
message=f"Undeploying {len(self.req_spec['nodes'])} node(s) "
f"for {self.cluster_name}({self.cluster_id})")
for vm_name in self.req_spec['nodes']:
vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name))
try:
task = vm.undeploy()
self.tenant_client.get_task_monitor().wait_for_status(task)
except Exception:
LOGGER.warning(f"Couldn't undeploy VM {vm_name}")
self.update_task(
TaskStatus.RUNNING,
message=f"Deleting {len(self.req_spec['nodes'])} VM(s) for "
f"{self.cluster_name}({self.cluster_id})")
task = vapp.delete_vms(self.req_spec['nodes'])
self.tenant_client.get_task_monitor().wait_for_status(task)
self.update_task(
TaskStatus.SUCCESS,
message=f"Deleted {len(self.req_spec['nodes'])} node(s) to "
f"cluster {self.cluster_name}({self.cluster_id})")
except Exception as e:
LOGGER.error(traceback.format_exc())
error_obj = error_to_json(e)
stack_trace = ''.join(error_obj[ERROR_MESSAGE][ERROR_STACKTRACE])
self.update_task(
TaskStatus.ERROR,
error_message=error_obj[ERROR_MESSAGE][ERROR_DESCRIPTION],
stack_trace=stack_trace)
finally:
self._disconnect_sys_admin()
def _connect_sysadmin(self):
if not self.verify:
LOGGER.warning('InsecureRequestWarning: '
'Unverified HTTPS request is being made. '
'Adding certificate verification is strongly '
'advised.')
requests.packages.urllib3.disable_warnings()
self.client_sysadmin = Client(uri=self.host,
api_version=self.version,
verify_ssl_certs=self.verify,
log_headers=True,
log_bodies=True)
credentials = BasicLoginCredentials(self.username, SYSTEM_ORG_NAME,
self.password)
self.client_sysadmin.set_credentials(credentials)
def _delete_nodes(self, *args,
cluster_name, cluster_vapp_href, node_names_list):
LOGGER.debug(f"About to delete nodes {node_names_list} "
f"from cluster {cluster_name}")
vapp = VApp(self.tenant_client, href=cluster_vapp_href)
try:
delete_nodes_from_cluster(vapp, node_names_list)
except Exception:
LOGGER.error(f"Couldn't delete node {node_names_list} "
f"from cluster:{cluster_name}")
for vm_name in node_names_list:
vm = VM(self.tenant_client, resource=vapp.get_vm(vm_name))
try:
task = vm.undeploy()
self.tenant_client.get_task_monitor().wait_for_status(task)
except Exception:
LOGGER.warning(f"Couldn't undeploy VM {vm_name}")
task = vapp.delete_vms(node_names_list)
self.tenant_client.get_task_monitor().wait_for_status(task)
def get_sys_admin_client(self):
if self.config is not None:
if not self.config['vcd']['verify']:
LOGGER.warning('InsecureRequestWarning: Unverified HTTPS '
'request is being made. Adding certificate '
'verification is strongly advised.')
requests.packages.urllib3.disable_warnings()
client = Client(uri=self.config['vcd']['host'],
api_version=self.config['vcd']['api_version'],
verify_ssl_certs=self.config['vcd']['verify'],
log_file=SERVER_DEBUG_WIRELOG_FILEPATH,
log_requests=True,
log_headers=True,
log_bodies=True)
credentials = BasicLoginCredentials(self.config['vcd']['username'],
SYSTEM_ORG_NAME,
self.config['vcd']['password'])
client.set_credentials(credentials)
return client
return None
def delete_nodes_thread(self):
LOGGER.debug('about to delete nodes from cluster with name: %s',
self.cluster_name)
try:
vapp = VApp(self.client_tenant, href=self.cluster['vapp_href'])
template = self.get_template()
self.update_task(
TaskStatus.RUNNING,
message='Deleting %s node(s) from %s(%s)' %
(len(self.body['nodes']), self.cluster_name, self.cluster_id))
try:
delete_nodes_from_cluster(self.config, vapp, template,
self.body['nodes'],
self.body['force'])
except Exception:
LOGGER.error("Couldn't delete node %s from cluster:%s" %
(self.body['nodes'], self.cluster_name))
self.update_task(
TaskStatus.RUNNING,
message='Undeploying %s node(s) for %s(%s)' %
(len(self.body['nodes']), self.cluster_name, self.cluster_id))
for vm_name in self.body['nodes']:
vm = VM(self.client_tenant, resource=vapp.get_vm(vm_name))
try:
task = vm.undeploy()
self.client_tenant.get_task_monitor().wait_for_status(task)
except Exception as e:
LOGGER.warning('couldn\'t undeploy VM %s' % vm_name)
self.update_task(
TaskStatus.RUNNING,
message='Deleting %s VM(s) for %s(%s)' %
(len(self.body['nodes']), self.cluster_name, self.cluster_id))
task = vapp.delete_vms(self.body['nodes'])
self.client_tenant.get_task_monitor().wait_for_status(task)
self.update_task(
TaskStatus.SUCCESS,
message='Deleted %s node(s) to cluster %s(%s)' %
(len(self.body['nodes']), self.cluster_name, self.cluster_id))
except Exception as e:
LOGGER.error(traceback.format_exc())
self.update_task(TaskStatus.ERROR, error_message=str(e))
def connect_vcd_user_via_token(vcd_uri, headers, verify_ssl_certs=True):
if not verify_ssl_certs:
LOGGER.warning("InsecureRequestWarning: Unverified HTTPS request is "
"being made. Adding certificate verification is "
"strongly advised.")
requests.packages.urllib3.disable_warnings()
token = headers.get('x-vcloud-authorization')
accept_header = headers.get('Accept')
version = accept_header.split('version=')[1]
client_tenant = Client(uri=vcd_uri,
api_version=version,
verify_ssl_certs=verify_ssl_certs,
log_file=SERVER_DEBUG_WIRELOG_FILEPATH,
log_requests=True,
log_headers=True,
log_bodies=True)
session = client_tenant.rehydrate_from_token(token)
return (
client_tenant,
session,
)
def record_user_action_details(cse_operation,
cse_params,
telemetry_settings=None):
"""Record CSE user operation details in telemetry server.
No exception should be leaked. Catch all exceptions and log them.
:param CseOperation cse_operation: CSE operation information
:param dict cse_params: CSE operation parameters
:param dict telemetry_settings: telemetry section of config->service
"""
try:
if not telemetry_settings:
telemetry_settings = get_server_runtime_config()['service'][
'telemetry'] # noqa: E501
if telemetry_settings['enable']:
payload = OPERATION_TO_PAYLOAD_GENERATOR[cse_operation](cse_params)
_send_data_to_telemetry_server(payload, telemetry_settings)
except Exception as err:
LOGGER.warning(f"Error in recording CSE operation details :{str(err)}"
) # noqa: E501
def on_channel_closed(self, channel, reply_code, reply_text):
LOGGER.warning('Channel %i was closed: (%s) %s', channel, reply_code,
reply_text)
self._connection.close()
def on_channel_closed(self, channel, reply_code, reply_text):
LOGGER.warning(f"Channel {channel} was closed: ({reply_code}) "
f"{reply_text}")
self._connection.close()
