def convert_cluster(ctx, config_file_name, skip_config_decryption,
cluster_name, admin_password, org_name, vdc_name,
skip_wait_for_gc):
if skip_config_decryption:
decryption_password = None
else:
decryption_password = os.getenv('CSE_CONFIG_PASSWORD') or prompt_text(
PASSWORD_FOR_CONFIG_DECRYPTION_MSG, color='green', hide_input=True)
try:
check_python_version()
except Exception as err:
click.secho(str(err), fg='red')
sys.exit(1)
client = None
try:
console_message_printer = ConsoleMessagePrinter()
config = get_validated_config(
config_file_name,
skip_config_decryption=skip_config_decryption,
decryption_password=decryption_password,
msg_update_callback=console_message_printer)
log_filename = None
log_wire = str_to_bool(config['service'].get('log_wire'))
if log_wire:
log_filename = 'cluster_convert_wire.log'
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
console_message_printer.general(msg)
cluster_records = get_all_clusters(client=client,
cluster_name=cluster_name,
org_name=org_name,
ovdc_name=vdc_name)
if len(cluster_records) == 0:
console_message_printer.info(f"No clusters were found.")
return
vms = []
for cluster in cluster_records:
console_message_printer.info(
f"Processing cluster '{cluster['name']}'.")
vapp_href = cluster['vapp_href']
vapp = VApp(client, href=vapp_href)
# this step removes the old 'cse.template' metadata and adds
# cse.template.name and cse.template.revision metadata
# using hard-coded values taken from github history
console_message_printer.info("Processing metadata of cluster.")
metadata_dict = metadata_to_dict(vapp.get_metadata())
old_template_name = metadata_dict.get(
ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME
) # noqa: E501
new_template_name = None
cse_version = metadata_dict.get(ClusterMetadataKey.CSE_VERSION)
if old_template_name:
console_message_printer.info(
"Determining k8s version on cluster.")
if 'photon' in old_template_name:
new_template_name = 'photon-v2'
if cse_version in ('1.0.0'):
new_template_name += '_k8s-1.8_weave-2.0.5'
elif cse_version in ('1.1.0', '1.2.0', '1.2.1', '1.2.2',
'1.2.3', '1.2.4'): # noqa: E501
new_template_name += '_k8s-1.9_weave-2.3.0'
elif cse_version in (
'1.2.5',
'1.2.6',
'1.2.7',
): # noqa: E501
new_template_name += '_k8s-1.10_weave-2.3.0'
elif cse_version in ('2.0.0'):
new_template_name += '_k8s-1.12_weave-2.3.0'
elif 'ubuntu' in old_template_name:
new_template_name = 'ubuntu-16.04'
if cse_version in ('1.0.0'):
new_template_name += '_k8s-1.9_weave-2.1.3'
elif cse_version in ('1.1.0', '1.2.0', '1.2.1', '1.2.2',
'1.2.3', '1.2.4', '1.2.5', '1.2.6',
'1.2.7'): # noqa: E501
new_template_name += '_k8s-1.10_weave-2.3.0'
elif cse_version in ('2.0.0'):
new_template_name += '_k8s-1.13_weave-2.3.0'
if new_template_name:
console_message_printer.info("Updating metadata of cluster.")
task = vapp.remove_metadata(
ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME
) # noqa: E501
client.get_task_monitor().wait_for_success(task)
new_metadata_to_add = {
ClusterMetadataKey.TEMPLATE_NAME: new_template_name,
ClusterMetadataKey.TEMPLATE_REVISION: 0
}
task = vapp.set_multiple_metadata(new_metadata_to_add)
client.get_task_monitor().wait_for_success(task)
# this step uses hard-coded data from the newly updated
# cse.template.name and cse.template.revision metadata fields as
# well as github history to add [cse.os, cse.docker.version,
# cse.kubernetes, cse.kubernetes.version, cse.cni, cse.cni.version]
# to the clusters
vapp.reload()
metadata_dict = metadata_to_dict(vapp.get_metadata())
template_name = metadata_dict.get(ClusterMetadataKey.TEMPLATE_NAME)
template_revision = str(
metadata_dict.get(ClusterMetadataKey.TEMPLATE_REVISION,
'0')) # noqa: E501
if template_name:
k8s_version, docker_version = get_k8s_and_docker_versions(
template_name,
template_revision=template_revision,
cse_version=cse_version) # noqa: E501
tokens = template_name.split('_')
new_metadata = {
ClusterMetadataKey.OS: tokens[0],
ClusterMetadataKey.DOCKER_VERSION: docker_version,
ClusterMetadataKey.KUBERNETES: 'upstream',
ClusterMetadataKey.KUBERNETES_VERSION: k8s_version,
ClusterMetadataKey.CNI: tokens[2].split('-')[0],
ClusterMetadataKey.CNI_VERSION: tokens[2].split('-')[1],
}
task = vapp.set_multiple_metadata(new_metadata)
client.get_task_monitor().wait_for_success(task)
console_message_printer.general(
"Finished processing metadata of cluster.")
reset_admin_pw = False
vm_resources = vapp.get_all_vms()
for vm_resource in vm_resources:
try:
vapp.get_admin_password(vm_resource.get('name'))
except EntityNotFoundException:
reset_admin_pw = True
break
if reset_admin_pw:
try:
console_message_printer.info(
f"Undeploying the vApp '{cluster['name']}'")
task = vapp.undeploy()
client.get_task_monitor().wait_for_success(task)
console_message_printer.general(
"Successfully undeployed the vApp.")
except Exception as err:
console_message_printer.error(str(err))
for vm_resource in vm_resources:
console_message_printer.info(
f"Processing vm '{vm_resource.get('name')}'.")
vm = VM(client, href=vm_resource.get('href'))
vms.append(vm)
console_message_printer.info("Updating vm admin password")
task = vm.update_guest_customization_section(
enabled=True,
admin_password_enabled=True,
admin_password_auto=not admin_password,
admin_password=admin_password,
)
client.get_task_monitor().wait_for_success(task)
console_message_printer.general("Successfully updated vm")
console_message_printer.info("Deploying vm.")
task = vm.power_on_and_force_recustomization()
client.get_task_monitor().wait_for_success(task)
console_message_printer.general("Successfully deployed vm")
console_message_printer.info("Deploying cluster")
task = vapp.deploy(power_on=True)
client.get_task_monitor().wait_for_success(task)
console_message_printer.general(
"Successfully deployed cluster") # noqa: E501
console_message_printer.general(
f"Successfully processed cluster '{cluster['name']}'")
if skip_wait_for_gc:
return
while True:
to_remove = []
for vm in vms:
status = vm.get_guest_customization_status()
if status != 'GC_PENDING':
to_remove.append(vm)
for vm in to_remove:
vms.remove(vm)
console_message_printer.info(
f"Waiting on guest customization to finish on {len(vms)} vms.")
if not len(vms) == 0:
time.sleep(5)
else:
break
except cryptography.fernet.InvalidToken:
click.secho(CONFIG_DECRYPTION_ERROR_MSG, fg='red')
except Exception as err:
click.secho(str(err), fg='red')
finally:
if client:
client.logout()
def _create_cluster_async(self, *args,
org_name, ovdc_name, cluster_name, cluster_id,
template_name, template_revision, num_workers,
network_name, num_cpu, mb_memory,
storage_profile_name, ssh_key_filepath,
enable_nfs, rollback):
org = vcd_utils.get_org(self.tenant_client, org_name=org_name)
vdc = vcd_utils.get_vdc(
self.tenant_client, vdc_name=ovdc_name, org=org)
LOGGER.debug(f"About to create cluster {cluster_name} on {ovdc_name}"
f" with {num_workers} worker nodes, "
f"storage profile={storage_profile_name}")
try:
self._update_task(
TaskStatus.RUNNING,
message=f"Creating cluster vApp {cluster_name}({cluster_id})")
try:
vapp_resource = \
vdc.create_vapp(cluster_name,
description=f"cluster {cluster_name}",
network=network_name,
fence_mode='bridged')
except Exception as e:
msg = f"Error while creating vApp: {e}"
LOGGER.debug(str(e))
raise ClusterOperationError(msg)
self.tenant_client.get_task_monitor().wait_for_status(vapp_resource.Tasks.Task[0]) # noqa: E501
template = get_template(template_name, template_revision)
tags = {
ClusterMetadataKey.CLUSTER_ID: cluster_id,
ClusterMetadataKey.CSE_VERSION: pkg_resources.require('container-service-extension')[0].version, # noqa: E501
ClusterMetadataKey.TEMPLATE_NAME: template[LocalTemplateKey.NAME], # noqa: E501
ClusterMetadataKey.TEMPLATE_REVISION: template[LocalTemplateKey.REVISION] # noqa: E501
}
vapp = VApp(self.tenant_client, href=vapp_resource.get('href'))
task = vapp.set_multiple_metadata(tags)
self.tenant_client.get_task_monitor().wait_for_status(task)
self._update_task(
TaskStatus.RUNNING,
message=f"Creating master node for "
f"{cluster_name} ({cluster_id})")
vapp.reload()
server_config = utils.get_server_runtime_config()
catalog_name = server_config['broker']['catalog']
try:
add_nodes(client=self.tenant_client,
num_nodes=1,
node_type=NodeType.MASTER,
org=org,
vdc=vdc,
vapp=vapp,
catalog_name=catalog_name,
template=template,
network_name=network_name,
num_cpu=num_cpu,
memory_in_mb=mb_memory,
storage_profile=storage_profile_name,
ssh_key_filepath=ssh_key_filepath)
except Exception as e:
raise MasterNodeCreationError("Error adding master node:",
str(e))
self._update_task(
TaskStatus.RUNNING,
message=f"Initializing cluster {cluster_name} ({cluster_id})")
vapp.reload()
init_cluster(vapp, template[LocalTemplateKey.NAME],
template[LocalTemplateKey.REVISION])
master_ip = get_master_ip(vapp)
task = vapp.set_metadata('GENERAL', 'READWRITE', 'cse.master.ip',
master_ip)
self.tenant_client.get_task_monitor().wait_for_status(task)
self._update_task(
TaskStatus.RUNNING,
message=f"Creating {num_workers} node(s) for "
f"{cluster_name}({cluster_id})")
try:
add_nodes(client=self.tenant_client,
num_nodes=num_workers,
node_type=NodeType.WORKER,
org=org,
vdc=vdc,
vapp=vapp,
catalog_name=catalog_name,
template=template,
network_name=network_name,
num_cpu=num_cpu,
memory_in_mb=mb_memory,
storage_profile=storage_profile_name,
ssh_key_filepath=ssh_key_filepath)
except Exception as e:
raise WorkerNodeCreationError("Error creating worker node:",
str(e))
self._update_task(
TaskStatus.RUNNING,
message=f"Adding {num_workers} node(s) to "
f"{cluster_name}({cluster_id})")
vapp.reload()
join_cluster(vapp, template[LocalTemplateKey.NAME],
template[LocalTemplateKey.REVISION])
if enable_nfs:
self._update_task(
TaskStatus.RUNNING,
message=f"Creating NFS node for "
f"{cluster_name} ({cluster_id})")
try:
add_nodes(client=self.tenant_client,
num_nodes=1,
node_type=NodeType.NFS,
org=org,
vdc=vdc,
vapp=vapp,
catalog_name=catalog_name,
template=template,
network_name=network_name,
num_cpu=num_cpu,
memory_in_mb=mb_memory,
storage_profile=storage_profile_name,
ssh_key_filepath=ssh_key_filepath)
except Exception as e:
raise NFSNodeCreationError("Error creating NFS node:",
str(e))
self._update_task(
TaskStatus.SUCCESS,
message=f"Created cluster {cluster_name} ({cluster_id})")
except (MasterNodeCreationError, WorkerNodeCreationError,
NFSNodeCreationError, ClusterJoiningError,
ClusterInitializationError, ClusterOperationError) as e:
if rollback:
msg = f"Error creating cluster {cluster_name}. " \
f"Deleting cluster (rollback=True)"
self._update_task(TaskStatus.RUNNING, message=msg)
LOGGER.info(msg)
try:
cluster = get_cluster(self.tenant_client,
cluster_name,
cluster_id=cluster_id,
org_name=org_name,
ovdc_name=ovdc_name)
self._delete_cluster(cluster_name=cluster_name,
cluster_vdc_href=cluster['vdc_href'])
except Exception:
LOGGER.error(f"Failed to delete cluster {cluster_name}",
exc_info=True)
LOGGER.error(f"Error creating cluster {cluster_name}",
exc_info=True)
error_obj = error_to_json(e)
stack_trace = ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) # noqa: E501
self._update_task(
TaskStatus.ERROR,
error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501
stack_trace=stack_trace)
# raising an exception here prints a stacktrace to server console
except Exception as e:
LOGGER.error(f"Unknown error creating cluster {cluster_name}",
exc_info=True)
error_obj = error_to_json(e)
stack_trace = ''.join(error_obj[ERROR_MESSAGE_KEY][ERROR_STACKTRACE_KEY]) # noqa: E501
self._update_task(
TaskStatus.ERROR,
error_message=error_obj[ERROR_MESSAGE_KEY][ERROR_DESCRIPTION_KEY], # noqa: E501
stack_trace=stack_trace)
finally:
self.logout_sys_admin_client()
def convert_cluster(ctx, config_file_name, cluster_name, password, org_name,
vdc_name, skip_wait_for_gc):
try:
check_python_version()
except Exception as err:
click.secho(str(err), fg='red')
sys.exit(1)
client = None
try:
console_message_printer = ConsoleMessagePrinter()
config = get_validated_config(
config_file_name, msg_update_callback=console_message_printer)
log_filename = None
log_wire = str_to_bool(config['service'].get('log_wire'))
if log_wire:
log_filename = 'cluster_convert_wire.log'
client = Client(config['vcd']['host'],
api_version=config['vcd']['api_version'],
verify_ssl_certs=config['vcd']['verify'],
log_file=log_filename,
log_requests=log_wire,
log_headers=log_wire,
log_bodies=log_wire)
credentials = BasicLoginCredentials(config['vcd']['username'],
SYSTEM_ORG_NAME,
config['vcd']['password'])
client.set_credentials(credentials)
msg = f"Connected to vCD as system administrator: " \
f"{config['vcd']['host']}:{config['vcd']['port']}"
console_message_printer.general(msg)
cluster_records = get_all_clusters(client=client,
cluster_name=cluster_name,
org_name=org_name,
ovdc_name=vdc_name)
if len(cluster_records) == 0:
console_message_printer.info(f"No clusters were found.")
return
vms = []
for cluster in cluster_records:
console_message_printer.info(
f"Processing cluster '{cluster['name']}'.")
vapp_href = cluster['vapp_href']
vapp = VApp(client, href=vapp_href)
console_message_printer.info("Processing metadata of cluster.")
metadata = metadata_to_dict(vapp.get_metadata())
old_template_name = None
new_template_name = None
if ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME in metadata: # noqa: E501
old_template_name = metadata.pop(ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME) # noqa: E501
version = metadata.get(ClusterMetadataKey.CSE_VERSION)
if old_template_name:
console_message_printer.info(
"Determining k8s version on cluster.")
if 'photon' in old_template_name:
new_template_name = 'photon-v2'
if '1.0.0' in version:
new_template_name += '_k8s-1.8_weave-2.0.5'
elif any(ver in version for ver in ('1.1.0', '1.2.0', '1.2.1', '1.2.2', '1.2.3', '1.2.4',)): # noqa: E501
new_template_name += '_k8s-1.9_weave-2.3.0'
elif any(ver in version for ver in ('1.2.5', '1.2.6', '1.2.7',)): # noqa: E501
new_template_name += '_k8s-1.10_weave-2.3.0'
elif '2.0.0' in version:
new_template_name += '_k8s-1.12_weave-2.3.0'
elif 'ubuntu' in old_template_name:
new_template_name = 'ubuntu-16.04'
if '1.0.0' in version:
new_template_name += '_k8s-1.9_weave-2.1.3'
elif any(ver in version for ver in ('1.1.0', '1.2.0', '1.2.1', '1.2.2', '1.2.3', '1.2.4', '1.2.5', '1.2.6', '1.2.7')): # noqa: E501
new_template_name += '_k8s-1.10_weave-2.3.0'
elif '2.0.0' in version:
new_template_name += '_k8s-1.13_weave-2.3.0'
if new_template_name:
console_message_printer.info("Updating metadata of cluster.")
task = vapp.remove_metadata(ClusterMetadataKey.BACKWARD_COMPATIBILE_TEMPLATE_NAME) # noqa: E501
client.get_task_monitor().wait_for_success(task)
new_metadata_to_add = {
ClusterMetadataKey.TEMPLATE_NAME: new_template_name,
ClusterMetadataKey.TEMPLATE_REVISION: 0
}
task = vapp.set_multiple_metadata(new_metadata_to_add)
client.get_task_monitor().wait_for_success(task)
console_message_printer.general(
"Finished processing metadata of cluster.")
try:
console_message_printer.info(
f"Undeploying the vApp '{cluster['name']}'")
task = vapp.undeploy()
client.get_task_monitor().wait_for_success(task)
console_message_printer.general(
"Successfully undeployed the vApp.")
except Exception as err:
console_message_printer.error(str(err))
vm_resources = vapp.get_all_vms()
for vm_resource in vm_resources:
console_message_printer.info(
f"Processing vm '{vm_resource.get('name')}'.")
vm = VM(client, href=vm_resource.get('href'))
vms.append(vm)
console_message_printer.info("Updating vm admin password.")
task = vm.update_guest_customization_section(
enabled=True,
admin_password_enabled=True,
admin_password_auto=not password,
admin_password=password,
)
client.get_task_monitor().wait_for_success(task)
console_message_printer.general("Successfully updated vm .")
console_message_printer.info("Deploying vm.")
task = vm.power_on_and_force_recustomization()
client.get_task_monitor().wait_for_success(task)
console_message_printer.general("Successfully deployed vm.")
console_message_printer.info("Deploying cluster")
task = vapp.deploy(power_on=True)
client.get_task_monitor().wait_for_success(task)
console_message_printer.general("Successfully deployed cluster.")
console_message_printer.general(
f"Successfully processed cluster '{cluster['name']}'.")
if skip_wait_for_gc:
return
while True:
for vm in vms:
status = vm.get_guest_customization_status()
if status != 'GC_PENDING':
vms.remove(vm)
console_message_printer.info(
f"Waiting on guest customization to finish on {len(vms)} vms.")
if not len(vms) == 0:
time.sleep(5)
else:
break
except Exception as err:
click.secho(str(err), fg='red')
finally:
if client:
client.logout()
