def download_logs(client: InventoryClient, cluster: dict, dest: str,
must_gather: bool, retry_interval: int = RETRY_INTERVAL):
output_folder = get_logs_output_folder(dest, cluster)
if os.path.isdir(output_folder):
log.info(f"Skipping. The logs directory {output_folder} already exists.")
return
recreate_folder(output_folder)
recreate_folder(os.path.join(output_folder, "cluster_files"))
try:
write_metadata_file(client, cluster, os.path.join(output_folder, 'metdata.json'))
with suppress(assisted_service_client.rest.ApiException):
client.download_ignition_files(cluster['id'], os.path.join(output_folder, "cluster_files"))
for host_id in map(lambda host: host['id'], cluster['hosts']):
with suppress(assisted_service_client.rest.ApiException):
client.download_host_ignition(cluster['id'], host_id, os.path.join(output_folder, "cluster_files"))
with suppress(assisted_service_client.rest.ApiException):
client.download_cluster_events(cluster['id'], os.path.join(output_folder, f"cluster_{cluster['id']}_events.json"))
shutil.copy2(os.path.join(os.path.dirname(os.path.realpath(__file__)), "events.html"), output_folder)
with suppress(assisted_service_client.rest.ApiException):
for i in range(MAX_RETRIES):
cluster_logs_tar = os.path.join(output_folder, f"cluster_{cluster['id']}_logs.tar")
with suppress(FileNotFoundError):
os.remove(cluster_logs_tar)
client.download_cluster_logs(cluster['id'], cluster_logs_tar)
min_number_of_logs = len(cluster['hosts']) + 1 if cluster['status'] == ClusterStatus.INSTALLED else len(cluster['hosts'])
try:
verify_logs_uploaded(cluster_logs_tar, min_number_of_logs, cluster['status'] == ClusterStatus.INSTALLED)
break
except AssertionError as ex:
log.warn(f"Cluster logs verification failed: {ex}")
# Skip sleeping on last retry
if i < MAX_RETRIES - 1:
log.info(f"Going to retry in {retry_interval} seconds")
time.sleep(retry_interval)
kubeconfig_path = os.path.join(output_folder, "kubeconfig-noingress")
with suppress(assisted_service_client.rest.ApiException):
client.download_kubeconfig_no_ingress(cluster['id'], kubeconfig_path)
if must_gather:
recreate_folder(os.path.join(output_folder, "must-gather"))
config_etc_hosts(cluster['name'], cluster['base_dns_domain'],
helper_cluster.get_api_vip_from_cluster(client, cluster))
download_must_gather(kubeconfig_path, os.path.join(output_folder, "must-gather"))
finally:
run_command(f"chmod -R ugo+rx '{output_folder}'")
def download_logs(client: InventoryClient, cluster: dict, dest: str, must_gather: bool, update_by_events: bool = False,
retry_interval: int = RETRY_INTERVAL):
output_folder = get_logs_output_folder(dest, cluster)
if not is_update_needed(output_folder, update_by_events, client, cluster):
log.info(f"Skipping, no need to update {output_folder}.")
return
recreate_folder(output_folder)
recreate_folder(os.path.join(output_folder, "cluster_files"))
try:
write_metadata_file(client, cluster, os.path.join(output_folder, 'metdata.json'))
with suppressAndLog(AssertionError, ConnectionError, requests.exceptions.ConnectionError):
client.download_metrics(os.path.join(output_folder, "metrics.txt"))
for ignition_file in ("bootstrap.ign", "master.ign", "worker.ign", "install-config.yaml"):
with suppress(assisted_service_client.rest.ApiException):
client.download_and_save_file(cluster['id'], ignition_file,
os.path.join(output_folder, "cluster_files", ignition_file))
for host_id in map(lambda host: host['id'], cluster['hosts']):
with suppressAndLog(assisted_service_client.rest.ApiException):
client.download_host_ignition(cluster['id'], host_id, os.path.join(output_folder, "cluster_files"))
with suppressAndLog(assisted_service_client.rest.ApiException):
client.download_cluster_events(cluster['id'], get_cluster_events_path(cluster, output_folder))
shutil.copy2(os.path.join(os.path.dirname(os.path.realpath(__file__)), "events.html"), output_folder)
with suppressAndLog(assisted_service_client.rest.ApiException):
are_masters_in_configuring_state = are_host_progress_in_stage(
cluster['hosts'], [HostsProgressStages.CONFIGURING], 2)
are_masters_in_join_state = are_host_progress_in_stage(
cluster['hosts'], [HostsProgressStages.JOINED], 2)
max_retries = 2 * MAX_RETRIES if are_masters_in_join_state else MAX_RETRIES
is_controller_expected = cluster['status'] == ClusterStatus.INSTALLED or are_masters_in_configuring_state
min_number_of_logs = min_number_of_log_files(cluster, is_controller_expected)
for i in range(max_retries):
cluster_logs_tar = os.path.join(output_folder, f"cluster_{cluster['id']}_logs.tar")
with suppress(FileNotFoundError):
os.remove(cluster_logs_tar)
client.download_cluster_logs(cluster['id'], cluster_logs_tar)
try:
verify_logs_uploaded(cluster_logs_tar, min_number_of_logs,
installation_success=(cluster['status'] == ClusterStatus.INSTALLED),
check_oc=are_masters_in_join_state)
break
except AssertionError as ex:
log.warn(f"Cluster logs verification failed: {ex}")
# Skip sleeping on last retry
if i < MAX_RETRIES - 1:
log.info(f"Going to retry in {retry_interval} seconds")
time.sleep(retry_interval)
kubeconfig_path = os.path.join(output_folder, "kubeconfig-noingress")
with suppressAndLog(assisted_service_client.rest.ApiException):
client.download_kubeconfig_no_ingress(cluster['id'], kubeconfig_path)
if must_gather:
recreate_folder(os.path.join(output_folder, "must-gather"))
config_etc_hosts(cluster['name'], cluster['base_dns_domain'],
helper_cluster.get_api_vip_from_cluster(client, cluster))
download_must_gather(kubeconfig_path, os.path.join(output_folder, "must-gather"))
finally:
run_command(f"chmod -R ugo+rx '{output_folder}'")