def _safe_load(index: str, url: str):
"""
Load config if not loaded
"""
if index not in ConfigManager._conf:
Conf.load(index, url)
ConfigManager._conf.append(index)
async def _get_bundle_status(command):
"""
Initializes the process for Displaying the Status for Support Bundle.
command: Command Object :type: command
return: None
"""
try:
status = ''
node_id = Conf.machine_id
Conf.load(const.SB_INDEX,
'json://' + const.FILESTORE_PATH,
skip_reload=True)
bundle_id = command.options.get(const.SB_BUNDLE_ID)
if not bundle_id:
status = Conf.get(const.SB_INDEX, f'{node_id}')
else:
status = Conf.get(const.SB_INDEX,
f'{node_id}>{bundle_id}>status')
if not status:
return Response(output=(f"No status found for bundle_id: {bundle_id}" \
"in input config. Please check if the Bundle ID is correct"), \
rc=ERR_OP_FAILED)
return Response(output=status, rc=OPERATION_SUCESSFUL)
except Exception as e:
Log.error(f"Failed to get bundle status: {e}")
return Response(output=(f"Support Bundle status is not available " \
f"Failed to get status of bundle. Related error - {e}"), \
rc=str(errno.ENOENT))
def init(log_name) -> None:
"""
Initialize ha conf and log
Args:
log_name ([str]): service_name for log init.
"""
Conf.init(delim='.')
Conf.load(const.HA_GLOBAL_INDEX, f"yaml://{const.HA_CONFIG_FILE}")
Conf.load(const.RESOURCE_GLOBAL_INDEX, f"json://{const.RESOURCE_SCHEMA}")
log_path = Conf.get(const.HA_GLOBAL_INDEX, "LOG.path")
log_level = Conf.get(const.HA_GLOBAL_INDEX, "LOG.level")
Log.init(service_name=log_name, log_path=log_path, level=log_level)
async def get_active_nodes():
"""
This Method is for reading hostnames, node_list information.
:return: hostnames : List of Hostname :type: List
:return: node_list : : List of Node Name :type: List
"""
Log.info("Reading hostnames, node_list information")
Conf.load('cortx_cluster', 'json:///etc/cortx/cluster.conf')
node_hostname_map = Conf.get('cortx_cluster', 'cluster')
if not node_hostname_map:
response_msg = "Node list and hostname not found."
return Response(output=response_msg, rc=errno.ENODATA), None
return node_hostname_map
async def _get_active_nodes() -> dict:
"""This Method is for reading hostnames, node_list information
Returns:
dict: hosts in cluster eg {node1:fqd1}
Response: Response object if no host found in config file
"""
Log.info("Reading hostnames, node_list information")
Conf.load('cortx_cluster', 'json:///etc/cortx/cluster.conf', \
skip_reload=True)
node_hostname_map = Conf.get('cortx_cluster', 'cluster')
if not node_hostname_map:
response_msg = "Node list and hostname not found."
return Response(output=response_msg, rc=errno.ENODATA), None
return node_hostname_map
async def _generate_bundle(command):
"""
Initializes the process for Generating Support Bundle on Each CORTX Node.
command: Command Object :type: command
return: None.
"""
bundle_id = SupportBundle._generate_bundle_id()
provisioner = ProvisionerServices()
if not provisioner:
return Response(output="Provisioner package not found.", \
rc=errno.ENOENT)
# Get Arguments From Command
comment = command.options.get(const.SB_COMMENT)
components = command.options.get(const.SB_COMPONENTS)
if not components:
components = []
if command.options.get(const.SOS_COMP, False) == 'true':
components.append('os')
Conf.load('cortx_conf', 'json:///etc/cortx/cortx.conf', \
skip_reload=True)
# Get HostNames and Node Names.
node_hostname_map = await SupportBundle._get_active_nodes()
if not isinstance(node_hostname_map, dict):
return node_hostname_map
shared_path = Storage.get_path(name='support_bundle')
path = shared_path if shared_path else Conf.get('cortx_conf',\
'support>local_path')
bundle_path = os.path.join(path, bundle_id)
os.makedirs(bundle_path)
bundle_obj = Bundle(bundle_id=bundle_id, bundle_path=path, \
comment=comment,is_shared=True if shared_path else False)
support_bundle_file = os.path.join(Conf.get('cortx_conf', 'install_path'),\
'cortx/utils/conf/support_bundle.yaml')
Conf.load('sb_yaml', f'yaml://{support_bundle_file}', skip_reload=True)
all_components = Conf.get('sb_yaml', 'COMPONENTS')
invalid_comps = [
component for component in components
if component not in all_components.keys()
]
if invalid_comps:
components = list(set(components) - set(invalid_comps))
ComponentsBundle._publish_log(f"""Invalid components - '{", ".join(invalid_comps)}'""", \
'error', bundle_id, '', comment)
if invalid_comps and not components:
return bundle_obj
comp_list = SupportBundle._get_components(components)
# Start SB Generation on all Nodes.
for nodename, hostname in node_hostname_map.items():
Log.debug(f"Connect to {hostname}")
try:
# TODO: pass bundle_path to bundle_generation when args implemented
await provisioner.begin_bundle_generation(
f"bundle_generate '{bundle_id}' '{comment}' "
f"'{hostname}' {comp_list}", nodename)
except BundleError as be:
Log.error(f"Bundle generation failed.{be}")
ComponentsBundle._publish_log(f"Bundle generation failed.{be}", \
'error', bundle_id, nodename, comment)
except Exception as e:
Log.error(f"Internal error, bundle generation failed {e}")
ComponentsBundle._publish_log(
f"Internal error, bundle generation failed \
{e}", 'error', bundle_id, nodename, comment)
# Create common tar.
#if bundle_obj.is_shared:
# tar_dest_file = f"{bundle_id}.tar.gz"
# Log.debug(f"Merging all bundle to {bundle_path}/{tar_dest_file}")
# try:
# Tar(os.path.join(bundle_path, tar_dest_file)).dump([bundle_path])
# except:
# Log.debug("Merging of node support bundle failed")
# return Response(output="Bundle Generation Failed in merging",
# rc=errno.EINVAL)
if command.sub_command_name == 'generate':
display_string_len = len(bundle_obj.bundle_id) + 4
response_msg = (
f"Please use the below bundle id for checking the status of support bundle."
f"\n{'-' * display_string_len}"
f"\n| {bundle_obj.bundle_id} |"
f"\n{'-' * display_string_len}"
f"\nPlease Find the file on -> {bundle_obj.bundle_path} .\n")
return Response(output=response_msg, rc=OPERATION_SUCESSFUL)
return bundle_obj
async def init(bundle_obj, node_id, config_url, **kwargs):
"""
Initializes the Process of Support Bundle Generation for Every Component.
command: cli Command Object :type: command
return: None
"""
cluster_conf = MappedConf(config_url)
log_path = os.path.join(cluster_conf.get(CLUSTER_CONF_LOG_KEY), \
f'utils/{Conf.machine_id}/support')
log_level = cluster_conf.get('utils>log_level', 'INFO')
Log.init('support_bundle_node', log_path, level=log_level, \
backup_count=5, file_size_in_mb=5)
bundle_id = bundle_obj.bundle_id
node_name = bundle_obj.node_name
comment = bundle_obj.comment
components_list = bundle_obj.components
services_dict = bundle_obj.services
Log.info(f"components:{components_list}")
bundle_path = bundle_obj.bundle_path
duration = kwargs.get('duration')
size_limit = kwargs.get('size_limit')
binlogs = kwargs.get('binlogs')
coredumps = kwargs.get('coredumps')
stacktrace = kwargs.get('stacktrace')
Log.debug((f"{const.SB_BUNDLE_ID}: {bundle_id}, {const.SB_NODE_NAME}: "
f"{node_name}, {const.SB_COMMENT}: {comment}, "
f"{const.SB_COMPONENTS}: {components_list}, {const.SOS_COMP}"))
# Read support_bundle.Yaml and Check's If It Exists.
cmd_setup_file = os.path.join(
cluster_conf.get('install_path', DEFAULT_INSTALL_PATH),
const.SUPPORT_YAML)
try:
support_bundle_config = Yaml(cmd_setup_file).load()
except Exception as e:
Log.error(f"Internal error while parsing YAML file {cmd_setup_file}{e}")
if not support_bundle_config:
Log.error(f"No such file {cmd_setup_file}. ERROR:{ERROR}")
# Start Execution for each Component Command.
command_files_info = support_bundle_config.get('COMPONENTS')
# OS Logs are specifically generated hence here Even
# When All is Selected O.S. Logs Will Be Skipped.
for each_component in components_list:
services = services_dict[each_component]
components_commands = []
components_files = command_files_info[each_component]
for file_path in components_files:
if not os.path.exists(file_path):
Log.error(f"'{file_path}' file does not exist!")
continue
try:
file_data = Yaml(file_path).load()
except Exception as e:
Log.error(f"Internal error while parsing YAML file {file_path}{e}")
file_data = None
break
if file_data:
components_commands = file_data.get(
const.SUPPORT_BUNDLE.lower(), [])
else:
Log.error(f"Support.yaml file is empty: {file_path}")
break
if components_commands:
component, return_code = await(\
ComponentsBundle._exc_components_cmd(\
components_commands, f'{bundle_id}_{node_id}_{each_component}',
f'{bundle_path}{os.sep}', each_component,
node_name, comment, config_url, services,
binlogs, coredumps, stacktrace, duration,
size_limit))
if return_code != 0:
Log.error(
f"Bundle generation failed for component - '{component}'")
else:
Log.info(
f"Bundle generation started for component - '{component}'")
tar_file_name = os.path.join(bundle_path, \
f'{bundle_id}_{node_id}.tar.gz')
ComponentsBundle._create_summary_file(bundle_id, node_name, \
comment, bundle_path)
try:
Log.debug(f"Generating tar.gz file on path {tar_file_name} "
f"from {bundle_path}")
Tar(tar_file_name).dump([bundle_path])
bundle_status = f"Successfully generated SB at path:{bundle_path}"
except Exception as e:
bundle_status = f"Failed to generate tar file. ERROR:{e}"
Log.error(f"Could not generate tar file {e}")
finally:
if os.path.exists(bundle_path):
for each_dir in os.listdir(bundle_path):
comp_dir = os.path.join(bundle_path, each_dir)
if os.path.isdir(comp_dir):
shutil.rmtree(comp_dir)
if os.path.exists(os.path.join(bundle_path, 'summary.yaml')):
os.remove(os.path.join(bundle_path, 'summary.yaml'))
Log.info("Support bundle generation completed.")
# Update Status in ConfStor
Conf.load(const.SB_INDEX, 'json://' + const.FILESTORE_PATH, fail_reload=False)
Conf.set(const.SB_INDEX, f'{node_id}>{bundle_id}>status', bundle_status)
Conf.save(const.SB_INDEX)
#!/usr/bin/env python3 # CORTX-Py-Utils: CORTX Python common library. # Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published # by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # For any questions about this software or licensing, # please email [email protected] or [email protected]. from cortx.utils.conf_store.conf_store import Conf Conf.load('cortx_conf', 'json:///etc/cortx/cortx.conf')
async def init(command: List):
"""
Initializes the Process of Support Bundle Generation for Every Component.
command: cli Command Object :type: command
return: None
"""
bundle_id = command.options.get(const.SB_BUNDLE_ID, '')
node_name = command.options.get(const.SB_NODE_NAME, '')
comment = command.options.get(const.SB_COMMENT, '')
components = command.options.get(const.SB_COMPONENTS, [])
Log.debug((f"{const.SB_BUNDLE_ID}: {bundle_id}, {const.SB_NODE_NAME}: "
f"{node_name}, {const.SB_COMMENT}: {comment}, "
f"{const.SB_COMPONENTS}: {components}, {const.SOS_COMP}"))
# Read Commands.Yaml and Check's If It Exists.
Conf.load('cortx_conf', 'json:///etc/cortx/cortx.conf', \
skip_reload=True)
cmd_setup_file = os.path.join(Conf.get('cortx_conf', 'install_path'),\
'cortx/utils/conf/support_bundle.yaml')
try:
support_bundle_config = Yaml(cmd_setup_file).load()
except Exception as e:
Log.error(
f"Internal error while parsing YAML file {cmd_setup_file}{e}")
ComponentsBundle._publish_log(f"Internal error while parsing YAML file " \
f"{cmd_setup_file}{e}", ERROR, bundle_id, node_name, comment)
if not support_bundle_config:
ComponentsBundle._publish_log(f"No such file {cmd_setup_file}", \
ERROR, bundle_id, node_name, comment)
# Shared/Local path Location for creating Support Bundle.
from cortx.utils.shared_storage import Storage
path = Storage.get_path('support_bundle')
if not path:
path = Conf.get('cortx_conf', 'support>local_path')
bundle_path = os.path.join(path, bundle_id, node_name)
try:
os.makedirs(bundle_path, exist_ok=True)
except PermissionError as e:
Log.error(f"Incorrect permissions for path:{bundle_path} - {e}")
ComponentsBundle._publish_log(f"Incorrect permissions for path: {bundle_path} - {e}", \
ERROR, bundle_id, node_name, comment)
# Start Execution for each Component Command.
threads = []
command_files_info = support_bundle_config.get('COMPONENTS')
# OS Logs are specifically generated hence here Even
# When All is Selected O.S. Logs Will Be Skipped.
if components:
if 'all' not in components:
components_list = list(set(command_files_info.keys()\
).intersection(set(components)))
else:
components_list = list(command_files_info.keys())
components_list.remove(const.SOS_COMP)
Log.debug(
f"Generating for manifest and {const.SB_COMPONENTS} {' '.join(components_list)}"
)
thread_que = Queue()
# Manifest component supportbundle generation
try:
thread_obj = threading.Thread(
ManifestSupportBundle.generate(f'{bundle_id}_manifiest',
f'{bundle_path}{os.sep}'))
thread_obj.start()
Log.debug(f"Started thread -> {thread_obj.ident} " \
f"Component -> manifest")
threads.append(thread_obj)
except Exception as e:
Log.error(f"Internal error while calling ManifestSupportBundle"\
f" generate api {e}")
ComponentsBundle._publish_log(f"Internal error at while bundling"\
f" Manifest component: {bundle_path} - {e}", ERROR,
bundle_id, node_name, comment)
for each_component in components_list:
components_commands = []
components_files = command_files_info[each_component]
for file_path in components_files:
if not os.path.exists(file_path):
ComponentsBundle._publish_log(f"'{file_path}' file does not exist!", \
ERROR, bundle_id, node_name, comment)
continue
try:
file_data = Yaml(file_path).load()
except Exception as e:
Log.error(
f"Internal error while parsing YAML file {file_path}{e}"
)
file_data = None
ComponentsBundle._publish_log(f"Internal error while parsing YAML file: " \
f"{file_path} - {e}", ERROR, bundle_id, node_name, comment)
break
if file_data:
components_commands = file_data.get(
const.SUPPORT_BUNDLE.lower(), [])
else:
ComponentsBundle._publish_log(f"Support.yaml file is empty: " \
f"{file_path}", ERROR, bundle_id, node_name, comment)
break
if components_commands:
thread_obj = threading.Thread(\
ComponentsBundle._exc_components_cmd(\
components_commands, f'{bundle_id}_{each_component}',
f'{bundle_path}{os.sep}', each_component,
node_name, comment, thread_que))
thread_obj.start()
Log.debug(f"Started thread -> {thread_obj.ident} " \
f"Component -> {each_component}")
threads.append(thread_obj)
# directory_path = Conf.get('cortx_conf', 'support')
tar_file_name = os.path.join(bundle_path, \
f'{bundle_id}_{node_name}.tar.gz')
ComponentsBundle._create_summary_file(bundle_id, node_name, \
comment, bundle_path)
# Wait Until all the Threads Execution is not Complete.
for each_thread in threads:
Log.debug(
f"Waiting for thread - {each_thread.ident} to complete process"
)
each_thread.join(timeout=1800)
if not thread_que.empty():
component, return_code = thread_que.get()
if return_code != 0:
ComponentsBundle._publish_log(
f"Bundle generation failed for component - '{component}'",
ERROR, bundle_id, node_name, comment)
else:
ComponentsBundle._publish_log(
f"Bundle generation started for component - '{component}'",
INFO, bundle_id, node_name, comment)
try:
Log.debug(f"Generating tar.gz file on path {tar_file_name} "
f"from {bundle_path}")
Tar(tar_file_name).dump([bundle_path])
except Exception as e:
ComponentsBundle._publish_log(f"Could not generate tar file {e}", \
ERROR, bundle_id, node_name, comment)
finally:
if os.path.exists(bundle_path):
for each_dir in os.listdir(bundle_path):
comp_dir = os.path.join(bundle_path, each_dir)
if os.path.isdir(comp_dir):
shutil.rmtree(comp_dir)
msg = "Support bundle generation completed."
ComponentsBundle._publish_log(msg, INFO, bundle_id, node_name, comment)
async def _generate_bundle(command):
"""
Initializes the process for Generating Support Bundle at shared path.
command: Command Object :type: command
return: None.
"""
# Get Arguments From Command
bundle_id = command.options.get(const.SB_BUNDLE_ID)
comment = command.options.get(const.SB_COMMENT)
duration = command.options.get(const.SB_DURATION)
size_limit = command.options.get(const.SB_SIZE)
config_url = command.options.get('config_url')
binlogs = command.options.get('binlogs')
coredumps = command.options.get('coredumps')
stacktrace = command.options.get('stacktrace')
components = command.options.get('components')
config_path = config_url.split('//')[1] if '//' in config_url else ''
path = command.options.get('target_path')
bundle_path = os.path.join(path, bundle_id)
try:
os.makedirs(bundle_path)
except FileExistsError:
raise BundleError(
errno.EINVAL, "Bundle ID already exists,"
"Please use Unique Bundle ID")
cluster_conf = MappedConf(config_url)
# Get Node ID
node_id = Conf.machine_id
if node_id is None:
raise BundleError(errno.EINVAL, "Invalid node_id: %s", \
node_id)
# Update SB status in Filestore.
# load conf for Support Bundle
Conf.load(const.SB_INDEX,
'json://' + const.FILESTORE_PATH,
skip_reload=True)
data = {
'status': 'In-Progress',
'start_time': datetime.strftime(datetime.now(),
'%Y-%m-%d %H:%M:%S')
}
Conf.set(const.SB_INDEX, f'{node_id}>{bundle_id}', data)
Conf.save(const.SB_INDEX)
node_name = cluster_conf.get(f'node>{node_id}>name')
Log.info(f'Starting SB Generation on {node_id}:{node_name}')
# Get required SB size per component
components_list, service_per_comp = SupportBundle._get_component_and_services(
cluster_conf, node_id, components)
if not components_list:
Log.warn(f"No component specified for {node_name} in CORTX config")
Log.warn(f"Skipping SB generation on node:{node_name}.")
return
num_components = len(components_list)
size_limit_per_comp = SupportBundle.get_component_size_limit(
size_limit, num_components)
bundle_obj = Bundle(bundle_id=bundle_id, bundle_path=bundle_path, \
comment=comment,node_name=node_name, components=components_list,
services=service_per_comp)
# Start SB Generation on Node.
# Adding CORTX manifest data inside support Bundle.
try:
# Copying config file into support bundle.
common_locations = set()
if config_path and os.path.exists(config_path):
Log.info(f'For manifest data collection, taking config from \
{config_path} location.')
# Remove secrets from the input config.
conf_name = config_path.split('/')[-1]
sb_config = config_path.replace(conf_name, 'sb_cluster.conf')
with open(sb_config, 'w+') as sb_file:
with open(config_path, 'r') as f:
content = f.read()
if 'secret:' in content:
content = re.sub(r'secret:.+', r'secret: ****',
content)
sb_file.write(content)
conf_target = os.path.join(bundle_path, 'common' + config_path)
os.makedirs(conf_target.replace(f'/{conf_name}', ''),
exist_ok=True)
shutil.move(sb_config, conf_target)
common_locations.add(config_path.split('/')[1])
# Copying "/etc/cortx/solution" directory into support bundle
# except for "secret" folder.
sln_target = os.path.join(bundle_path, 'common' + const\
.CORTX_SOLUTION_DIR)
if os.path.exists(sln_target):
shutil.rmtree(sln_target)
if os.path.exists(const.CORTX_SOLUTION_DIR):
_ = shutil.copytree(const.CORTX_SOLUTION_DIR, sln_target, \
ignore=shutil.ignore_patterns('secret'))
common_locations.add(const.CORTX_SOLUTION_DIR.split('/')[1])
# Copying RELEASE.INFO file into support bundle.
if os.path.exists(const.CORTX_RELEASE_INFO):
rel_target = os.path.join(bundle_path, 'common' + const\
.CORTX_RELEASE_INFO)
os.makedirs(rel_target.replace('/RELEASE.INFO', ''),
exist_ok=True)
shutil.copyfile(const.CORTX_RELEASE_INFO, rel_target)
common_locations.add(const.CORTX_RELEASE_INFO.split('/')[1])
else:
Log.warn(f'{const.CORTX_RELEASE_INFO} file not found.')
# Adding node resources health into the support bundle.
health_target = os.path.join(bundle_path, 'common' + '/health')
os.makedirs(health_target, exist_ok=True)
with open(health_target + '/node_health.json', 'w') as fp:
info = {}
info["resource_usage"] = {}
info["resource_usage"]["cpu_usage"] = SupportBundle.\
get_cpu_overall_usage()
info["resource_usage"]["uptime"] = SupportBundle.\
get_system_uptime()
info["resource_usage"]["disk_usage"] = SupportBundle.\
get_disk_overall_usage()
info["resource_usage"]["memory_usage"] = SupportBundle.\
get_mem_overall_usage()
json.dump(info, fp, indent=4)
common_locations.add('health')
try:
common_path = os.path.join(bundle_path, 'common')
common_tar = os.path.join(common_path, 'common.tar.gz')
with tarfile.open(common_tar, "w:gz") as tar:
if os.path.exists(common_path):
tar.add(common_path, arcname='common')
# Deleting untar directories from the common folder.
for location in common_locations:
untar_location = os.path.join(common_path, location)
if os.path.exists(untar_location):
shutil.rmtree(untar_location)
except (OSError, tarfile.TarError) as err:
Log.error(
"Facing issues while adding manifest data into common "
"directory: {0}".format(err))
except BundleError as be:
Log.error(
f"Failed to add CORTX manifest data inside Support Bundle.{be}"
)
try:
await ComponentsBundle.init(bundle_obj,
node_id,
config_url,
duration=duration,
size_limit=size_limit_per_comp,
binlogs=binlogs,
coredumps=coredumps,
stacktrace=stacktrace)
except BundleError as be:
Log.error(f"Bundle generation failed.{be}")
except Exception as e:
Log.error(f"Internal error, bundle generation failed {e}")
if command.sub_command_name == 'generate':
display_string_len = len(bundle_obj.bundle_id) + 4
response_msg = (
f"Please use the below bundle id for checking the status of support bundle."
f"\n{'-' * display_string_len}"
f"\n| {bundle_obj.bundle_id} |"
f"\n{'-' * display_string_len}"
f"\nPlease Find the file on -> {bundle_obj.bundle_path} .\n")
return Response(output=response_msg, rc=OPERATION_SUCESSFUL)
return bundle_obj
