async def get_bundle_status(command):
"""
Initializes the process for Displaying the Status for Support Bundle.
:param command: Csm_cli Command Object :type: command
:return: None
"""
try:
bundle_id = command.options.get(const.SB_BUNDLE_ID)
conf = GeneralConfig(database.DATABASE)
db = DataBaseProvider(conf)
repo = SupportBundleRepository(db)
all_nodes_status = await repo.retrieve_all(bundle_id)
response = {
"status": [
each_status.to_primitive()
for each_status in all_nodes_status
]
}
return Response(output=response, rc=OPERATION_SUCESSFUL)
except DataAccessExternalError as e:
Log.warn(f"Failed to connect to elasticsearch: {e}")
return Response(
output=("Support Bundle status is not available currently"
" as required services are not running."
" Please wait and check the /tmp/support_bundle"
" folder for newly generated support bundle."),
rc=str(errno.ECONNREFUSED))
except Exception as e:
Log.error(f"Failed to get bundle status: {e}")
return Response(
output=("Support Bundle status is not available currently"
" as required services are not running."
" Failed to get status of bundle."),
rc=str(errno.ENOENT))
async def _get_bundle_status(command):
"""
Initializes the process for Displaying the Status for Support Bundle.
command: Command Object :type: command
return: None
"""
try:
status = ''
node_id = Conf.machine_id
Conf.load(const.SB_INDEX,
'json://' + const.FILESTORE_PATH,
skip_reload=True)
bundle_id = command.options.get(const.SB_BUNDLE_ID)
if not bundle_id:
status = Conf.get(const.SB_INDEX, f'{node_id}')
else:
status = Conf.get(const.SB_INDEX,
f'{node_id}>{bundle_id}>status')
if not status:
return Response(output=(f"No status found for bundle_id: {bundle_id}" \
"in input config. Please check if the Bundle ID is correct"), \
rc=ERR_OP_FAILED)
return Response(output=status, rc=OPERATION_SUCESSFUL)
except Exception as e:
Log.error(f"Failed to get bundle status: {e}")
return Response(output=(f"Support Bundle status is not available " \
f"Failed to get status of bundle. Related error - {e}"), \
rc=str(errno.ENOENT))
async def generate_bundle(command) -> sys.stdout:
"""
Initializes the process for Generating Support Bundle on Each CORTX Node.
:param command: Csm_cli Command Object :type: command
:return: None.
"""
current_user = str(getpass.getuser())
# Check if User is Root User.
if current_user.lower() != "root":
response_msg = "Support Bundle Command requires root privileges"
return Response(output=response_msg, rc=errno.EACCES)
bundle_id = SupportBundle._generate_bundle_id()
provisioner = ProvisionerServices()
if not provisioner:
return Response(output="Provisioner package not found.",
rc=errno.ENOENT)
# Get Arguments From Command
comment = command.options.get(const.SB_COMMENT)
components = command.options.get(const.SB_COMPONENTS)
if not components:
components = []
if command.options.get(const.SOS_COMP, False) == "true":
components.append("os")
comp_list = SupportBundle._get_components(components)
# Get HostNames and Node Names.
node_hostname_map = await SupportBundle.get_active_nodes()
if not isinstance(node_hostname_map, dict):
return node_hostname_map
# Start SB Generation on all Nodes.
for nodename, hostname in node_hostname_map.items():
Log.debug(f"Connect to {hostname}")
try:
await provisioner.begin_bundle_generation(
f"bundle_generate '{bundle_id}' '{comment}' "
f"'{hostname}' {comp_list}", nodename)
except BundleError as be:
Log.error(f"Bundle generation failed.{be}")
return Response(
"Bundle generation failed.\nPlease "
"check CLI for details.",
rc=errno.EINVAL)
except Exception as e:
Log.error(f"Provisioner API call failed : {e}")
return Response(output="Bundle Generation Failed.",
rc=errno.ENOENT)
symlink_path = const.SYMLINK_PATH
display_string_len = len(bundle_id) + 4
response_msg = (
f"Please use the below bundle id for checking the status of support bundle."
f"\n{'-' * display_string_len}"
f"\n| {bundle_id} |"
f"\n{'-' * display_string_len}"
f"\nPlease Find the file on -> {symlink_path} .\n")
return Response(output=response_msg, rc=OPERATION_SUCESSFUL)
async def get_active_nodes():
"""
This Method is for reading hostnames, node_list information.
:return: hostnames : List of Hostname :type: List
:return: node_list : : List of Node Name :type: List
"""
Log.info("Reading hostnames, node_list information")
Conf.load('cortx_cluster', 'json:///etc/cortx/cluster.conf')
node_hostname_map = Conf.get('cortx_cluster', 'cluster')
if not node_hostname_map:
response_msg = "Node list and hostname not found."
return Response(output=response_msg, rc=errno.ENODATA), None
return node_hostname_map
async def _get_active_nodes() -> dict:
"""This Method is for reading hostnames, node_list information
Returns:
dict: hosts in cluster eg {node1:fqd1}
Response: Response object if no host found in config file
"""
Log.info("Reading hostnames, node_list information")
Conf.load('cortx_cluster', 'json:///etc/cortx/cluster.conf', \
skip_reload=True)
node_hostname_map = Conf.get('cortx_cluster', 'cluster')
if not node_hostname_map:
response_msg = "Node list and hostname not found."
return Response(output=response_msg, rc=errno.ENODATA), None
return node_hostname_map
def test_output(self):
self.command = CommandFactory.get_command(self.cli_command,
self.permissions,
self.directory_path)
response = Response(output=self.cmd_response)
op = Output(self.command, response)
with open(self.obtained_output_file, "w") as obtained_output:
op.dump(out=obtained_output,
err=sys.stderr,
response=response,
output_type=self.command.options.get("format"),
**self.command.options.get("output"))
with open(self.expected_output_file) as expected_output:
with open(self.obtained_output_file) as obtained_output:
self.assertEqual(expected_output.readlines(),
obtained_output.readlines())
async def _generate_bundle(command):
"""
Initializes the process for Generating Support Bundle on Each CORTX Node.
command: Command Object :type: command
return: None.
"""
bundle_id = SupportBundle._generate_bundle_id()
provisioner = ProvisionerServices()
if not provisioner:
return Response(output="Provisioner package not found.", \
rc=errno.ENOENT)
# Get Arguments From Command
comment = command.options.get(const.SB_COMMENT)
components = command.options.get(const.SB_COMPONENTS)
if not components:
components = []
if command.options.get(const.SOS_COMP, False) == 'true':
components.append('os')
Conf.load('cortx_conf', 'json:///etc/cortx/cortx.conf', \
skip_reload=True)
# Get HostNames and Node Names.
node_hostname_map = await SupportBundle._get_active_nodes()
if not isinstance(node_hostname_map, dict):
return node_hostname_map
shared_path = Storage.get_path(name='support_bundle')
path = shared_path if shared_path else Conf.get('cortx_conf',\
'support>local_path')
bundle_path = os.path.join(path, bundle_id)
os.makedirs(bundle_path)
bundle_obj = Bundle(bundle_id=bundle_id, bundle_path=path, \
comment=comment,is_shared=True if shared_path else False)
support_bundle_file = os.path.join(Conf.get('cortx_conf', 'install_path'),\
'cortx/utils/conf/support_bundle.yaml')
Conf.load('sb_yaml', f'yaml://{support_bundle_file}', skip_reload=True)
all_components = Conf.get('sb_yaml', 'COMPONENTS')
invalid_comps = [
component for component in components
if component not in all_components.keys()
]
if invalid_comps:
components = list(set(components) - set(invalid_comps))
ComponentsBundle._publish_log(f"""Invalid components - '{", ".join(invalid_comps)}'""", \
'error', bundle_id, '', comment)
if invalid_comps and not components:
return bundle_obj
comp_list = SupportBundle._get_components(components)
# Start SB Generation on all Nodes.
for nodename, hostname in node_hostname_map.items():
Log.debug(f"Connect to {hostname}")
try:
# TODO: pass bundle_path to bundle_generation when args implemented
await provisioner.begin_bundle_generation(
f"bundle_generate '{bundle_id}' '{comment}' "
f"'{hostname}' {comp_list}", nodename)
except BundleError as be:
Log.error(f"Bundle generation failed.{be}")
ComponentsBundle._publish_log(f"Bundle generation failed.{be}", \
'error', bundle_id, nodename, comment)
except Exception as e:
Log.error(f"Internal error, bundle generation failed {e}")
ComponentsBundle._publish_log(
f"Internal error, bundle generation failed \
{e}", 'error', bundle_id, nodename, comment)
# Create common tar.
#if bundle_obj.is_shared:
# tar_dest_file = f"{bundle_id}.tar.gz"
# Log.debug(f"Merging all bundle to {bundle_path}/{tar_dest_file}")
# try:
# Tar(os.path.join(bundle_path, tar_dest_file)).dump([bundle_path])
# except:
# Log.debug("Merging of node support bundle failed")
# return Response(output="Bundle Generation Failed in merging",
# rc=errno.EINVAL)
if command.sub_command_name == 'generate':
display_string_len = len(bundle_obj.bundle_id) + 4
response_msg = (
f"Please use the below bundle id for checking the status of support bundle."
f"\n{'-' * display_string_len}"
f"\n| {bundle_obj.bundle_id} |"
f"\n{'-' * display_string_len}"
f"\nPlease Find the file on -> {bundle_obj.bundle_path} .\n")
return Response(output=response_msg, rc=OPERATION_SUCESSFUL)
return bundle_obj
async def _generate_bundle(command):
"""
Initializes the process for Generating Support Bundle at shared path.
command: Command Object :type: command
return: None.
"""
# Get Arguments From Command
bundle_id = command.options.get(const.SB_BUNDLE_ID)
comment = command.options.get(const.SB_COMMENT)
duration = command.options.get(const.SB_DURATION)
size_limit = command.options.get(const.SB_SIZE)
config_url = command.options.get('config_url')
binlogs = command.options.get('binlogs')
coredumps = command.options.get('coredumps')
stacktrace = command.options.get('stacktrace')
components = command.options.get('components')
config_path = config_url.split('//')[1] if '//' in config_url else ''
path = command.options.get('target_path')
bundle_path = os.path.join(path, bundle_id)
try:
os.makedirs(bundle_path)
except FileExistsError:
raise BundleError(
errno.EINVAL, "Bundle ID already exists,"
"Please use Unique Bundle ID")
cluster_conf = MappedConf(config_url)
# Get Node ID
node_id = Conf.machine_id
if node_id is None:
raise BundleError(errno.EINVAL, "Invalid node_id: %s", \
node_id)
# Update SB status in Filestore.
# load conf for Support Bundle
Conf.load(const.SB_INDEX,
'json://' + const.FILESTORE_PATH,
skip_reload=True)
data = {
'status': 'In-Progress',
'start_time': datetime.strftime(datetime.now(),
'%Y-%m-%d %H:%M:%S')
}
Conf.set(const.SB_INDEX, f'{node_id}>{bundle_id}', data)
Conf.save(const.SB_INDEX)
node_name = cluster_conf.get(f'node>{node_id}>name')
Log.info(f'Starting SB Generation on {node_id}:{node_name}')
# Get required SB size per component
components_list, service_per_comp = SupportBundle._get_component_and_services(
cluster_conf, node_id, components)
if not components_list:
Log.warn(f"No component specified for {node_name} in CORTX config")
Log.warn(f"Skipping SB generation on node:{node_name}.")
return
num_components = len(components_list)
size_limit_per_comp = SupportBundle.get_component_size_limit(
size_limit, num_components)
bundle_obj = Bundle(bundle_id=bundle_id, bundle_path=bundle_path, \
comment=comment,node_name=node_name, components=components_list,
services=service_per_comp)
# Start SB Generation on Node.
# Adding CORTX manifest data inside support Bundle.
try:
# Copying config file into support bundle.
common_locations = set()
if config_path and os.path.exists(config_path):
Log.info(f'For manifest data collection, taking config from \
{config_path} location.')
# Remove secrets from the input config.
conf_name = config_path.split('/')[-1]
sb_config = config_path.replace(conf_name, 'sb_cluster.conf')
with open(sb_config, 'w+') as sb_file:
with open(config_path, 'r') as f:
content = f.read()
if 'secret:' in content:
content = re.sub(r'secret:.+', r'secret: ****',
content)
sb_file.write(content)
conf_target = os.path.join(bundle_path, 'common' + config_path)
os.makedirs(conf_target.replace(f'/{conf_name}', ''),
exist_ok=True)
shutil.move(sb_config, conf_target)
common_locations.add(config_path.split('/')[1])
# Copying "/etc/cortx/solution" directory into support bundle
# except for "secret" folder.
sln_target = os.path.join(bundle_path, 'common' + const\
.CORTX_SOLUTION_DIR)
if os.path.exists(sln_target):
shutil.rmtree(sln_target)
if os.path.exists(const.CORTX_SOLUTION_DIR):
_ = shutil.copytree(const.CORTX_SOLUTION_DIR, sln_target, \
ignore=shutil.ignore_patterns('secret'))
common_locations.add(const.CORTX_SOLUTION_DIR.split('/')[1])
# Copying RELEASE.INFO file into support bundle.
if os.path.exists(const.CORTX_RELEASE_INFO):
rel_target = os.path.join(bundle_path, 'common' + const\
.CORTX_RELEASE_INFO)
os.makedirs(rel_target.replace('/RELEASE.INFO', ''),
exist_ok=True)
shutil.copyfile(const.CORTX_RELEASE_INFO, rel_target)
common_locations.add(const.CORTX_RELEASE_INFO.split('/')[1])
else:
Log.warn(f'{const.CORTX_RELEASE_INFO} file not found.')
# Adding node resources health into the support bundle.
health_target = os.path.join(bundle_path, 'common' + '/health')
os.makedirs(health_target, exist_ok=True)
with open(health_target + '/node_health.json', 'w') as fp:
info = {}
info["resource_usage"] = {}
info["resource_usage"]["cpu_usage"] = SupportBundle.\
get_cpu_overall_usage()
info["resource_usage"]["uptime"] = SupportBundle.\
get_system_uptime()
info["resource_usage"]["disk_usage"] = SupportBundle.\
get_disk_overall_usage()
info["resource_usage"]["memory_usage"] = SupportBundle.\
get_mem_overall_usage()
json.dump(info, fp, indent=4)
common_locations.add('health')
try:
common_path = os.path.join(bundle_path, 'common')
common_tar = os.path.join(common_path, 'common.tar.gz')
with tarfile.open(common_tar, "w:gz") as tar:
if os.path.exists(common_path):
tar.add(common_path, arcname='common')
# Deleting untar directories from the common folder.
for location in common_locations:
untar_location = os.path.join(common_path, location)
if os.path.exists(untar_location):
shutil.rmtree(untar_location)
except (OSError, tarfile.TarError) as err:
Log.error(
"Facing issues while adding manifest data into common "
"directory: {0}".format(err))
except BundleError as be:
Log.error(
f"Failed to add CORTX manifest data inside Support Bundle.{be}"
)
try:
await ComponentsBundle.init(bundle_obj,
node_id,
config_url,
duration=duration,
size_limit=size_limit_per_comp,
binlogs=binlogs,
coredumps=coredumps,
stacktrace=stacktrace)
except BundleError as be:
Log.error(f"Bundle generation failed.{be}")
except Exception as e:
Log.error(f"Internal error, bundle generation failed {e}")
if command.sub_command_name == 'generate':
display_string_len = len(bundle_obj.bundle_id) + 4
response_msg = (
f"Please use the below bundle id for checking the status of support bundle."
f"\n{'-' * display_string_len}"
f"\n| {bundle_obj.bundle_id} |"
f"\n{'-' * display_string_len}"
f"\nPlease Find the file on -> {bundle_obj.bundle_path} .\n")
return Response(output=response_msg, rc=OPERATION_SUCESSFUL)
return bundle_obj