def generate(bundle_id: str, target_path: str, cluster_conf_url: str,
**filters):
"""Generate a tar file."""
# duration = filters.get('duration', 'P5D')
# size_limit = filters.get('size_limit', '500MB')
# binlogs = filters.get('binlogs', False)
# coredumps = filters.get('coredumps', False)
# stacktrace = filters.get('stacktrace', False)
# TODO process duration, size_limit, binlogs, coredumps and stacktrace
# Find log dirs
cluster_conf = MappedConf(cluster_conf_url)
log_base = cluster_conf.get(CLUSTER_CONF_LOG_KEY)
local_base = cluster_conf.get(
f'cortx{_DELIM}common{_DELIM}storage{_DELIM}local')
machine_id = Conf.machine_id
if os.path.exists(_sb_tmp_src):
HASupportBundle.__clear_tmp_files()
else:
os.makedirs(_sb_tmp_src)
# Copy log files
shutil.copytree(os.path.join(log_base, f'ha/{machine_id}'),\
os.path.join(_sb_tmp_src, 'logs'))
# Copy configuration files
shutil.copytree(os.path.join(local_base, 'ha'),\
os.path.join(_sb_tmp_src, 'conf'))
HASupportBundle.__generate_tar(bundle_id, target_path)
HASupportBundle.__clear_tmp_files()
def main(argv: dict):
try:
desc = "CORTX Kafka Setup command"
command = Cmd.get_command(desc, argv[1:])
# Get kafka server list from template file
kafka_config = 'kafka_config'
Conf.load(kafka_config, command.url)
kafka_servers = Conf.get(kafka_config, 'cortx>software>kafka>servers')
# Get log path and initialise Log
cluster_conf = MappedConf(command.cluster_conf)
log_dir = cluster_conf.get(CLUSTER_CONF_LOG_KEY)
log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}')
log_level = cluster_conf.get('utils>log_level', 'INFO')
Log.init('kafka_setup', log_path, level=log_level, backup_count=5,\
file_size_in_mb=5)
rc = command.process(kafka_servers)
if rc != 0:
raise ValueError(f"Failed to run {argv[1]}")
except KafkaSetupError as e:
sys.stderr.write("%s\n" % str(e))
Cmd.usage(argv[0])
return e.rc()
except Exception as e:
sys.stderr.write("error: %s\n\n" % str(e))
sys.stderr.write("%s\n" % traceback.format_exc())
Cmd.usage(argv[0])
return errno.EINVAL
def _provision_components(cortx_conf: MappedConf, interfaces: Enum,
apply_phase: str):
"""Invoke Mini Provisioners of cluster components."""
node_id, node_name = CortxProvisioner._get_node_info(cortx_conf)
num_components = int(cortx_conf.get(f'node>{node_id}>num_components'))
for interface in interfaces:
for comp_idx in range(0, num_components):
key_prefix = f'node>{node_id}>components[{comp_idx}]'
component_name = cortx_conf.get(f'{key_prefix}>name')
# Get services.
service_idx = 0
services = []
while (cortx_conf.get(f'{key_prefix}>services[{service_idx}]')
is not None):
services.append(
cortx_conf.get(
f'{key_prefix}>services[{service_idx}]'))
service_idx = service_idx + 1
service = 'all' if service_idx == 0 else ','.join(services)
if apply_phase == ProvisionerStages.UPGRADE.value:
version = cortx_conf.get(f'{key_prefix}>version')
# Skip update for component if it is already updated.
is_updated = CortxProvisioner._is_component_updated(
component_name, version)
if is_updated is True:
Log.info(
f'{component_name} is already updated with {version} version.'
)
continue
CortxProvisioner._update_provisioning_status(
cortx_conf, node_id, apply_phase,
ProvisionerStatus.PROGRESS.value)
cmd = (
f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}"
f" --config {cortx_conf._conf_url} --services {service}")
Log.info(f"{cmd}")
cmd_proc = SimpleProcess(cmd)
_, err, rc = cmd_proc.run()
if rc != 0:
CortxProvisioner._update_provisioning_status(
cortx_conf, node_id, apply_phase,
ProvisionerStatus.ERROR.value)
raise CortxProvisionerError(rc,
"%s phase of %s, failed. %s",
interface.value,
component_name, err)
# Update version for each component if Provisioning successful.
component_version = CortxProvisioner.cortx_release.get_component_version(
component_name)
cortx_conf.set(f'{key_prefix}>version', component_version)
def _get_node_list(version_conf: MappedConf):
"""
Get list of node Id.
Parameters:
version_conf - ConfStore instance of Gconf.
"""
node_list = []
num_storage_set = int(version_conf.get(const.NUM_STORAGESET_KEY))
for storage_set_idx in range(0, num_storage_set):
num_nodes = int(
version_conf.get(const.NUM_NODES_KEY % storage_set_idx))
for node_idx in range(0, num_nodes):
# Add node id to node list. ex: cluster>storage_set[storage_set_idx]>nodes[node_idx].
# node list: [ '0cdf725015124cbbbc578114dbc51982' ].
node_list.append(
version_conf.get(const.NODE_ID_KEY %
(storage_set_idx, node_idx)))
return node_list
def main():
from cortx.utils.log import Log
from cortx.utils.conf_store import Conf
# Setup Parser
parser = argparse.ArgumentParser(description='Cortx Support Bundle Interface', \
formatter_class=RawTextHelpFormatter)
sub_parser = parser.add_subparsers(title='command', \
help='represents the action from: generate, get_status\n\n', \
dest='command')
# Add Command Parsers
members = inspect.getmembers(sys.modules[__name__])
for name, cls in members:
if name != "Cmd" and name.endswith("Cmd"):
cls.add_args(sub_parser)
# Parse and Process Arguments
try:
args = parser.parse_args()
cluster_conf_path = args.cluster_conf_path[0]
cluster_conf = MappedConf(cluster_conf_path)
log_path = os.path.join(cluster_conf.get(CLUSTER_CONF_LOG_KEY), \
f'utils/{Conf.machine_id}/support')
log_level = cluster_conf.get('utils>log_level', 'INFO')
Log.init('support_bundle', log_path, level=log_level, backup_count=5, \
file_size_in_mb=5)
out = args.func(args)
if out is not None and len(out) > 0:
print(out)
return 0
except Exception as e:
sys.stderr.write("%s\n\n" % str(e))
sys.stderr.write("%s\n" % traceback.format_exc())
return errno.EINVAL
def _get_node_info(cortx_conf: MappedConf):
"""To get the node information."""
node_id = Conf.machine_id
if node_id is None:
raise CortxProvisionerError(errno.EINVAL, "Invalid node_id: %s", \
node_id)
# Reinitialize logging with configured log path
log_path = os.path.join(cortx_conf.get('cortx>common>storage>log'),
const.APP_NAME, node_id)
log_level = os.getenv('CORTX_PROVISIONER_DEBUG_LEVEL',
const.DEFAULT_LOG_LEVEL)
CortxProvisionerLog.reinitialize(const.SERVICE_NAME,
log_path,
level=log_level)
if cortx_conf.get(f'node>{node_id}>name') is None:
raise CortxProvisionerError(
errno.EINVAL,
f'Node name not found in cortx config for node {node_id}.')
node_name = cortx_conf.get(f'node>{node_id}>name')
return node_id, node_name
def get_installed_version(resource_id: str, conf_url: str):
"""
Get current deployed versions on the node.
Parameters:
resource_id - Name of the node (as in gconf).
conf_url - Global Configuration URL.
"""
node_id = None
version_info = {}
version_conf = MappedConf(conf_url)
# Get list of all the nodes in the cluster.
node_list = Release._get_node_list(version_conf)
for node in node_list:
if resource_id == version_conf.get(const.NODE_NAME_KEY % node):
node_id = node
if node_id is None:
raise SetupError(errno.EINVAL,
"Invalid Resource Id %s." % resource_id)
# Get version details of all the components of a node
num_components = int(
version_conf.get(const.NUM_COMPONENTS_KEY % node_id))
for component in range(0, num_components):
_name = version_conf.get(const.COMPONENT_NAME_KEY %
(node_id, component))
_version = version_conf.get(const.COMPONENT_VERSION_KEY %
(node_id, component))
if _version is not None:
version_info[_name] = _version
else:
raise SetupError(
INTERNAL_ERROR,
"No installed version found for component %s" % _name)
# get cluster release version
release_name = version_conf.get(const.RELEASE_NAME_KEY)
release_version = version_conf.get(const.RELEASE_VERSION_KEY)
version_info[release_name] = release_version
return version_info if len(version_info) > 1 else 0
web.post('/AuditLog/webhook/', \
AuditLogRequestHandler.send_webhook_info)
])
super().run_app(web, message_server_port)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Utils server CLI',
formatter_class=RawTextHelpFormatter)
parser.add_argument('-c', '--config', dest='cluster_conf',\
help="Cluster config file path for Support Bundle")
args=parser.parse_args()
cluster_conf_url = args.cluster_conf
cluster_conf = MappedConf(cluster_conf_url)
# Get the log path
log_dir = cluster_conf.get(CLUSTER_CONF_LOG_KEY)
if not log_dir:
raise UtilsServerError(errno.EINVAL, "Fail to initialize logger."+\
" Unable to find log_dir path entry")
utils_log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}/utils_server')
# Get the log level
log_level = cluster_conf.get('utils>log_level', 'INFO')
Log.init('utils_server', utils_log_path, level=log_level, backup_count=5, \
file_size_in_mb=5)
message_bus_backend = cluster_conf.get('cortx>utils>message_bus_backend')
message_server_endpoints = cluster_conf.get(
f'cortx>external>{message_bus_backend}>endpoints')
message_server_port = cluster_conf.get('cortx>utils>message_server_port', 28300)
cluster_id = cluster_conf.get('cluster>id')
MessageServer(message_server_endpoints, message_server_port, cluster_id)
import re
import time
import errno
import psutil
from datetime import datetime
from cortx.utils import const
from cortx.utils.conf_store import MappedConf
from cortx.utils.kv_store import KvStoreFactory
from cortx.utils.discovery.error import DiscoveryError
from cortx.utils.discovery.resource import Resource, ResourceFactory
# Load cortx common config
store_type = "json"
cluster_conf = MappedConf(const.CLUSTER_CONF)
local_storage_path = cluster_conf.get('cortx>common>storage>local')
config_url = "%s://%s" % (
store_type, os.path.join(local_storage_path, 'utils/conf/cortx.conf'))
common_config = KvStoreFactory.get_instance(config_url)
common_config.load()
# Load Discovery request status tracker
try:
os.makedirs(common_config.get(["discovery>resource_map>location"])[0],
exist_ok=True)
except PermissionError as err:
raise DiscoveryError(errno.EACCES,
"Failed to create default store directory. %s" % err)
requests_url = "%s://%s" % (store_type,
os.path.join(
common_config.get([
def config_apply(solution_config_url: str, cortx_conf_url: str = None,
force_override: bool = False):
"""
Description:
Parses input config and store in CORTX config location
Parameters:
[IN] Solution Config URL
[OUT] CORTX Config URL
"""
if Log.logger is None:
CortxProvisionerLog.initialize(const.SERVICE_NAME, const.TMP_LOG_PATH)
if cortx_conf_url is None:
cortx_conf_url = CortxProvisioner._cortx_conf_url
cortx_conf = MappedConf(CortxProvisioner._tmp_cortx_conf_url)
# Load same config again if force_override is True
try:
cs_option = {"fail_reload": False} if force_override else {"skip_reload": True}
Log.info('Applying config %s' % solution_config_url)
Conf.load(CortxProvisioner._solution_index, solution_config_url,
**cs_option)
except ConfError as e:
Log.error(f'Unable to load {solution_config_url} url, Error:{e}')
# Secrets path from config file
if cortx_conf.get('cortx>common>storage>local'):
CortxProvisioner._secrets_path = cortx_conf.get('cortx>common>storage>local')+CortxProvisioner._rel_secret_path
# source code for encrypting and storing secret key
if Conf.get(CortxProvisioner._solution_index, 'cluster') is not None:
CortxProvisioner.apply_cluster_config(cortx_conf, CortxProvisioner.cortx_release)
if Conf.get(CortxProvisioner._solution_index, 'cortx') is not None:
# generating cipher key
cipher_key = None
cluster_id = Conf.get(CortxProvisioner._solution_index, 'cluster>id')
if cluster_id is None:
cluster_id = cortx_conf.get('cluster>id')
if cluster_id is None:
raise CortxProvisionerError(errno.EINVAL, 'Cluster ID not specified')
cipher_key = Cipher.gen_key(cluster_id, 'cortx')
if cipher_key is None:
raise CortxProvisionerError(errno.EINVAL, 'Cipher key not specified')
for key in Conf.get_keys(CortxProvisioner._solution_index):
# using path /etc/cortx/solution/secret to confirm secret
if key.endswith('secret'):
secret_val = Conf.get(CortxProvisioner._solution_index, key)
val = None
with open(os.path.join(CortxProvisioner._secrets_path, secret_val), 'rb') as secret:
val = secret.read()
if val is None:
raise CortxProvisionerError(errno.EINVAL,
f'Could not find the Secret in {CortxProvisioner._secrets_path}')
val = Cipher.encrypt(cipher_key, val)
# decoding the byte string in val variable
Conf.set(CortxProvisioner._solution_index, key, val.decode('utf-8'))
CortxProvisioner.apply_cortx_config(cortx_conf, CortxProvisioner.cortx_release)
# Adding array count key in conf
cortx_conf.add_num_keys()
Conf.save(cortx_conf._conf_idx)
async def init(bundle_obj, node_id, config_url, **kwargs):
"""
Initializes the Process of Support Bundle Generation for Every Component.
command: cli Command Object :type: command
return: None
"""
cluster_conf = MappedConf(config_url)
log_path = os.path.join(cluster_conf.get(CLUSTER_CONF_LOG_KEY), \
f'utils/{Conf.machine_id}/support')
log_level = cluster_conf.get('utils>log_level', 'INFO')
Log.init('support_bundle_node', log_path, level=log_level, \
backup_count=5, file_size_in_mb=5)
bundle_id = bundle_obj.bundle_id
node_name = bundle_obj.node_name
comment = bundle_obj.comment
components_list = bundle_obj.components
services_dict = bundle_obj.services
Log.info(f"components:{components_list}")
bundle_path = bundle_obj.bundle_path
duration = kwargs.get('duration')
size_limit = kwargs.get('size_limit')
binlogs = kwargs.get('binlogs')
coredumps = kwargs.get('coredumps')
stacktrace = kwargs.get('stacktrace')
Log.debug((f"{const.SB_BUNDLE_ID}: {bundle_id}, {const.SB_NODE_NAME}: "
f"{node_name}, {const.SB_COMMENT}: {comment}, "
f"{const.SB_COMPONENTS}: {components_list}, {const.SOS_COMP}"))
# Read support_bundle.Yaml and Check's If It Exists.
cmd_setup_file = os.path.join(
cluster_conf.get('install_path', DEFAULT_INSTALL_PATH),
const.SUPPORT_YAML)
try:
support_bundle_config = Yaml(cmd_setup_file).load()
except Exception as e:
Log.error(f"Internal error while parsing YAML file {cmd_setup_file}{e}")
if not support_bundle_config:
Log.error(f"No such file {cmd_setup_file}. ERROR:{ERROR}")
# Start Execution for each Component Command.
command_files_info = support_bundle_config.get('COMPONENTS')
# OS Logs are specifically generated hence here Even
# When All is Selected O.S. Logs Will Be Skipped.
for each_component in components_list:
services = services_dict[each_component]
components_commands = []
components_files = command_files_info[each_component]
for file_path in components_files:
if not os.path.exists(file_path):
Log.error(f"'{file_path}' file does not exist!")
continue
try:
file_data = Yaml(file_path).load()
except Exception as e:
Log.error(f"Internal error while parsing YAML file {file_path}{e}")
file_data = None
break
if file_data:
components_commands = file_data.get(
const.SUPPORT_BUNDLE.lower(), [])
else:
Log.error(f"Support.yaml file is empty: {file_path}")
break
if components_commands:
component, return_code = await(\
ComponentsBundle._exc_components_cmd(\
components_commands, f'{bundle_id}_{node_id}_{each_component}',
f'{bundle_path}{os.sep}', each_component,
node_name, comment, config_url, services,
binlogs, coredumps, stacktrace, duration,
size_limit))
if return_code != 0:
Log.error(
f"Bundle generation failed for component - '{component}'")
else:
Log.info(
f"Bundle generation started for component - '{component}'")
tar_file_name = os.path.join(bundle_path, \
f'{bundle_id}_{node_id}.tar.gz')
ComponentsBundle._create_summary_file(bundle_id, node_name, \
comment, bundle_path)
try:
Log.debug(f"Generating tar.gz file on path {tar_file_name} "
f"from {bundle_path}")
Tar(tar_file_name).dump([bundle_path])
bundle_status = f"Successfully generated SB at path:{bundle_path}"
except Exception as e:
bundle_status = f"Failed to generate tar file. ERROR:{e}"
Log.error(f"Could not generate tar file {e}")
finally:
if os.path.exists(bundle_path):
for each_dir in os.listdir(bundle_path):
comp_dir = os.path.join(bundle_path, each_dir)
if os.path.isdir(comp_dir):
shutil.rmtree(comp_dir)
if os.path.exists(os.path.join(bundle_path, 'summary.yaml')):
os.remove(os.path.join(bundle_path, 'summary.yaml'))
Log.info("Support bundle generation completed.")
# Update Status in ConfStor
Conf.load(const.SB_INDEX, 'json://' + const.FILESTORE_PATH, fail_reload=False)
Conf.set(const.SB_INDEX, f'{node_id}>{bundle_id}>status', bundle_status)
Conf.save(const.SB_INDEX)
def _validate_provisioning_status(cortx_conf: MappedConf, node_id: str,
apply_phase: str):
"""Validate provisioning."""
ret_code = 0
recent_phase = cortx_conf.get(f'node>{node_id}>provisioning>phase')
recent_status = cortx_conf.get(f'node>{node_id}>provisioning>status')
msg = f'Recent phase for this node is {recent_phase} and ' + \
f'recent status is {recent_status}. '
# {apply_phase: {recent_phase: {recent_status: [boolean_result,rc]}}}
validations_checks = {
ProvisionerStages.DEPLOYMENT.value: {
ProvisionerStages.DEPLOYMENT.value: {
ProvisionerStatus.DEFAULT.value: [True, 0],
ProvisionerStatus.ERROR.value: [True, 0],
ProvisionerStatus.PROGRESS.value: [True, 0],
ProvisionerStatus.SUCCESS.value: [False, 0]
},
ProvisionerStages.UPGRADE.value: {
ProvisionerStatus.DEFAULT.value: [True, 0],
ProvisionerStatus.ERROR.value: [True, 0],
ProvisionerStatus.PROGRESS.value: [True, 0],
ProvisionerStatus.SUCCESS.value: [True, 0]
}
},
ProvisionerStages.UPGRADE.value: {
ProvisionerStages.DEPLOYMENT.value: {
ProvisionerStatus.DEFAULT.value: [False, 1],
ProvisionerStatus.ERROR.value: [False, 1],
ProvisionerStatus.PROGRESS.value: [False, 1],
ProvisionerStatus.SUCCESS.value: [True, 0]
},
ProvisionerStages.UPGRADE.value: {
ProvisionerStatus.DEFAULT.value: [True, 0],
ProvisionerStatus.ERROR.value: [True, 0],
ProvisionerStatus.PROGRESS.value: [True, 0],
ProvisionerStatus.SUCCESS.value: [True, 0]
}
}
}
if recent_phase is None and recent_status is None:
Log.info(msg + f'Performing {apply_phase} on this node.')
return True, ret_code
if (not validations_checks.get(apply_phase)
or not validations_checks.get(apply_phase).get(recent_phase)
or not validations_checks.get(apply_phase).get(
recent_phase).get(recent_status)):
Log.error('Invalid phase or status.')
ret_code = 1
return False, ret_code
validate_result = validations_checks.get(apply_phase).get(
recent_phase).get(recent_status)
if validate_result[1] != 0:
Log.error(msg + f'{apply_phase} is not possible on this node.')
if apply_phase == ProvisionerStages.UPGRADE.value:
# Reset status.
recent_status = cortx_conf.set(
f'node>{node_id}>provisioning>status',
ProvisionerStatus.DEFAULT.value)
else:
Log.info(msg)
return validate_result[0], validate_result[1]