def save(self, cortx_conf, cortx_solution_config):
"""Save cortx-config into confstore"""
try:
cortx_solution_config_keys = filter(
lambda x: x.startswith('cortx'),
Conf.get_keys(cortx_solution_config))
cortx_conf.copy(cortx_solution_config, cortx_solution_config_keys)
# Change environment_type to setup_type
# TODO: remove this code once setup_type key is deleted.
cortx_conf.set('cortx>common>setup_type',
cortx_conf.get('cortx>common>environment_type'))
cortx_conf.delete('cortx>common>environment_type')
# Check for release key.
release_spec = self._cortx_solution_config.get('common').get(
'release')
is_valid, release_info = self._cortx_release.validate(release_spec)
if is_valid is False:
for key in release_info.keys():
release_key_path = f'cortx>common>release>{key}'
Log.warn(
f'Release key {release_key_path} is missing or has '
'incorrect value.')
Log.info(f'Adding key "{release_key_path}" '
f'and value "{release_info[key]}" in confstore.')
cortx_conf.set(release_key_path, release_info[key])
except KeyError as e:
raise CortxProvisionerError(
errno.EINVAL,
f'Error occurred while adding CORTX config information into confstore {e}'
)
def _provision_components(cortx_conf_url: str, _conf_idx: str, interfaces: Enum, apply_phase: str):
"""Invoke Mini Provisioners of cluster components."""
node_id, _ = CortxProvisioner._get_node_info(_conf_idx)
num_components = int(Conf.get(_conf_idx, f'node>{node_id}>num_components'))
for interface in interfaces:
for comp_idx in range(0, num_components):
key_prefix = f'node>{node_id}>components[{comp_idx}]'
component_name = Conf.get(_conf_idx, f'{key_prefix}>name')
# Check if RPM exists for the component, if it does exist get the build version
component_version = CortxProvisioner.cortx_release.get_component_version(
component_name)
# Get services.
service_idx = 0
services = []
while (Conf.get(_conf_idx, f'{key_prefix}>services[{service_idx}]') is not None):
services.append(Conf.get(_conf_idx, f'{key_prefix}>services[{service_idx}]'))
service_idx = service_idx + 1
service = 'all' if service_idx == 0 else ','.join(services)
if apply_phase == ProvisionerStages.UPGRADE.value:
version = Conf.get(_conf_idx, f'{key_prefix}>version')
# Skip update for component if it is already updated.
is_updated = CortxProvisioner._is_component_updated(component_name, version)
if is_updated is True:
Log.info(f'{component_name} is already updated with {version} version.')
continue
CortxProvisioner._update_provisioning_status(
_conf_idx, node_id, apply_phase, ProvisionerStatus.PROGRESS.value)
if interface.value == 'upgrade':
# TODO: add --changeset parameter once all components support config upgrade
cmd = (
f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}"
f" --config {cortx_conf_url} --services {service}")
else:
cmd = (
f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}"
f" --config {cortx_conf_url} --services {service}")
Log.info(f"{cmd}")
cmd_proc = SimpleProcess(cmd)
_, err, rc = cmd_proc.run()
if rc != 0:
CortxProvisioner._update_provisioning_status(
_conf_idx, node_id, apply_phase, ProvisionerStatus.ERROR.value)
raise CortxProvisionerError(
rc, "%s phase of %s, failed. %s", interface.value,
component_name, err)
# Update version for each component if Provisioning successful.
Conf.set(_conf_idx, f'{key_prefix}>version', component_version)
# TODO: Remove the following code when gconf is completely moved to consul.
CortxProvisioner._load_consul_conf(CortxProvisioner._cortx_gconf_consul_index)
Conf.set(CortxProvisioner._cortx_gconf_consul_index,
f'{key_prefix}>version', component_version)
Conf.save(CortxProvisioner._cortx_gconf_consul_index)
def _provision_components(cortx_conf: MappedConf, interfaces: Enum,
apply_phase: str):
"""Invoke Mini Provisioners of cluster components."""
node_id, node_name = CortxProvisioner._get_node_info(cortx_conf)
num_components = int(cortx_conf.get(f'node>{node_id}>num_components'))
for interface in interfaces:
for comp_idx in range(0, num_components):
key_prefix = f'node>{node_id}>components[{comp_idx}]'
component_name = cortx_conf.get(f'{key_prefix}>name')
# Get services.
service_idx = 0
services = []
while (cortx_conf.get(f'{key_prefix}>services[{service_idx}]')
is not None):
services.append(
cortx_conf.get(
f'{key_prefix}>services[{service_idx}]'))
service_idx = service_idx + 1
service = 'all' if service_idx == 0 else ','.join(services)
if apply_phase == ProvisionerStages.UPGRADE.value:
version = cortx_conf.get(f'{key_prefix}>version')
# Skip update for component if it is already updated.
is_updated = CortxProvisioner._is_component_updated(
component_name, version)
if is_updated is True:
Log.info(
f'{component_name} is already updated with {version} version.'
)
continue
CortxProvisioner._update_provisioning_status(
cortx_conf, node_id, apply_phase,
ProvisionerStatus.PROGRESS.value)
cmd = (
f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}"
f" --config {cortx_conf._conf_url} --services {service}")
Log.info(f"{cmd}")
cmd_proc = SimpleProcess(cmd)
_, err, rc = cmd_proc.run()
if rc != 0:
CortxProvisioner._update_provisioning_status(
cortx_conf, node_id, apply_phase,
ProvisionerStatus.ERROR.value)
raise CortxProvisionerError(rc,
"%s phase of %s, failed. %s",
interface.value,
component_name, err)
# Update version for each component if Provisioning successful.
component_version = CortxProvisioner.cortx_release.get_component_version(
component_name)
cortx_conf.set(f'{key_prefix}>version', component_version)
def cluster_deploy(cortx_conf_url: str, force_override: bool = False):
"""
Description:
Configures Cluster Components
1. Reads Cortx Config and obtain cluster components
2. Invoke Mini Provisioners of cluster components
Paramaters:
[IN] CORTX Config URL
"""
apply_phase = ProvisionerStages.DEPLOYMENT.value
node_id, node_name = CortxProvisioner._get_node_info(CortxProvisioner._conf_index)
is_valid, ret_code = CortxProvisioner._validate_provisioning_status(
CortxProvisioner._conf_index, node_id, apply_phase)
if is_valid is False:
if force_override is False:
Log.warn('Validation check failed, Aborting cluster bootstarp'
f' with return code {ret_code}')
return ret_code
else:
Log.info('Validation check failed, Forcefully overriding deployment.')
Log.info(f"Starting cluster bootstrap on {node_id}:{node_name}")
CortxProvisioner._update_provisioning_status(
CortxProvisioner._conf_index, node_id, apply_phase)
CortxProvisioner._provision_components(cortx_conf_url, CortxProvisioner._conf_index, DeploymentInterfaces, apply_phase)
CortxProvisioner._add_version_info(CortxProvisioner._conf_index, node_id)
CortxProvisioner._update_provisioning_status(
CortxProvisioner._conf_index, node_id, apply_phase, ProvisionerStatus.SUCCESS.value)
Log.info(f"Finished cluster bootstrap on {node_id}:{node_name}")
def cluster_upgrade(cortx_conf_url: str, force_override: bool = False):
"""
Description:
Upgrades Cluster Components
1. Reads Cortx Config and obtain cluster components
2. Invoke upgrade phase of cluster components
Paramaters:
[IN] CORTX Config URL
"""
cortx_conf = MappedConf(cortx_conf_url)
apply_phase = ProvisionerStages.UPGRADE.value
node_id, node_name = CortxProvisioner._get_node_info(cortx_conf)
is_valid, ret_code = CortxProvisioner._validate_provisioning_status(
cortx_conf, node_id, apply_phase)
if is_valid is False:
if force_override is False:
Log.warn('Validation check failed, Aborting upgrade with '
f'return code {ret_code}.')
return ret_code
else:
Log.info(
'Validation check failed, Forcefully overriding upgrade.')
Log.info(f"Starting cluster upgrade on {node_id}:{node_name}")
CortxProvisioner._update_provisioning_status(cortx_conf, node_id,
apply_phase)
CortxProvisioner._provision_components(cortx_conf, UpgradeInterfaces,
apply_phase)
# Update CORTX version, once the upgrade is successful
CortxProvisioner._add_version_info(cortx_conf, node_id)
CortxProvisioner._update_provisioning_status(
cortx_conf, node_id, apply_phase, ProvisionerStatus.SUCCESS.value)
Log.info(f"Finished cluster upgrade on {node_id}:{node_name}")
def _validate_provisioning_status(_conf_idx: str, node_id: str, apply_phase: str):
"""Validate provisioning."""
ret_code = 0
recent_phase = Conf.get(_conf_idx, f'node>{node_id}>provisioning>phase')
recent_status = Conf.get(_conf_idx, f'node>{node_id}>provisioning>status')
msg = f'Recent phase for this node is {recent_phase} and ' + \
f'recent status is {recent_status}. '
# {apply_phase: {recent_phase: {recent_status: [boolean_result,rc]}}}
validations_checks = {
ProvisionerStages.DEPLOYMENT.value: {
ProvisionerStages.DEPLOYMENT.value: {
ProvisionerStatus.DEFAULT.value: [True, 0],
ProvisionerStatus.ERROR.value: [True, 0],
ProvisionerStatus.PROGRESS.value: [True, 0],
ProvisionerStatus.SUCCESS.value: [False, 0]
},
ProvisionerStages.UPGRADE.value: {
ProvisionerStatus.DEFAULT.value: [True, 0],
ProvisionerStatus.ERROR.value: [True, 0],
ProvisionerStatus.PROGRESS.value: [True, 0],
ProvisionerStatus.SUCCESS.value: [True, 0]
}},
ProvisionerStages.UPGRADE.value: {
ProvisionerStages.DEPLOYMENT.value: {
ProvisionerStatus.DEFAULT.value: [False, 1],
ProvisionerStatus.ERROR.value: [False, 1],
ProvisionerStatus.PROGRESS.value: [False, 1],
ProvisionerStatus.SUCCESS.value: [True, 0]
},
ProvisionerStages.UPGRADE.value: {
ProvisionerStatus.DEFAULT.value: [True, 0],
ProvisionerStatus.ERROR.value: [True, 0],
ProvisionerStatus.PROGRESS.value: [True, 0],
ProvisionerStatus.SUCCESS.value: [True, 0]
}
}}
if recent_phase is None and recent_status is None:
Log.info(msg + f'Performing {apply_phase} on this node.')
return True, ret_code
if (not validations_checks.get(apply_phase) or
not validations_checks.get(apply_phase).get(recent_phase) or
not validations_checks.get(apply_phase).get(recent_phase).get(recent_status)):
Log.error('Invalid phase or status.')
ret_code = 1
return False, ret_code
validate_result = validations_checks.get(apply_phase).get(recent_phase).get(recent_status)
if validate_result[1] != 0:
Log.error(msg + f'{apply_phase} is not possible on this node.')
if apply_phase == ProvisionerStages.UPGRADE.value:
# Reset status.
recent_status = Conf.set(_conf_idx, f'node>{node_id}>provisioning>status',
ProvisionerStatus.DEFAULT.value)
else:
Log.info(msg)
return validate_result[0], validate_result[1]
def _validate(self, s_set: dict):
"""
validates a give storage_sets to have required properties
Raises exception if there is any entry missing
"""
s_set_name = s_set.get('name')
if s_set_name is None:
raise CortxProvisionerError(
errno.EINVAL, 'Missing name for the storage_set entry')
Log.debug("Validating storage set '%s' properties" % s_set_name)
required_keys_for_storage_set = ['durability', 'nodes']
for k in required_keys_for_storage_set:
if s_set.get(k) is None:
raise VError(
errno.EINVAL,
f"'{k}' property is unspecified for storage_set {s_set_name}."
)
def _check_storage_sets(self):
"""Validate storage_sets present in cortx_conf."""
solution_config_storage_sets = self._get_config(
ConfigValidator._key_storage_set_sc)
storage_set_counter = 0
while self.cortx_conf.get(
f'cluster>storage_set[{storage_set_counter}]>name'
) is not None:
storage_set_counter = storage_set_counter + 1
if len(solution_config_storage_sets) != storage_set_counter:
Log.debug(
f'Number of storage_sets define in {self.solution_conf_url} is '
f'{len(solution_config_storage_sets)} and in {self.cortx_conf_url} '
f'is {storage_set_counter}')
raise CortxProvisionerError(
errno.EINVAL,
f'Number of storage_sets define in {self.cortx_conf_url} '
f'and {self.solution_conf_url} is not equal.')
return 0
def _check_number_of_nodes(self):
"""Validate number of nodes specified in cortx_conf."""
solution_config_storage_sets = self._get_config(
ConfigValidator._key_storage_set_sc)
for storage_set in solution_config_storage_sets:
storage_set_name = storage_set['name']
solution_config_nodes = storage_set['nodes']
# Get number of nodes from confstore which has same storage_set_name.
conf_store_nodes = self.cortx_conf.search('node', 'storage_set',
storage_set_name)
if len(solution_config_nodes) != len(conf_store_nodes):
Log.debug(
f'Number of nodes define in {self.solution_conf_url} is '
f'{len(solution_config_nodes)} and {self.cortx_conf_url} is '
f'{len(conf_store_nodes)}')
raise CortxProvisionerError(
errno.EINVAL,
f'Number of nodes define in {self.cortx_conf_url} and '
f'{self.solution_conf_url} is not equal.')
return 0
def _validate(self, node: dict):
"""
validates a give node to have required properties
Raises exception if there is any entry missing
"""
node_name = node.get('name')
if node_name is None:
raise CortxProvisionerError(errno.EINVAL,
'Missing name for the node entry')
Log.debug("Validating node '%s' properties" % node_name)
required_keys_for_node = [
'id', 'components', 'storage_set', 'hostname'
]
for k in required_keys_for_node:
if node.get(k) is None:
raise CortxProvisionerError(
errno.EINVAL,
f"'{k}' property is unspecified for node {node_name}")
def _validate_services(self, service_list, component_name):
"""Verify services defined in cluster.yaml is supported in constant file."""
for service in service_list:
# check if service name is define in utils const.py file.
if service not in Const._value2member_map_:
raise CortxProvisionerError(errno.EINVAL,
f'{service} is not supported.')
# Verify service_name is define for the specific component.
# Get all keys from constant file which has same 'service_name'.
constant_service_keys = [
key for key, enum_ele in Const.__members__.items()
if enum_ele.value.lower() == service.lower()
]
if not any(component_name.upper() in key
for key in constant_service_keys):
Log.debug(
f'"{service}" service defined in "{self.solution_conf_url}" for '
f'"{component_name}", is not supported.')
raise CortxProvisionerError(
errno.EINVAL,
f'Component "{component_name}" does not support service "{service}".'
)
return 0
def cluster_upgrade(cortx_conf_url: str, force_override: bool = False):
"""
Description:
Upgrades Cluster Components
1. Reads Cortx Config and obtain cluster components
2. Invoke upgrade phase of cluster components
Paramaters:
[IN] CORTX Config URL
"""
# query to get cluster health
upgrade_mode = os.getenv(const.UPGRADE_MODE_KEY, const.UPGRADE_MODE_VAL).upper()
if upgrade_mode != "COLD" and not CortxProvisioner.is_cluster_healthy():
Log.error('Cluster is unhealthy, Aborting upgrade with return code 1')
return 1
apply_phase = ProvisionerStages.UPGRADE.value
node_id, node_name = CortxProvisioner._get_node_info(CortxProvisioner._conf_index)
is_valid, ret_code = CortxProvisioner._validate_provisioning_status(
CortxProvisioner._conf_index, node_id, apply_phase)
if is_valid is False:
if force_override is False:
Log.warn('Validation check failed, Aborting upgrade with '
f'return code {ret_code}.')
return ret_code
else:
Log.info('Validation check failed, Forcefully overriding upgrade.')
Log.info(f"Starting cluster upgrade on {node_id}:{node_name}")
CortxProvisioner._update_provisioning_status(
CortxProvisioner._conf_index, node_id, apply_phase)
CortxProvisioner._provision_components(cortx_conf_url, CortxProvisioner._conf_index, UpgradeInterfaces, apply_phase)
# Update CORTX version, once the upgrade is successful
CortxProvisioner._add_version_info(CortxProvisioner._conf_index, node_id)
CortxProvisioner._update_provisioning_status(
CortxProvisioner._conf_index, node_id, apply_phase, ProvisionerStatus.SUCCESS.value)
Log.info(f"Finished cluster upgrade on {node_id}:{node_name}")
def main():
CortxProvisionerLog.initialize(const.SERVICE_NAME, const.TMP_LOG_PATH)
try:
# Parse and Process Arguments
command = Cmd.get_command(sys.modules[__name__], 'cortx_setup', \
sys.argv[1:])
rc = command.process()
except CortxProvisionerError as e:
Log.error('%s' % str(e))
Log.error('%s' % traceback.format_exc())
rc = e.rc
except Exception as e:
Log.error('%s' % str(e))
Log.error('%s' % traceback.format_exc())
rc = errno.EINVAL
return rc
def config_apply(solution_config_url: str, cortx_conf_url: str = None,
force_override: bool = False):
"""
Description:
Parses input config and store in CORTX config location
Parameters:
[IN] Solution Config URL
[OUT] CORTX Config URL
"""
if Log.logger is None:
CortxProvisionerLog.initialize(const.SERVICE_NAME, const.TMP_LOG_PATH)
if cortx_conf_url is None:
cortx_conf_url = CortxProvisioner._cortx_conf_url
cortx_conf = MappedConf(CortxProvisioner._tmp_cortx_conf_url)
# Load same config again if force_override is True
try:
cs_option = {"fail_reload": False} if force_override else {"skip_reload": True}
Log.info('Applying config %s' % solution_config_url)
Conf.load(CortxProvisioner._solution_index, solution_config_url,
**cs_option)
except ConfError as e:
Log.error(f'Unable to load {solution_config_url} url, Error:{e}')
# Secrets path from config file
if cortx_conf.get('cortx>common>storage>local'):
CortxProvisioner._secrets_path = cortx_conf.get('cortx>common>storage>local')+CortxProvisioner._rel_secret_path
# source code for encrypting and storing secret key
if Conf.get(CortxProvisioner._solution_index, 'cluster') is not None:
CortxProvisioner.apply_cluster_config(cortx_conf, CortxProvisioner.cortx_release)
if Conf.get(CortxProvisioner._solution_index, 'cortx') is not None:
# generating cipher key
cipher_key = None
cluster_id = Conf.get(CortxProvisioner._solution_index, 'cluster>id')
if cluster_id is None:
cluster_id = cortx_conf.get('cluster>id')
if cluster_id is None:
raise CortxProvisionerError(errno.EINVAL, 'Cluster ID not specified')
cipher_key = Cipher.gen_key(cluster_id, 'cortx')
if cipher_key is None:
raise CortxProvisionerError(errno.EINVAL, 'Cipher key not specified')
for key in Conf.get_keys(CortxProvisioner._solution_index):
# using path /etc/cortx/solution/secret to confirm secret
if key.endswith('secret'):
secret_val = Conf.get(CortxProvisioner._solution_index, key)
val = None
with open(os.path.join(CortxProvisioner._secrets_path, secret_val), 'rb') as secret:
val = secret.read()
if val is None:
raise CortxProvisionerError(errno.EINVAL,
f'Could not find the Secret in {CortxProvisioner._secrets_path}')
val = Cipher.encrypt(cipher_key, val)
# decoding the byte string in val variable
Conf.set(CortxProvisioner._solution_index, key, val.decode('utf-8'))
CortxProvisioner.apply_cortx_config(cortx_conf, CortxProvisioner.cortx_release)
# Adding array count key in conf
cortx_conf.add_num_keys()
Conf.save(cortx_conf._conf_idx)