def write_role_manifest_vars(self, ansible_dir, role, kind):
enabled_kinds = {
"configuration/haproxy", "configuration/node-exporter"
}
if kind not in enabled_kinds:
return # skip
try:
cluster_model = select_single(
self.manifest_docs, lambda x: x.kind == 'epiphany-cluster')
except ExpectedSingleResultException:
return # skip
document = select_first(self.manifest_docs, lambda x: x.kind == kind)
if document is None:
# If there is no document provided by the user, then fallback to defaults
document = load_yaml_obj(types.DEFAULT, 'common', kind)
# Inject the required "version" attribute
document['version'] = VERSION
# Copy the "provider" value from the cluster model
document['provider'] = cluster_model['provider']
# Merge the document with defaults
with DefaultMerger([document]) as doc_merger:
document = doc_merger.run()[0]
self.write_role_vars(ansible_dir,
role,
document,
vars_file_name='manifest.yml')
def populate_group_vars(self, ansible_dir):
main_vars = ObjDict()
main_vars['admin_user'] = self.cluster_model.specification.admin_user
main_vars['validate_certs'] = Config().validate_certs
main_vars['offline_requirements'] = Config().offline_requirements
main_vars['wait_for_pods'] = Config().wait_for_pods
main_vars['is_upgrade_run'] = self.is_upgrade_run
main_vars['roles_with_generated_vars'] = sorted(
self.roles_with_generated_vars)
if self.is_upgrade_run:
shared_config_doc = self.get_shared_config_from_manifest()
else:
shared_config_doc = select_first(
self.config_docs,
lambda x: x.kind == 'configuration/shared-config')
if shared_config_doc is None:
shared_config_doc = load_yaml_obj(types.DEFAULT, 'common',
'configuration/shared-config')
self.set_vault_path(shared_config_doc)
main_vars.update(shared_config_doc.specification)
vars_dir = os.path.join(ansible_dir, 'group_vars')
if not os.path.exists(vars_dir):
os.makedirs(vars_dir)
vars_file_name = 'all.yml'
vars_file_path = os.path.join(vars_dir, vars_file_name)
with open(vars_file_path, 'a') as stream:
dump(main_vars, stream)
def run(self):
configuration_docs = []
def append_config(doc):
doc['version'] = VERSION
configuration_docs.append(doc)
for document_kind in ConfigurationAppender.REQUIRED_DOCS:
doc = select_first(self.input_docs, lambda x: x.kind == document_kind)
if doc is None:
doc = load_yaml_obj(types.DEFAULT, 'common', document_kind)
self.logger.info("Adding: " + doc.kind)
append_config(doc)
else:
append_config(doc)
for component_key, component_value in self.cluster_model.specification.components.items():
if component_value.count < 1:
continue
features_map = select_first(configuration_docs, lambda x: x.kind == 'configuration/feature-mapping')
config_selector = component_value.configuration
for feature_key in features_map.specification.roles_mapping[component_key]:
config = select_first(self.input_docs, lambda x: x.kind == 'configuration/' + feature_key and x.name == config_selector)
if config is not None:
append_config(config)
if config is None:
config = select_first(configuration_docs, lambda
x: x.kind == 'configuration/' + feature_key and x.name == config_selector)
if config is None:
config = merge_with_defaults('common', 'configuration/' + feature_key, config_selector)
self.logger.info("Adding: " + config.kind)
append_config(config)
return configuration_docs
def __init__(self, cluster_model, validation_docs):
super().__init__(__name__)
self.cluster_model = cluster_model
self.validation_docs = validation_docs
base = load_yaml_obj(types.VALIDATION, self.cluster_model.provider,
'core/base')
self.definitions = load_yaml_obj(types.VALIDATION,
self.cluster_model.provider,
'core/definitions')
self.base_schema = dict_to_objdict(deepcopy(base))
self.base_schema['definitions'] = self.definitions
self.base_schema_no_provider = dict_to_objdict(deepcopy(base))
self.base_schema_no_provider['definitions'] = self.definitions
del self.base_schema_no_provider.required[0]
del self.base_schema_no_provider.properties['provider']
def run(self):
for doc in self.validation_docs:
self.logger.info('Validating: ' + doc.kind)
schema = self.get_base_schema(doc.kind)
schema['specification'] = load_yaml_obj(
types.VALIDATION, self.cluster_model.provider, doc.kind)
if schema["specification"][
'$ref'] == '#/definitions/unvalidated_specification':
self.logger.warn('No specification validation for ' + doc.kind)
validate(instance=objdict_to_dict(doc),
schema=objdict_to_dict(schema))
def run(self):
for doc in self.validation_docs:
self.logger.info(f'Validating: {doc.kind}')
schema = self.get_base_schema(doc.kind)
schema['properties']['specification'] = load_yaml_obj(
types.VALIDATION, self.cluster_model.provider, doc.kind)
if hasattr(schema['properties']["specification"], '$ref'):
if schema['properties']["specification"][
'$ref'] == '#/definitions/unvalidated_specification':
self.logger.warn('No specification validation for ' +
doc.kind)
try:
validate(instance=objdict_to_dict(doc),
schema=objdict_to_dict(schema))
except Exception as e:
self.logger.error(f'Failed validating: {doc.kind}')
self.logger.error(e)
raise Exception(
'Schema validation error, see the error above.')
def run(self):
configuration_docs = []
for component_key, component_value in self.cluster_model.specification.components.items(
):
if component_value.count < 1:
continue
features_map = select_first(
self.input_docs,
lambda x: x.kind == 'configuration/feature-mapping')
if features_map is None:
features_map = select_first(
configuration_docs,
lambda x: x.kind == 'configuration/feature-mapping')
if features_map is None:
features_map = load_yaml_obj(types.DEFAULT, 'common',
'configuration/feature-mapping')
self.logger.info("Adding: " + features_map.kind)
configuration_docs.append(features_map)
config_selector = component_value.configuration
for feature_key in features_map.specification.roles_mapping[
component_key]:
config = select_first(
self.input_docs, lambda x: x.kind == 'configuration/' +
feature_key and x.name == config_selector)
if config is None:
config = select_first(
configuration_docs,
lambda x: x.kind == 'configuration/' + feature_key and
x.name == config_selector)
if config is None:
config = merge_with_defaults(
'common', 'configuration/' + feature_key,
config_selector)
self.logger.info("Adding: " + config.kind)
configuration_docs.append(config)
return configuration_docs
def populate_group_vars(self, ansible_dir):
main_vars = ObjDict()
main_vars['admin_user'] = self.cluster_model.specification.admin_user
main_vars['validate_certs'] = Config().validate_certs
main_vars['offline_requirements'] = Config().offline_requirements
main_vars['wait_for_pods'] = Config().wait_for_pods
shared_config_doc = select_first(self.config_docs, lambda x: x.kind == 'configuration/shared-config')
if shared_config_doc == None:
shared_config_doc = load_yaml_obj(types.DEFAULT, 'common', 'configuration/shared-config')
main_vars.update(shared_config_doc.specification)
vars_dir = os.path.join(ansible_dir, 'group_vars')
if not os.path.exists(vars_dir):
os.makedirs(vars_dir)
vars_file_name = 'all.yml'
vars_file_path = os.path.join(vars_dir, vars_file_name)
with open(vars_file_path, 'a') as stream:
dump(main_vars, stream)
def run_for_individual_documents(self):
for doc in self.validation_docs:
# Load document schema
schema = load_yaml_obj(types.VALIDATION,
self.cluster_model.provider, doc.kind)
# Include "definitions"
schema['definitions'] = self.definitions
# Warn the user about the missing validation
if hasattr(schema, '$ref'):
if schema['$ref'] == '#/definitions/unvalidated_specification':
self.logger.warn('No specification validation for ' +
doc.kind)
# Assert the schema
try:
validate(instance=objdict_to_dict(doc),
schema=objdict_to_dict(schema))
except Exception as e:
self.logger.error(f'Failed validating: {doc.kind}')
self.logger.error(e)
raise Exception(
'Schema validation error, see the error above.')
def _process_configuration_docs(self):
"""Populate input yaml documents with additional required ad-hoc data."""
# Seed the self.configuration_docs
self.configuration_docs = copy.deepcopy(self.input_docs)
# Please notice using DefaultMerger is not needed here, since it is done already at this point.
# We just check if documents are missing and insert default ones without the unneeded merge operation.
for kind in {'configuration/backup', 'configuration/recovery'}:
try:
# Check if the required document is in user inputs
document = select_single(self.configuration_docs, lambda x: x.kind == kind)
except ExpectedSingleResultException:
# If there is no document provided by the user, then fallback to defaults
document = load_yaml_obj(data_types.DEFAULT, 'common', kind)
# Inject the required "version" attribute
document['version'] = VERSION
# Copy the "provider" value from the cluster model
document['provider'] = self.cluster_model.provider
# Save the document for later use
self.configuration_docs.append(document)
finally:
# Copy the "provider" value to the specification as well
document.specification['provider'] = document['provider']
def get_config_or_default(docs, kind):
config = select_first(docs, lambda x: x.kind == kind)
if config is None:
config = load_yaml_obj(types.DEFAULT, 'azure', kind)
config['version'] = VERSION
return config
def upgrade(self):
inventory_path = get_inventory_path_for_build(self.backup_build_dir)
build_version = check_build_output_version(self.backup_build_dir)
self.logger.info(f'Loading backup Ansible inventory: {inventory_path}')
loaded_inventory = InventoryManager(loader=DataLoader(),
sources=inventory_path)
# move loaded inventory to templating structure
new_inventory = []
for key in loaded_inventory.groups:
if key != 'all' and key != 'ungrouped':
group_hosts = loaded_inventory.groups[key].hosts
new_hosts = []
for host in group_hosts:
new_hosts.append(
AnsibleHostModel(host.address,
host.vars['ansible_host']))
new_inventory.append(AnsibleInventoryItem(key, new_hosts))
if build_version == BUILD_LEGACY:
self.logger.info(f'Upgrading Ansible inventory Epiphany < 0.3.0')
# Epiphany < 0.3.0 did not have manifest file in build folder so lets create bare minimum cluster model from inventory
self.cluster_model = dict_to_objdict({
'provider': 'any',
'specification': {
'admin_user': {
'name':
loaded_inventory.groups['all'].vars['ansible_user'],
'key_path':
loaded_inventory.groups['all'].
vars['ansible_ssh_private_key_file']
}
}
})
# Remap roles
self.rename_role(new_inventory, 'master', 'kubernetes_master')
self.rename_role(new_inventory, 'worker', 'kubernetes_node')
self.rename_role(new_inventory, 'deployments', 'applications')
self.rename_role(new_inventory, 'elasticsearch-curator',
'elasticsearch_curator')
self.rename_role(new_inventory, 'jmx-exporter', 'jmx_exporter')
self.rename_role(new_inventory, 'kafka-exporter', 'kafka_exporter')
self.rename_role(new_inventory, 'haproxy_tls_termination',
'haproxy')
# remove linux and reboot roles if present
self.delete_role(new_inventory, 'linux')
self.delete_role(new_inventory, 'reboot')
else:
self.logger.info(f'Upgrading Ansible inventory Epiphany => 0.3.0')
# load cluster model from manifest
self.manifest_docs = load_manifest_docs(self.backup_build_dir)
self.cluster_model = select_single(
self.manifest_docs, lambda x: x.kind == 'epiphany-cluster')
# Merge manifest cluster config with newer defaults
default_cluster_model = load_yaml_obj(data_types.DEFAULT, 'common',
'epiphany-cluster')
merge_objdict(default_cluster_model, self.cluster_model)
self.cluster_model = default_cluster_model
# Check if repo roles are present and if not add them
master = self.get_role(new_inventory, 'kubernetes_master')
if master == None:
raise Exception('No kubernetes_master to use as repository')
master_node = master.hosts[0]
# add image_registry
image_registry = self.get_role(new_inventory, 'image_registry')
if image_registry == None:
hosts = []
hosts.append(AnsibleHostModel(master_node.name, master_node.ip))
new_inventory.append(AnsibleInventoryItem('image_registry', hosts))
# add repository
repository = self.get_role(new_inventory, 'repository')
if repository == None:
hosts = []
hosts.append(AnsibleHostModel(master_node.name, master_node.ip))
new_inventory.append(AnsibleInventoryItem('repository', hosts))
# save new inventory
save_inventory(new_inventory, self.cluster_model, self.build_dir)
return 0
def get_config_or_default(docs, kind):
config = select_first(docs, lambda x: x.kind == kind)
if config is None:
return load_yaml_obj(types.DEFAULT, 'aws', kind)
return config
