def __init__(self, tool_name):
self.tool_name = tool_name
cwd_config_filename = os.path.join(os.getcwd(), tool_name + CONFIG_FILE_EXT)
root_clouni_config_filename = os.path.join(utils.get_project_root_path(),
tool_name + CONFIG_FILE_EXT)
tools_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), tool_name)
tool_config_filename = os.path.join(tools_directory, 'configuration_tool' + CONFIG_FILE_EXT)
tool_name_config_filename = os.path.join(tools_directory, tool_name + CONFIG_FILE_EXT)
filename_variants_priority = [
cwd_config_filename,
root_clouni_config_filename,
tool_config_filename,
tool_name_config_filename
]
self.config_filename = None
for filename in filename_variants_priority:
if os.path.isfile(filename):
self.config_filename = filename
break
if self.config_filename is None:
ExceptionCollector.appendException(ConfigurationNotFound(
what=filename_variants_priority
))
super(ConfigurationToolConfiguration, self).__init__(self.config_filename)
def __init__(self, provider):
self.provider = provider
cwd_config_filename = os.path.join(os.getcwd(),
provider + CONFIG_FILE_EXT)
root_clouni_config_filename = os.path.join(
utils.get_project_root_path(), provider + CONFIG_FILE_EXT)
providers_directory = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
provider)
provider_config_filename = os.path.join(providers_directory,
'provider' + CONFIG_FILE_EXT)
provider_name_config_filename = os.path.join(
providers_directory, provider + CONFIG_FILE_EXT)
filename_variants_priority = [
cwd_config_filename, root_clouni_config_filename,
provider_config_filename, provider_name_config_filename
]
self.config_filename = None
for filename in filename_variants_priority:
if os.path.isfile(filename):
self.config_filename = filename
break
if self.config_filename is None:
logging.error(
"Configuration files were missing in possible locations: %s" %
json.dumps(filename_variants_priority))
sys.exit(1)
super(ProviderConfiguration, self).__init__(self.config_filename)
def test_validation(self):
example_path = os.path.join(get_project_root_path(), 'examples',
'tosca-server-example-kubernetes.yaml')
shell.main([
'--template-file', example_path, '--validate-only',
'--cluster-name', 'test'
])
def __init__(self, tool_name):
self.tool_name = tool_name
cwd_config_filename = os.path.join(os.getcwd(), tool_name + CONFIG_FILE_EXT)
root_clouni_config_filename = os.path.join(utils.get_project_root_path(),
tool_name + CONFIG_FILE_EXT)
tools_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), tool_name)
tool_config_filename = os.path.join(tools_directory, 'configuration_tool' + CONFIG_FILE_EXT)
tool_name_config_filename = os.path.join(tools_directory, tool_name + CONFIG_FILE_EXT)
filename_variants_priority = [
cwd_config_filename,
root_clouni_config_filename,
tool_config_filename,
tool_name_config_filename
]
self.config_filename = None
for filename in filename_variants_priority:
if os.path.isfile(filename):
self.config_filename = filename
break
if self.config_filename is None:
logging.error("Configuration file was not found. It must be one of the variants: %s"
% json.dumps(filename_variants_priority))
sys.exit(1)
super(ConfigurationToolConfiguration, self).__init__(self.config_filename)
def test_k8s_translate(self):
example_path = os.path.join(get_project_root_path(), 'examples',
'tosca-server-example-kubernetes.yaml')
shell.main([
'--template-file', example_path, '--provider', self.PROVIDER,
'--configuration-tool', 'kubernetes', '--cluster-name', 'test'
])
def __init__(self, provider):
self.provider = provider
cwd_config_filename = os.path.join(os.getcwd(),
provider + CONFIG_FILE_EXT)
root_clouni_config_filename = os.path.join(
utils.get_project_root_path(), provider + CONFIG_FILE_EXT)
providers_directory = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
provider)
provider_config_filename = os.path.join(providers_directory,
'provider' + CONFIG_FILE_EXT)
provider_name_config_filename = os.path.join(
providers_directory, provider + CONFIG_FILE_EXT)
filename_variants_priority = [
cwd_config_filename, root_clouni_config_filename,
provider_config_filename, provider_name_config_filename
]
self.config_filename = None
for filename in filename_variants_priority:
if os.path.isfile(filename):
self.config_filename = filename
break
if self.config_filename is None:
ExceptionCollector.appendException(
ProviderConfigurationNotFound(what=filename_variants_priority))
super(ProviderConfiguration, self).__init__(self.config_filename)
def execute(new_global_elements_map_total_implementation, is_delete, cluster_name, target_parameter=None):
if not is_delete:
default_executor = 'ansible'
configuration_class = get_configuration_tool_class(default_executor)()
new_ansible_artifacts = copy.deepcopy(new_global_elements_map_total_implementation)
for i in range(len(new_ansible_artifacts)):
new_ansible_artifacts[i]['configuration_tool'] = new_ansible_artifacts[i]['executor']
configuration_class = get_configuration_tool_class(new_ansible_artifacts[i]['configuration_tool'])()
extension = configuration_class.get_artifact_extension()
seed(time())
new_ansible_artifacts[i]['name'] = '_'.join(
[SOURCE, str(randint(ARTIFACT_RANGE_START, ARTIFACT_RANGE_END))]) + extension
artifacts_with_brackets = utils.replace_brackets(new_ansible_artifacts, False)
new_ansible_tasks, filename = utils.generate_artifacts(configuration_class, artifacts_with_brackets,
configuration_class.initial_artifacts_directory,
store=False)
os.remove(filename)
q = Queue()
playbook = {
'hosts': 'localhost',
'tasks': new_ansible_tasks
}
os.makedirs(os.path.join(utils.get_tmp_clouni_dir(), cluster_name, configuration_class.initial_artifacts_directory), exist_ok=True)
copy_tree(utils.get_project_root_path() + '/toscatranslator/configuration_tools/ansible/artifacts',
os.path.join(utils.get_tmp_clouni_dir(), cluster_name, configuration_class.initial_artifacts_directory))
configuration_class.parallel_run([playbook], 'artifacts', 'artifacts', q, cluster_name)
# there is such a problem that if runner cotea was launched in the current process, then all subsequent launches
# of other playbooks from this process are impossible because the initial playbook will be launched, and
# even if this process is forked, the effect remains so that something inside cotea
# is preserved in the context of the process
results = q.get()
if target_parameter is not None:
value = 'not_found'
if_failed = False
for result in results:
if result.is_failed or result.is_unreachable:
logging.error("Task %s has failed because of exception: \n%s" %
(result.task_name, result.result.get('exception', '(Unknown reason)')))
if_failed = True
if 'results' in result.result and len(result.result['results']) > 0 and 'ansible_facts' in \
result.result['results'][0] and 'matched_object' in result.result['results'][0]['ansible_facts']:
value = result.result['results'][0]['ansible_facts']['matched_object'][target_parameter.split('.')[-1]]
if if_failed:
value = 'not_found'
return value
return None
def translate(template_file,
validate_only,
provider,
configuration_tool,
cluster_name='',
is_delete=False,
a_file=True,
extra=None):
if a_file:
template_file = os.path.join(os.getcwd(), template_file)
with open(template_file, 'r') as f:
template_content = f.read()
else:
template_content = template_file
template = yaml_parse(template_content)
default_import_file = os.path.join(utils.get_project_root_path(),
TOSCA_DEFINITION_FILE)
if not template.get(IMPORTS):
template[IMPORTS] = [default_import_file]
else:
for i in range(len(template[IMPORTS])):
template[IMPORTS][i] = os.path.abspath(template[IMPORTS][i])
template[IMPORTS].append(default_import_file)
tosca_parser_template_object = ToscaTemplate(yaml_dict_tpl=template,
a_file=a_file)
if validate_only:
msg = 'The input "%(template_file)s" successfully passed validation.' \
% {'template_file': template_file if a_file else 'template'}
return msg
if not provider:
ExceptionCollector.appendException(
UnspecifiedParameter(what=('validate-only', 'provider')))
tosca = ProviderToscaTemplate(tosca_parser_template_object, provider,
cluster_name)
return tosca.to_configuration_dsl(configuration_tool,
is_delete,
extra=extra)
def __init__(self, file=None):
if file is None:
file = CLOUNI + CONFIG_FILE_EXT
if os.path.isabs(file):
if os.path.isfile(file):
self.config_filename = file
else:
logging.error("Configuration file \'%s\' not found" % file)
sys.exit(1)
else:
cwd_config_filename = os.path.join(os.getcwd(), file)
root_clouni_config_filename = os.path.join(utils.get_project_root_path(),
'toscatranslator',
file)
filename_variants_priority = [
cwd_config_filename,
root_clouni_config_filename,
]
self.config_filename = None
for filename in filename_variants_priority:
if os.path.isfile(filename):
self.config_filename = filename
break
if self.config_filename is None:
logging.error("Configuration files were missing in possible locations: %s" % json.dumps(filename_variants_priority))
sys.exit(1)
logging.info("Configuration file \'%s\' is used" % self.config_filename)
self.config_directory = os.path.dirname(self.config_filename)
self.config = configparser.ConfigParser()
self.config.read(self.config_filename)
if not self.MAIN_SECTION in self.config.sections():
logging.error("Main section is missing in configuration file")
sys.exit(1)
def __init__(self, file=None):
if file is None:
file = CLOUNI + CONFIG_FILE_EXT
if os.path.isabs(file):
if os.path.isfile(file):
self.config_filename = file
else:
ExceptionCollector.appendException(FileNotFoundError(file))
else:
cwd_config_filename = os.path.join(os.getcwd(), file)
root_clouni_config_filename = os.path.join(
utils.get_project_root_path(), 'toscatranslator', file)
filename_variants_priority = [
cwd_config_filename,
root_clouni_config_filename,
]
self.config_filename = None
for filename in filename_variants_priority:
if os.path.isfile(filename):
self.config_filename = filename
break
if self.config_filename is None:
ExceptionCollector.appendException(
ConfigurationNotFound(what=filename_variants_priority))
self.config_directory = os.path.dirname(self.config_filename)
self.config = configparser.ConfigParser()
self.config.read(self.config_filename)
if not self.MAIN_SECTION in self.config.sections():
ExceptionCollector.appendException(
ConfigurationParameterError(what=self.MAIN_SECTION))
def __init__(self, provider):
self.provider = provider
self.config = configparser.ConfigParser()
cwd_config_filename = os.path.join(os.getcwd(), provider + CONFIG_FILE_EXT)
root_clouni_config_filename = os.path.join(utils.get_project_root_path(),
provider + CONFIG_FILE_EXT)
providers_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
provider)
provider_config_filename = os.path.join(providers_directory, 'provider' + CONFIG_FILE_EXT)
provider_name_config_filename = os.path.join(providers_directory, provider + CONFIG_FILE_EXT)
filename_variants_priority = [
cwd_config_filename,
root_clouni_config_filename,
provider_config_filename,
provider_name_config_filename
]
self.config_filename = None
for filename in filename_variants_priority:
if os.path.isfile(filename):
self.config_filename = filename
break
if self.config_filename is None:
ExceptionCollector.appendException(ProviderConfigurationNotFound(
what=filename_variants_priority
))
self.config_directory = os.path.dirname(self.config_filename)
self.config.read(self.config_filename)
if not self.MAIN_SECTION in self.config.sections():
ExceptionCollector.appendException(ProviderConfigurationParameterError(
what=self.MAIN_SECTION
))
def translate(template_file,
validate_only,
provider,
configuration_tool,
cluster_name,
is_delete=False,
a_file=True,
extra=None,
log_level='info',
host_ip_parameter='public_address',
public_key_path='~/.ssh/id_rsa.pub',
debug=False):
"""
Main function, is called by different shells, i.e. bash, Ansible module, grpc
:param template_file: filename of TOSCA template or TOSCA template data if a_file is False
:param validate_only: boolean, if template should be only validated
:param provider: key of cloud provider
:param configuration_tool: key of configuration tool
:param cluster_name: name to point to desired infrastructure as one component
:param is_delete: generate dsl scripts for infrastructure deletion
:param a_file: if template_file is filename
:param extra: extra for template
:return: string that is a script to deploy or delete infrastructure
"""
log_map = dict(debug=logging.DEBUG,
info=logging.INFO,
warning=logging.WARNING,
error=logging.ERROR,
critical=logging.ERROR)
logging_format = "%(asctime)s %(levelname)s %(message)s"
logging.basicConfig(filename=os.path.join(os.getenv('HOME'),
'.clouni.log'),
filemode='a',
level=log_map[log_level],
format=logging_format,
datefmt='%Y-%m-%d %H:%M:%S')
logging.info(
"Started translation of TOSCA template \'%s\' for provider \'%s\' and configuration tool \'%s\'"
% (template_file if a_file else 'raw', provider, configuration_tool))
logging.info("Cluster name set to \'%s\'" % cluster_name)
logging.info("Deploying script for cluster %s will be created" %
'deletion' if is_delete else 'creation')
logging.info(
"Extra parameters to the unit of deployment scripts will be added: %s"
% json.dumps(extra))
logging.info("Log level is set to %s" % log_level)
config = Configuration()
for sec in REQUIRED_CONFIGURATION_PARAMS:
if sec not in config.get_section(config.MAIN_SECTION).keys():
logging.error(
'Provider configuration parameter "%s" is missing in configuration file'
% sec)
sys.exit(1)
if a_file:
template_file = os.path.join(os.getcwd(), template_file)
with open(template_file, 'r') as f:
template_content = f.read()
else:
template_content = template_file
try:
template = yaml.load(template_content, Loader=yaml.SafeLoader)
except yaml.scanner.ScannerError as e:
logging.error("Error parsing TOSCA template: %s%s" %
(e.problem, e.context_mark))
sys.exit(1)
def_files = config.get_section(
config.MAIN_SECTION).get(TOSCA_ELEMENTS_DEFINITION_FILE)
if isinstance(def_files, six.string_types):
def_files = [def_files]
default_import_files = []
for def_file in def_files:
default_import_files.append(
os.path.join(utils.get_project_root_path(), def_file))
logging.info(
"Default TOSCA template definition file to be imported \'%s\'" %
json.dumps(default_import_files))
# Add default import of normative TOSCA types to the template
template[IMPORTS] = template.get(IMPORTS, [])
for i in range(len(template[IMPORTS])):
if isinstance(template[IMPORTS][i], dict):
for import_key, import_value in template[IMPORTS][i].items():
if isinstance(import_value, six.string_types):
template[IMPORTS][i] = import_value
elif isinstance(import_value, dict):
if import_value.get('file', None) is None:
logging.error(
"Imports %s doesn't contain \'file\' key" %
import_key)
sys.exit(1)
else:
template[IMPORTS][i] = import_value['file']
if import_value.get('repository', None) is not None:
logging.warning(
"Clouni doesn't support imports \'repository\'")
template[IMPORTS].extend(default_import_files)
for i in range(len(template[IMPORTS])):
template[IMPORTS][i] = os.path.abspath(template[IMPORTS][i])
try:
tosca_parser_template_object = ToscaTemplate(yaml_dict_tpl=template,
a_file=a_file)
except:
logging.exception("Got exception from OpenStack tosca-parser")
sys.exit(1)
# After validation, all templates are imported
if validate_only:
msg = 'The input "%(template_file)s" successfully passed validation.' \
% {'template_file': template_file if a_file else 'TOSCA template'}
return msg
if not provider:
logging.error(
"Provider must be specified unless \'validate-only\' flag is used")
sys.exit(1)
map_files = config.get_section(
config.MAIN_SECTION).get(TOSCA_ELEMENTS_MAP_FILE)
if isinstance(map_files, six.string_types):
map_files = [map_files]
default_map_files = []
for map_file in map_files:
default_map_files.append(
os.path.join(utils.get_project_root_path(), map_file))
logging.info("Default TOSCA template map file to be used \'%s\'" %
json.dumps(default_map_files))
# Parse and generate new TOSCA service template with only provider specific TOSCA types from normative types
tosca = ProviderToscaTemplate(tosca_parser_template_object,
provider,
configuration_tool,
cluster_name,
host_ip_parameter,
public_key_path,
is_delete,
common_map_files=default_map_files)
# Init configuration tool class
tool = get_configuration_tool_class(configuration_tool)()
default_artifacts_directory = config.get_section(
config.MAIN_SECTION).get(DEFAULT_ARTIFACTS_DIRECTORY)
# Copy used conditions from intermediate service template
if tosca.used_conditions_set:
tool.copy_conditions_to_the_directory(tosca.used_conditions_set,
default_artifacts_directory)
# Manage new artifacts for intermediate template
tool_artifacts = []
for art in tosca.artifacts:
executor = art.get(EXECUTOR)
if bool(executor) and executor != configuration_tool:
art_list = [art]
configuration_class = get_configuration_tool_class(
art['executor'])()
_, new_art = utils.generate_artifacts(configuration_class,
art_list,
default_artifacts_directory)
tosca.artifacts.append(new_art)
else:
tool_artifacts.append(art)
if not extra:
extra = {}
extra_full = utils.deep_update_dict(
extra,
tosca.extra_configuration_tool_params.get(configuration_tool, {}))
configuration_content = tool.to_dsl(
tosca.provider,
tosca.provider_operations,
tosca.reversed_provider_operations,
tosca.cluster_name,
is_delete,
artifacts=tool_artifacts,
target_directory=default_artifacts_directory,
inputs=tosca.inputs,
outputs=tosca.outputs,
extra=extra_full,
debug=debug)
return configuration_content
def translate(template_file,
validate_only,
provider,
configuration_tool,
cluster_name,
is_delete=False,
a_file=True,
extra=None):
"""
Main function, is called by different shells, i.e. bash, Ansible module, grpc
:param template_file: filename of TOSCA template or TOSCA template data if a_file is False
:param validate_only: boolean, if template should be only validated
:param provider: key of cloud provider
:param configuration_tool: key of configuration tool
:param cluster_name: name to point to desired infrastructure as one component
:param is_delete: generate dsl scripts for infrastructure deletion
:param a_file: if template_file is filename
:param extra: extra for template
:return: string that is a script to deploy or delete infrastructure
"""
config = Configuration()
for sec in REQUIRED_CONFIGURATION_PARAMS:
if sec not in config.get_section(config.MAIN_SECTION).keys():
raise ProviderConfigurationParameterError(what=sec)
if a_file:
template_file = os.path.join(os.getcwd(), template_file)
with open(template_file, 'r') as f:
template_content = f.read()
else:
template_content = template_file
template = yaml_parse(template_content)
def_file = config.get_section(
config.MAIN_SECTION).get(TOSCA_DEFINITION_FILE)
default_import_file = os.path.join(utils.get_project_root_path(), def_file)
# Add default import of normative TOSCA types to the template
if not template.get(IMPORTS):
template[IMPORTS] = [default_import_file]
else:
for i in range(len(template[IMPORTS])):
template[IMPORTS][i] = os.path.abspath(template[IMPORTS][i])
template[IMPORTS].append(default_import_file)
tosca_parser_template_object = ToscaTemplate(yaml_dict_tpl=template,
a_file=a_file)
if validate_only:
msg = 'The input "%(template_file)s" successfully passed validation.' \
% {'template_file': template_file if a_file else 'template'}
return msg
if not provider:
ExceptionCollector.appendException(
UnspecifiedParameter(what=('validate-only', 'provider')))
# Parse and generate new TOSCA service template with only provider specific TOSCA types from normative types
tosca = ProviderToscaTemplate(tosca_parser_template_object, provider,
cluster_name)
# Init configuration tool class
tool = get_configuration_tool_class(configuration_tool)()
default_artifacts_directory = config.get_section(
config.MAIN_SECTION).get(DEFAULT_ARTIFACTS_DIRECTORY)
# Copy used conditions from intermediate service template
if tosca.used_conditions_set:
tool.copy_conditions_to_the_directory(tosca.used_conditions_set,
default_artifacts_directory)
# Manage new artifacts for intermediate template
tool_artifacts = []
for art in tosca.artifacts:
executor = art.get(EXECUTOR)
if bool(executor) and executor != configuration_tool:
art_list = [art]
new_arts = generate_artifacts(art_list,
default_artifacts_directory)
tosca.artifacts.extend(new_arts)
else:
tool_artifacts.append(art)
if not extra:
extra = {}
extra_full = utils.deep_update_dict(
extra,
tosca.extra_configuration_tool_params.get(configuration_tool, {}))
configuration_content = tool.to_dsl(tosca.provider,
tosca.provider_nodes_queue,
tosca.cluster_name,
is_delete,
tool_artifacts,
default_artifacts_directory,
inputs=tosca.inputs,
outputs=tosca.outputs,
extra=extra_full)
return configuration_content
