def __init__(self, tool_name): self.tool_name = tool_name cwd_config_filename = os.path.join(os.getcwd(), tool_name + CONFIG_FILE_EXT) root_clouni_config_filename = os.path.join(utils.get_project_root_path(), tool_name + CONFIG_FILE_EXT) tools_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), tool_name) tool_config_filename = os.path.join(tools_directory, 'configuration_tool' + CONFIG_FILE_EXT) tool_name_config_filename = os.path.join(tools_directory, tool_name + CONFIG_FILE_EXT) filename_variants_priority = [ cwd_config_filename, root_clouni_config_filename, tool_config_filename, tool_name_config_filename ] self.config_filename = None for filename in filename_variants_priority: if os.path.isfile(filename): self.config_filename = filename break if self.config_filename is None: ExceptionCollector.appendException(ConfigurationNotFound( what=filename_variants_priority )) super(ConfigurationToolConfiguration, self).__init__(self.config_filename)
def __init__(self, provider): self.provider = provider cwd_config_filename = os.path.join(os.getcwd(), provider + CONFIG_FILE_EXT) root_clouni_config_filename = os.path.join( utils.get_project_root_path(), provider + CONFIG_FILE_EXT) providers_directory = os.path.join( os.path.dirname(os.path.dirname(os.path.realpath(__file__))), provider) provider_config_filename = os.path.join(providers_directory, 'provider' + CONFIG_FILE_EXT) provider_name_config_filename = os.path.join( providers_directory, provider + CONFIG_FILE_EXT) filename_variants_priority = [ cwd_config_filename, root_clouni_config_filename, provider_config_filename, provider_name_config_filename ] self.config_filename = None for filename in filename_variants_priority: if os.path.isfile(filename): self.config_filename = filename break if self.config_filename is None: logging.error( "Configuration files were missing in possible locations: %s" % json.dumps(filename_variants_priority)) sys.exit(1) super(ProviderConfiguration, self).__init__(self.config_filename)
def test_validation(self): example_path = os.path.join(get_project_root_path(), 'examples', 'tosca-server-example-kubernetes.yaml') shell.main([ '--template-file', example_path, '--validate-only', '--cluster-name', 'test' ])
def __init__(self, tool_name): self.tool_name = tool_name cwd_config_filename = os.path.join(os.getcwd(), tool_name + CONFIG_FILE_EXT) root_clouni_config_filename = os.path.join(utils.get_project_root_path(), tool_name + CONFIG_FILE_EXT) tools_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), tool_name) tool_config_filename = os.path.join(tools_directory, 'configuration_tool' + CONFIG_FILE_EXT) tool_name_config_filename = os.path.join(tools_directory, tool_name + CONFIG_FILE_EXT) filename_variants_priority = [ cwd_config_filename, root_clouni_config_filename, tool_config_filename, tool_name_config_filename ] self.config_filename = None for filename in filename_variants_priority: if os.path.isfile(filename): self.config_filename = filename break if self.config_filename is None: logging.error("Configuration file was not found. It must be one of the variants: %s" % json.dumps(filename_variants_priority)) sys.exit(1) super(ConfigurationToolConfiguration, self).__init__(self.config_filename)
def test_k8s_translate(self): example_path = os.path.join(get_project_root_path(), 'examples', 'tosca-server-example-kubernetes.yaml') shell.main([ '--template-file', example_path, '--provider', self.PROVIDER, '--configuration-tool', 'kubernetes', '--cluster-name', 'test' ])
def __init__(self, provider): self.provider = provider cwd_config_filename = os.path.join(os.getcwd(), provider + CONFIG_FILE_EXT) root_clouni_config_filename = os.path.join( utils.get_project_root_path(), provider + CONFIG_FILE_EXT) providers_directory = os.path.join( os.path.dirname(os.path.dirname(os.path.realpath(__file__))), provider) provider_config_filename = os.path.join(providers_directory, 'provider' + CONFIG_FILE_EXT) provider_name_config_filename = os.path.join( providers_directory, provider + CONFIG_FILE_EXT) filename_variants_priority = [ cwd_config_filename, root_clouni_config_filename, provider_config_filename, provider_name_config_filename ] self.config_filename = None for filename in filename_variants_priority: if os.path.isfile(filename): self.config_filename = filename break if self.config_filename is None: ExceptionCollector.appendException( ProviderConfigurationNotFound(what=filename_variants_priority)) super(ProviderConfiguration, self).__init__(self.config_filename)
def execute(new_global_elements_map_total_implementation, is_delete, cluster_name, target_parameter=None): if not is_delete: default_executor = 'ansible' configuration_class = get_configuration_tool_class(default_executor)() new_ansible_artifacts = copy.deepcopy(new_global_elements_map_total_implementation) for i in range(len(new_ansible_artifacts)): new_ansible_artifacts[i]['configuration_tool'] = new_ansible_artifacts[i]['executor'] configuration_class = get_configuration_tool_class(new_ansible_artifacts[i]['configuration_tool'])() extension = configuration_class.get_artifact_extension() seed(time()) new_ansible_artifacts[i]['name'] = '_'.join( [SOURCE, str(randint(ARTIFACT_RANGE_START, ARTIFACT_RANGE_END))]) + extension artifacts_with_brackets = utils.replace_brackets(new_ansible_artifacts, False) new_ansible_tasks, filename = utils.generate_artifacts(configuration_class, artifacts_with_brackets, configuration_class.initial_artifacts_directory, store=False) os.remove(filename) q = Queue() playbook = { 'hosts': 'localhost', 'tasks': new_ansible_tasks } os.makedirs(os.path.join(utils.get_tmp_clouni_dir(), cluster_name, configuration_class.initial_artifacts_directory), exist_ok=True) copy_tree(utils.get_project_root_path() + '/toscatranslator/configuration_tools/ansible/artifacts', os.path.join(utils.get_tmp_clouni_dir(), cluster_name, configuration_class.initial_artifacts_directory)) configuration_class.parallel_run([playbook], 'artifacts', 'artifacts', q, cluster_name) # there is such a problem that if runner cotea was launched in the current process, then all subsequent launches # of other playbooks from this process are impossible because the initial playbook will be launched, and # even if this process is forked, the effect remains so that something inside cotea # is preserved in the context of the process results = q.get() if target_parameter is not None: value = 'not_found' if_failed = False for result in results: if result.is_failed or result.is_unreachable: logging.error("Task %s has failed because of exception: \n%s" % (result.task_name, result.result.get('exception', '(Unknown reason)'))) if_failed = True if 'results' in result.result and len(result.result['results']) > 0 and 'ansible_facts' in \ result.result['results'][0] and 'matched_object' in result.result['results'][0]['ansible_facts']: value = result.result['results'][0]['ansible_facts']['matched_object'][target_parameter.split('.')[-1]] if if_failed: value = 'not_found' return value return None
def translate(template_file, validate_only, provider, configuration_tool, cluster_name='', is_delete=False, a_file=True, extra=None): if a_file: template_file = os.path.join(os.getcwd(), template_file) with open(template_file, 'r') as f: template_content = f.read() else: template_content = template_file template = yaml_parse(template_content) default_import_file = os.path.join(utils.get_project_root_path(), TOSCA_DEFINITION_FILE) if not template.get(IMPORTS): template[IMPORTS] = [default_import_file] else: for i in range(len(template[IMPORTS])): template[IMPORTS][i] = os.path.abspath(template[IMPORTS][i]) template[IMPORTS].append(default_import_file) tosca_parser_template_object = ToscaTemplate(yaml_dict_tpl=template, a_file=a_file) if validate_only: msg = 'The input "%(template_file)s" successfully passed validation.' \ % {'template_file': template_file if a_file else 'template'} return msg if not provider: ExceptionCollector.appendException( UnspecifiedParameter(what=('validate-only', 'provider'))) tosca = ProviderToscaTemplate(tosca_parser_template_object, provider, cluster_name) return tosca.to_configuration_dsl(configuration_tool, is_delete, extra=extra)
def __init__(self, file=None): if file is None: file = CLOUNI + CONFIG_FILE_EXT if os.path.isabs(file): if os.path.isfile(file): self.config_filename = file else: logging.error("Configuration file \'%s\' not found" % file) sys.exit(1) else: cwd_config_filename = os.path.join(os.getcwd(), file) root_clouni_config_filename = os.path.join(utils.get_project_root_path(), 'toscatranslator', file) filename_variants_priority = [ cwd_config_filename, root_clouni_config_filename, ] self.config_filename = None for filename in filename_variants_priority: if os.path.isfile(filename): self.config_filename = filename break if self.config_filename is None: logging.error("Configuration files were missing in possible locations: %s" % json.dumps(filename_variants_priority)) sys.exit(1) logging.info("Configuration file \'%s\' is used" % self.config_filename) self.config_directory = os.path.dirname(self.config_filename) self.config = configparser.ConfigParser() self.config.read(self.config_filename) if not self.MAIN_SECTION in self.config.sections(): logging.error("Main section is missing in configuration file") sys.exit(1)
def __init__(self, file=None): if file is None: file = CLOUNI + CONFIG_FILE_EXT if os.path.isabs(file): if os.path.isfile(file): self.config_filename = file else: ExceptionCollector.appendException(FileNotFoundError(file)) else: cwd_config_filename = os.path.join(os.getcwd(), file) root_clouni_config_filename = os.path.join( utils.get_project_root_path(), 'toscatranslator', file) filename_variants_priority = [ cwd_config_filename, root_clouni_config_filename, ] self.config_filename = None for filename in filename_variants_priority: if os.path.isfile(filename): self.config_filename = filename break if self.config_filename is None: ExceptionCollector.appendException( ConfigurationNotFound(what=filename_variants_priority)) self.config_directory = os.path.dirname(self.config_filename) self.config = configparser.ConfigParser() self.config.read(self.config_filename) if not self.MAIN_SECTION in self.config.sections(): ExceptionCollector.appendException( ConfigurationParameterError(what=self.MAIN_SECTION))
def __init__(self, provider): self.provider = provider self.config = configparser.ConfigParser() cwd_config_filename = os.path.join(os.getcwd(), provider + CONFIG_FILE_EXT) root_clouni_config_filename = os.path.join(utils.get_project_root_path(), provider + CONFIG_FILE_EXT) providers_directory = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), provider) provider_config_filename = os.path.join(providers_directory, 'provider' + CONFIG_FILE_EXT) provider_name_config_filename = os.path.join(providers_directory, provider + CONFIG_FILE_EXT) filename_variants_priority = [ cwd_config_filename, root_clouni_config_filename, provider_config_filename, provider_name_config_filename ] self.config_filename = None for filename in filename_variants_priority: if os.path.isfile(filename): self.config_filename = filename break if self.config_filename is None: ExceptionCollector.appendException(ProviderConfigurationNotFound( what=filename_variants_priority )) self.config_directory = os.path.dirname(self.config_filename) self.config.read(self.config_filename) if not self.MAIN_SECTION in self.config.sections(): ExceptionCollector.appendException(ProviderConfigurationParameterError( what=self.MAIN_SECTION ))
def translate(template_file, validate_only, provider, configuration_tool, cluster_name, is_delete=False, a_file=True, extra=None, log_level='info', host_ip_parameter='public_address', public_key_path='~/.ssh/id_rsa.pub', debug=False): """ Main function, is called by different shells, i.e. bash, Ansible module, grpc :param template_file: filename of TOSCA template or TOSCA template data if a_file is False :param validate_only: boolean, if template should be only validated :param provider: key of cloud provider :param configuration_tool: key of configuration tool :param cluster_name: name to point to desired infrastructure as one component :param is_delete: generate dsl scripts for infrastructure deletion :param a_file: if template_file is filename :param extra: extra for template :return: string that is a script to deploy or delete infrastructure """ log_map = dict(debug=logging.DEBUG, info=logging.INFO, warning=logging.WARNING, error=logging.ERROR, critical=logging.ERROR) logging_format = "%(asctime)s %(levelname)s %(message)s" logging.basicConfig(filename=os.path.join(os.getenv('HOME'), '.clouni.log'), filemode='a', level=log_map[log_level], format=logging_format, datefmt='%Y-%m-%d %H:%M:%S') logging.info( "Started translation of TOSCA template \'%s\' for provider \'%s\' and configuration tool \'%s\'" % (template_file if a_file else 'raw', provider, configuration_tool)) logging.info("Cluster name set to \'%s\'" % cluster_name) logging.info("Deploying script for cluster %s will be created" % 'deletion' if is_delete else 'creation') logging.info( "Extra parameters to the unit of deployment scripts will be added: %s" % json.dumps(extra)) logging.info("Log level is set to %s" % log_level) config = Configuration() for sec in REQUIRED_CONFIGURATION_PARAMS: if sec not in config.get_section(config.MAIN_SECTION).keys(): logging.error( 'Provider configuration parameter "%s" is missing in configuration file' % sec) sys.exit(1) if a_file: template_file = os.path.join(os.getcwd(), template_file) with open(template_file, 'r') as f: template_content = f.read() else: template_content = template_file try: template = yaml.load(template_content, Loader=yaml.SafeLoader) except yaml.scanner.ScannerError as e: logging.error("Error parsing TOSCA template: %s%s" % (e.problem, e.context_mark)) sys.exit(1) def_files = config.get_section( config.MAIN_SECTION).get(TOSCA_ELEMENTS_DEFINITION_FILE) if isinstance(def_files, six.string_types): def_files = [def_files] default_import_files = [] for def_file in def_files: default_import_files.append( os.path.join(utils.get_project_root_path(), def_file)) logging.info( "Default TOSCA template definition file to be imported \'%s\'" % json.dumps(default_import_files)) # Add default import of normative TOSCA types to the template template[IMPORTS] = template.get(IMPORTS, []) for i in range(len(template[IMPORTS])): if isinstance(template[IMPORTS][i], dict): for import_key, import_value in template[IMPORTS][i].items(): if isinstance(import_value, six.string_types): template[IMPORTS][i] = import_value elif isinstance(import_value, dict): if import_value.get('file', None) is None: logging.error( "Imports %s doesn't contain \'file\' key" % import_key) sys.exit(1) else: template[IMPORTS][i] = import_value['file'] if import_value.get('repository', None) is not None: logging.warning( "Clouni doesn't support imports \'repository\'") template[IMPORTS].extend(default_import_files) for i in range(len(template[IMPORTS])): template[IMPORTS][i] = os.path.abspath(template[IMPORTS][i]) try: tosca_parser_template_object = ToscaTemplate(yaml_dict_tpl=template, a_file=a_file) except: logging.exception("Got exception from OpenStack tosca-parser") sys.exit(1) # After validation, all templates are imported if validate_only: msg = 'The input "%(template_file)s" successfully passed validation.' \ % {'template_file': template_file if a_file else 'TOSCA template'} return msg if not provider: logging.error( "Provider must be specified unless \'validate-only\' flag is used") sys.exit(1) map_files = config.get_section( config.MAIN_SECTION).get(TOSCA_ELEMENTS_MAP_FILE) if isinstance(map_files, six.string_types): map_files = [map_files] default_map_files = [] for map_file in map_files: default_map_files.append( os.path.join(utils.get_project_root_path(), map_file)) logging.info("Default TOSCA template map file to be used \'%s\'" % json.dumps(default_map_files)) # Parse and generate new TOSCA service template with only provider specific TOSCA types from normative types tosca = ProviderToscaTemplate(tosca_parser_template_object, provider, configuration_tool, cluster_name, host_ip_parameter, public_key_path, is_delete, common_map_files=default_map_files) # Init configuration tool class tool = get_configuration_tool_class(configuration_tool)() default_artifacts_directory = config.get_section( config.MAIN_SECTION).get(DEFAULT_ARTIFACTS_DIRECTORY) # Copy used conditions from intermediate service template if tosca.used_conditions_set: tool.copy_conditions_to_the_directory(tosca.used_conditions_set, default_artifacts_directory) # Manage new artifacts for intermediate template tool_artifacts = [] for art in tosca.artifacts: executor = art.get(EXECUTOR) if bool(executor) and executor != configuration_tool: art_list = [art] configuration_class = get_configuration_tool_class( art['executor'])() _, new_art = utils.generate_artifacts(configuration_class, art_list, default_artifacts_directory) tosca.artifacts.append(new_art) else: tool_artifacts.append(art) if not extra: extra = {} extra_full = utils.deep_update_dict( extra, tosca.extra_configuration_tool_params.get(configuration_tool, {})) configuration_content = tool.to_dsl( tosca.provider, tosca.provider_operations, tosca.reversed_provider_operations, tosca.cluster_name, is_delete, artifacts=tool_artifacts, target_directory=default_artifacts_directory, inputs=tosca.inputs, outputs=tosca.outputs, extra=extra_full, debug=debug) return configuration_content
def translate(template_file, validate_only, provider, configuration_tool, cluster_name, is_delete=False, a_file=True, extra=None): """ Main function, is called by different shells, i.e. bash, Ansible module, grpc :param template_file: filename of TOSCA template or TOSCA template data if a_file is False :param validate_only: boolean, if template should be only validated :param provider: key of cloud provider :param configuration_tool: key of configuration tool :param cluster_name: name to point to desired infrastructure as one component :param is_delete: generate dsl scripts for infrastructure deletion :param a_file: if template_file is filename :param extra: extra for template :return: string that is a script to deploy or delete infrastructure """ config = Configuration() for sec in REQUIRED_CONFIGURATION_PARAMS: if sec not in config.get_section(config.MAIN_SECTION).keys(): raise ProviderConfigurationParameterError(what=sec) if a_file: template_file = os.path.join(os.getcwd(), template_file) with open(template_file, 'r') as f: template_content = f.read() else: template_content = template_file template = yaml_parse(template_content) def_file = config.get_section( config.MAIN_SECTION).get(TOSCA_DEFINITION_FILE) default_import_file = os.path.join(utils.get_project_root_path(), def_file) # Add default import of normative TOSCA types to the template if not template.get(IMPORTS): template[IMPORTS] = [default_import_file] else: for i in range(len(template[IMPORTS])): template[IMPORTS][i] = os.path.abspath(template[IMPORTS][i]) template[IMPORTS].append(default_import_file) tosca_parser_template_object = ToscaTemplate(yaml_dict_tpl=template, a_file=a_file) if validate_only: msg = 'The input "%(template_file)s" successfully passed validation.' \ % {'template_file': template_file if a_file else 'template'} return msg if not provider: ExceptionCollector.appendException( UnspecifiedParameter(what=('validate-only', 'provider'))) # Parse and generate new TOSCA service template with only provider specific TOSCA types from normative types tosca = ProviderToscaTemplate(tosca_parser_template_object, provider, cluster_name) # Init configuration tool class tool = get_configuration_tool_class(configuration_tool)() default_artifacts_directory = config.get_section( config.MAIN_SECTION).get(DEFAULT_ARTIFACTS_DIRECTORY) # Copy used conditions from intermediate service template if tosca.used_conditions_set: tool.copy_conditions_to_the_directory(tosca.used_conditions_set, default_artifacts_directory) # Manage new artifacts for intermediate template tool_artifacts = [] for art in tosca.artifacts: executor = art.get(EXECUTOR) if bool(executor) and executor != configuration_tool: art_list = [art] new_arts = generate_artifacts(art_list, default_artifacts_directory) tosca.artifacts.extend(new_arts) else: tool_artifacts.append(art) if not extra: extra = {} extra_full = utils.deep_update_dict( extra, tosca.extra_configuration_tool_params.get(configuration_tool, {})) configuration_content = tool.to_dsl(tosca.provider, tosca.provider_nodes_queue, tosca.cluster_name, is_delete, tool_artifacts, default_artifacts_directory, inputs=tosca.inputs, outputs=tosca.outputs, extra=extra_full) return configuration_content