def _get_template_engine(ctx): """ Initialize the template engine environment """ global engine_cache if engine_cache is not None: return engine_cache loader_map = {} loader_map[""] = FileSystemLoader( os.path.join(Project.get().project_path, "templates")) for name, module in Project.get().modules.items(): template_dir = os.path.join(module._path, "templates") if os.path.isdir(template_dir): loader_map[name] = FileSystemLoader(template_dir) # init the environment env = Environment(loader=PrefixLoader(loader_map), undefined=jinja2.StrictUndefined) env.context_class = ResolverContext # register all plugins as filters for name, cls in ctx.get_compiler().get_plugins().items(): def curywrapper(func): def safewrapper(*args): return JinjaDynamicProxy.return_value(func(*args)) return safewrapper env.filters[name.replace("::", ".")] = curywrapper(cls) engine_cache = env return env
def list(self, requires=False): """ List all modules in a table """ table = [] name_length = 10 version_length = 10 project = Project.get() project.get_complete_ast() names = sorted(project.modules.keys()) specs = project.collect_imported_requirements() for name in names: name_length = max(len(name), name_length) mod = Project.get().modules[name] version = str(mod.version) if name not in specs: specs[name] = [] try: if project._install_mode == InstallMode.master: reqv = "master" else: release_only = project._install_mode == InstallMode.release versions = Module.get_suitable_version_for( name, specs[name], mod._path, release_only=release_only) if versions is None: reqv = "None" else: reqv = str(versions) except Exception: LOGGER.exception("Problem getting version for module %s" % name) reqv = "ERROR" version_length = max(len(version), len(reqv), version_length) table.append((name, version, reqv, version == reqv)) if requires: print("requires:") for name, version, reqv, _ in table: print(" - %s==%s" % (name, version)) else: t = texttable.Texttable() t.set_deco(texttable.Texttable.HEADER | texttable.Texttable.BORDER | texttable.Texttable.VLINES) t.header(("Name", "Installed version", "Expected in project", "Matches")) for row in table: t.add_row(row) print(t.draw())
def sources(cls): """ Get all source files that define resources """ resource_to_sources = {} for resource, providers in cls.__command_functions.items(): sources = {} resource_to_sources[resource] = sources for provider in providers.values(): file_name = inspect.getsourcefile(provider) source_code = "" with open(file_name, "r") as fd: source_code = fd.read() sha1sum = hashlib.new("sha1") sha1sum.update(source_code.encode("utf-8")) hv = sha1sum.hexdigest() if hv not in sources: module_name = provider.__module__.split(".")[1] req = Project.get( ).modules[module_name].get_python_requirements_as_list() sources[hv] = (file_name, provider.__module__, source_code, req) return resource_to_sources
def sources(cls) -> dict: """ Get all source files that define resources """ resource_to_sources = {} for name, (resource, _options) in cls._resources.items(): sources = {} resource_to_sources[name] = sources file_name = inspect.getsourcefile(resource) source_code = "" with open(file_name, "r") as fd: source_code = fd.read() sha1sum = hashlib.new("sha1") sha1sum.update(source_code.encode("utf-8")) hv = sha1sum.hexdigest() if hv not in sources: module_name = resource.__module__.split(".")[1] req = Project.get( ).modules[module_name].get_python_requirements_as_list() sources[hv] = (file_name, resource.__module__, source_code, req) return resource_to_sources
def compile(self): """ This method will compile and prepare everything to start evaluation the configuration specification. This method will: - load all namespaces - compile the __config__ namespace - start resolving it and importing unknown namespaces """ project = Project.get() self.__root_ns = project.get_root_namespace() project.load() statements, blocks = project.get_complete_ast() # load plugins for name, cls in PluginMeta.get_functions().items(): mod_ns = cls.__module__.split(".") if mod_ns[0] != "inmanta_plugins": raise Exception( "All plugin modules should be loaded in the impera_plugins package not in %s" % cls.__module__) mod_ns = mod_ns[1:] ns = self.__root_ns for part in mod_ns: if ns is None: break ns = ns.get_child(part) if ns is None: raise Exception( "Unable to find namespace for plugin module %s" % (cls.__module__)) cls.namespace = ns name = name.split("::")[-1] statement = PluginStatement(ns, name, cls) statements.append(statement) # add the entity type (hack?) entity = DefineEntity(self.__root_ns.get_child_or_create("std"), "Entity", "The entity all other entities inherit from.", [], []) requires_rel = DefineRelation( ("std::Entity", "requires", [0, None], False), ("std::Entity", "provides", [0, None], False)) requires_rel.namespace = self.__root_ns.get_ns_from_string("std") statements.append(entity) statements.append(requires_rel) return (statements, blocks)
def requires(self) -> List[str]: """List of python requirements associated with this source file""" from inmanta.module import Project if self._requires is None: self._requires = Project.get().modules[self._get_module_name( )].get_strict_python_requirements_as_list() return self._requires
def determine_path(ctx, module_dir, path): """ Determine the real path based on the given path """ parts = path.split(os.path.sep) modules = Project.get().modules if parts[0] == "": module_path = Project.get().project_path elif parts[0] not in modules: raise Exception("Module %s does not exist for path %s" % (parts[0], path)) else: module_path = modules[parts[0]]._path return os.path.join(module_path, module_dir, os.path.sep.join(parts[1:]))
def get_siblings(self) -> Iterator["SourceInfo"]: """ Returns an iterator over SourceInfo objects for all plugin source files in this Inmanta module (including this one). """ from inmanta.module import Project return starmap( SourceInfo, Project.get().modules[self._get_module_name()].get_plugin_files())
def _extend_path(ctx: Context, path: str): current_module_prefix = "." + os.path.sep if path.startswith(current_module_prefix): module_and_submodule_name_parts = ctx.owner.namespace.get_full_name( ).split("::") module_name = module_and_submodule_name_parts[0] if module_name in Project.get().modules.keys(): return os.path.join(module_name, path[len(current_module_prefix):]) else: raise Exception( f"Unable to determine current module for path {path}, called from {ctx.owner.namespace.get_full_name()}" ) return path
def do_compile(refs={}): """ Run run run """ project = Project.get() compiler = Compiler(os.path.join(project.project_path, project.main_file), refs=refs) LOGGER.debug("Starting compile") (statements, blocks) = compiler.compile() sched = scheduler.Scheduler() success = sched.run(compiler, statements, blocks) LOGGER.debug("Compile done") if not success: sys.stderr.write("Unable to execute all statements.\n") return (sched.get_types(), compiler.get_ns())
def get_project(self, load=False) -> Project: project = Project.get() if load: project.load() return project
def verify(self): """ Verify dependencies and frozen module versions """ Project.get().verify()
def reset(self): Project.set(Project(self.project_dir, autostd=Project.get().autostd)) loader.unload_inmanta_plugins()
def test_v1_and_v2_module_installed_simultaneously(tmpdir: py.path.local, snippetcompiler_clean, capsys, caplog, modules_dir: str) -> None: """ When a module is installed both in V1 and V2 format, ensure that: * A warning is logged * The V2 module is loaded and not the V1 module. """ # Work around caching problem in venv feature_compiler_cache.set("False") module_name = "v1_print_plugin" def compile_and_verify( expected_message: str, expect_warning: bool, install_v2_modules: List[LocalPackagePath] = []) -> None: caplog.clear() snippetcompiler_clean.setup_for_snippet( f"import {module_name}", install_v2_modules=install_v2_modules, autostd=False) snippetcompiler_clean.do_export() assert expected_message in capsys.readouterr().out got_warning = f"Module {module_name} is installed as a V1 module and a V2 module" in caplog.text assert got_warning == expect_warning # Run compile. Only a V1 module is installed in the module path expected_message_v1 = "Hello world" compile_and_verify(expected_message=expected_message_v1, expect_warning=False) assert isinstance(Project.get().modules[module_name], ModuleV1) # Convert V1 module to V2 module and install it as well module_dir = os.path.join(modules_dir, module_name) v1_module_dir = os.path.join(tmpdir, "v1_module") shutil.copytree(module_dir, v1_module_dir) assert os.path.isdir(v1_module_dir) v2_module_dir = os.path.join(tmpdir, "v2_module") module = ModuleV1(project=DummyProject(autostd=False), path=v1_module_dir) ModuleConverter(module).convert(output_directory=v2_module_dir) # Print a different message in the V2 module, to detect which of both gets loaded expected_message_v2 = "Other message" with open(os.path.join(v2_module_dir, "model", "_init.cf"), "r+") as fd: content = fd.read() assert expected_message_v1 in content content = content.replace(expected_message_v1, expected_message_v2) assert expected_message_v2 in content fd.seek(0) fd.write(content) # Run compile again. V1 version and V2 version are installed simultaneously compile_and_verify( expected_message=expected_message_v2, expect_warning=True, install_v2_modules=[ LocalPackagePath(path=v2_module_dir, editable=False) ], ) assert isinstance(Project.get().modules[module_name], ModuleV2)
def list(self, requires: bool = False) -> None: """ List all modules in a table """ def show_bool(b: bool) -> str: return "yes" if b else "no" table = [] project = Project.get() project.get_complete_ast() names: Sequence[str] = sorted(project.modules.keys()) specs: Dict[str, List[ InmantaModuleRequirement]] = project.collect_imported_requirements( ) for name in names: mod: Module = Project.get().modules[name] version = str(mod.version) if name not in specs: specs[name] = [] generation: str = str(mod.GENERATION.name).lower() reqv: str matches: bool editable: bool if isinstance(mod, ModuleV1): try: if project.install_mode == InstallMode.master: reqv = "master" else: release_only = project.install_mode == InstallMode.release versions = ModuleV1.get_suitable_version_for( name, specs[name], mod._path, release_only=release_only) if versions is None: reqv = "None" else: reqv = str(versions) except Exception: LOGGER.exception("Problem getting version for module %s" % name) reqv = "ERROR" matches = version == reqv editable = True else: reqv = ",".join(req.version_spec_str() for req in specs[name] if req.specs) or "*" matches = all(version in req for req in specs[name]) editable = mod.is_editable() table.append((name, generation, editable, version, reqv, matches)) if requires: LOGGER.warning( "The `inmanta module list -r` command has been deprecated.") for name, _, _, version, _, _ in table: print(" - %s==%s" % (name, version)) else: t = texttable.Texttable() t.set_deco(texttable.Texttable.HEADER | texttable.Texttable.BORDER | texttable.Texttable.VLINES) t.header(("Name", "Type", "Editable", "Installed version", "Expected in project", "Matches")) t.set_cols_dtype(("t", "t", show_bool, "t", "t", show_bool)) for row in table: t.add_row(row) print(t.draw())