def load_config_file() -> Dict[str, Any]: """Load the configuration from pyproject.toml.""" try: with open("pyproject.toml", encoding="utf-8") as file: content = file.read() except OSError: return {} document = tomlkit.parse(content) tool_section = document.get("tool", tomlkit.document()) toml_sort_section = tool_section.get("tomlsort", tomlkit.document()) config = dict(toml_sort_section) clean_config: Dict[str, Any] = {} validate_and_copy(config, clean_config, "all", bool) validate_and_copy(config, clean_config, "in_place", bool) validate_and_copy(config, clean_config, "no_header", bool) validate_and_copy(config, clean_config, "check", bool) validate_and_copy(config, clean_config, "ignore_case", bool) if config: printerr(f"Unexpected configuration values: {config}") sys.exit(1) return clean_config
def secure(self) -> Iterator[TOMLDocument]: if self.file.exists(): initial_config = self.file.read() config = self.file.read() else: initial_config = document() config = document() new_file = not self.file.exists() yield config try: # Ensuring the file is only readable and writable # by the current user mode = 0o600 if new_file: self.file.touch(mode=mode) self.file.write(config) except Exception: self.file.write(initial_config) raise
def __init__(self, path: pathlib.Path) -> None: self.path = path if self.pyproject_path.is_file(): tomlkit.document() with open(self.pyproject_path) as config_fo: config_data = config_fo.read() self.pyproject = tomlkit.loads(config_data)
def test_activated(app, tmp_dir, config): app.poetry._config = config config.add_property("settings.virtualenvs.path", str(tmp_dir)) venv_name = EnvManager.generate_env_name("simple_project", str(app.poetry.file.parent)) (Path(tmp_dir) / "{}-py3.7".format(venv_name)).mkdir() (Path(tmp_dir) / "{}-py3.6".format(venv_name)).mkdir() envs_file = TomlFile(Path(tmp_dir) / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": "3.7", "patch": "3.7.0"} envs_file.write(doc) command = app.find("env list") tester = CommandTester(command) tester.execute() expected = """\ {}-py3.6 {}-py3.7 (Activated) """.format(venv_name, venv_name) assert expected == tester.io.fetch_output()
def test_get_prefers_explicitly_activated_virtualenvs_over_env_var( tmp_dir, config, mocker): os.environ["VIRTUAL_ENV"] = "/environment/prefix" venv_name = EnvManager.generate_env_name("simple_project", str(CWD)) config.add_property("settings.virtualenvs.path", str(tmp_dir)) (Path(tmp_dir) / "{}-py3.7".format(venv_name)).mkdir() envs_file = TomlFile(Path(tmp_dir) / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": "3.7", "patch": "3.7.0"} envs_file.write(doc) mocker.patch( "poetry.utils._compat.subprocess.check_output", side_effect=check_output_wrapper(), ) mocker.patch( "poetry.utils._compat.subprocess.Popen.communicate", side_effect=[("/prefix", None)], ) env = EnvManager(config).get(CWD) assert env.path == Path(tmp_dir) / "{}-py3.7".format(venv_name) assert env.base == Path("/prefix")
def format_lockfile(mapping, fetched_dependencies, summary_collection): """Format lock file from a dict of resolved candidates, a mapping of dependencies and a collection of package summaries. """ packages = tomlkit.aot() metadata = tomlkit.table() for k, v in sorted(mapping.items()): base = tomlkit.table() base.update(v.as_lockfile_entry()) base.add("summary", summary_collection[strip_extras(k)[0]]) deps = tomlkit.table() for r in fetched_dependencies[k].values(): name, req = r.as_req_dict() if getattr(req, "items", None) is not None: inline = tomlkit.inline_table() inline.update(req) deps.add(name, inline) else: deps.add(name, req) if len(deps) > 0: base.add("dependencies", deps) packages.append(base) if v.hashes: key = f"{k} {v.version}" array = tomlkit.array() array.multiline(True) for filename, hash_value in v.hashes.items(): inline = tomlkit.inline_table() inline.update({"file": filename, "hash": hash_value}) array.append(inline) if array: metadata.add(key, array) doc = tomlkit.document() doc.update({"package": packages, "metadata": metadata}) return doc
def _annotated_config2toml( x: AnnotatedConfig, ) -> tomlkit.toml_document.TOMLDocument: def _add_items( table: Union[tomlkit.toml_document.TOMLDocument, tomlkit.items.Table], items: OrderedDict[str, Item], ): for k, item in items.items(): if isinstance(item.value, OrderedDict): subtable = tomlkit.table() if item.comment: subtable.add(tomlkit.comment(item.comment)) _add_items(subtable, item.value) table.add(k, subtable) else: table.add(k, item.value) if item.comment: try: table[k].comment(item.comment) except AttributeError: # TODO pass config = tomlkit.document() _add_items(config, x) return config
def merge(value: _TOMLDocument, other: _TOMLDocument) -> _TOMLDocument: """Merge two versions of lock data. This function returns a TOML document with the following merged entries: * ``package`` * ``metadata.files`` Any other entries, e.g. ``metadata.content-hash``, are omitted. They are generated from pyproject.toml when the lock data is written to disk. Args: value: Our version of the lock data. other: Their version of the lock data. Returns: The merged lock data. """ document = tomlkit.document() document["package"] = merge_locked_packages(value["package"], other["package"]) document["metadata"] = { "files": merge_locked_package_files(value["metadata"]["files"], other["metadata"]["files"]) } return document
def test_remove_also_deactivates( tmp_dir: str, manager: EnvManager, poetry: "Poetry", config: "Config", mocker: "MockerFixture", ): config.merge({"virtualenvs": {"path": str(tmp_dir)}}) venv_name = manager.generate_env_name("simple-project", str(poetry.file.parent)) (Path(tmp_dir) / f"{venv_name}-py3.7").mkdir() (Path(tmp_dir) / f"{venv_name}-py3.6").mkdir() mocker.patch( "subprocess.check_output", side_effect=check_output_wrapper(Version.parse("3.6.6")), ) envs_file = TOMLFile(Path(tmp_dir) / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": "3.6", "patch": "3.6.6"} envs_file.write(doc) venv = manager.remove("python3.6") expected_venv_path = Path(tmp_dir) / f"{venv_name}-py3.6" assert venv.path == expected_venv_path assert not expected_venv_path.exists() envs = envs_file.read() assert venv_name not in envs
def set_lock_data(self, root, packages): # type: () -> bool hashes = {} packages = self._lock_packages(packages) # Retrieving hashes for package in packages: if package["name"] not in hashes: hashes[package["name"]] = [] hashes[package["name"]] += package["hashes"] del package["hashes"] lock = document() lock["package"] = packages if root.extras: lock["extras"] = { extra: [dep.pretty_name for dep in deps] for extra, deps in root.extras.items() } lock["metadata"] = { "python-versions": root.python_versions, "platform": root.platform, "content-hash": self._content_hash, "hashes": hashes, } if not self.is_locked() or lock != self.lock_data: self._write_lock_data(lock) return True return False
def dumps(self, reqs, project: RootDependency, content=None) -> str: doc = tomlkit.parse(content) if content else tomlkit.document() doc['package'] = [self._format_req(req=req) for req in reqs] # add extras extras = defaultdict(list) for req in reqs: if req.is_main: for extra in req.main_envs: extras[extra].append(req.name) if req.is_dev: for extra in req.dev_envs: extras[extra].append(req.name) if extras: doc['extras'] = dict(extras) doc['metadata'] = { # sha256 of tool.poetry section from pyproject.toml # 'content-hash': ..., # 'platform': '*', 'python-versions': str(project.python), } doc['metadata']['hashes'] = tomlkit.table() for req in reqs: doc['metadata']['hashes'][req.name] = list(req.hashes or []) return tomlkit.dumps(doc)
def toml_headers(): filename = Path("_netlify.toml") doc = document() doc.add(comment("netlify.toml")) doc.add(comment("Generated: " + datetime.now().isoformat())) build = table() env = table().indent(2) env["YARN_VERSION"] = "1.21.0" build["publish"] = "_site/" build["command"] = "make build" build["environment"] = env doc["build"] = build headers = aot() sw = make_headers("sw.js", { "service-worker-allowed": "/", "cache-control": NO_CACHE }) headers.append(sw) manifest = make_headers("**/manifest.json", {"cache-control": NO_CACHE}) headers.append(manifest) for pattern in FOREVER_PATTERNS: headers.append(make_headers(pattern, {"cache-control": CACHE_FOREVER})) doc["headers"] = headers output = dumps(doc) print(output) sz = filename.write_text(output) print(sz)
def create_configuration(self, config: Configuration, *style_urls: str) -> None: """Create a configuration file.""" from nitpick.style import StyleManager # pylint: disable=import-outside-toplevel if config.file: doc: TOMLDocument = tomlkit.parse(config.file.read_text()) else: doc = tomlkit.document() config.file = self.root / DOT_NITPICK_TOML if not style_urls: style_urls = (str(StyleManager.get_default_style_url()), ) tool_nitpick = tomlkit.table() tool_nitpick.add( tomlkit.comment("Generated by the 'nitpick init' command")) tool_nitpick.add( tomlkit.comment( f"More info at {READ_THE_DOCS_URL}configuration.html")) tool_nitpick.add( "style", tomlkit.array([tomlkit.string(url) for url in style_urls])) doc.add(SingleKey(TOOL_NITPICK_KEY, KeyType.Bare), tool_nitpick) # config.file will always have a value at this point, but mypy can't see it. config.file.write_text(tomlkit.dumps(doc, sort_keys=True)) # type: ignore
def _write(self): """ Writes build definition details into build.toml file, which would be used by the next build. build.toml file will contain the same information as build graph, function details will only be preserved as function names layer details will only be preserved as layer names """ # convert build definition list into toml table function_build_definitions_table = tomlkit.table() for build_definition in self._function_build_definitions: build_definition_as_table = _function_build_definition_to_toml_table(build_definition) function_build_definitions_table.add(build_definition.uuid, build_definition_as_table) layer_build_definitions_table = tomlkit.table() for build_definition in self._layer_build_definitions: build_definition_as_table = _layer_build_definition_to_toml_table(build_definition) layer_build_definitions_table.add(build_definition.uuid, build_definition_as_table) # create toml document and add build definitions document = tomlkit.document() document.add(tomlkit.comment("This file is auto generated by SAM CLI build command")) document.add(BuildGraph.FUNCTION_BUILD_DEFINITIONS, function_build_definitions_table) document.add(BuildGraph.LAYER_BUILD_DEFINITIONS, layer_build_definitions_table) if not self._filepath.exists(): open(self._filepath, "a+").close() self._filepath.write_text(tomlkit.dumps(document))
def generate_queries(args: argparse.Namespace): client = MiniClient(args.host, args.port) # dump mapping mappings = client.get_mapping(index=args.index)['mappings'] all_fields = list(get_fields(mappings['properties'])) doc = tomlkit.document() dt_str = get_iso8601_dt_str() comment = tomlkit.comment(f"Automatically generated on {dt_str}") # build all queries queries = tomlkit.table() for field_name in all_fields: t = tomlkit.table() t.add(comment) replace_name = field_name.replace('.', '_') query_name = f'field_{replace_name}_exists' query = json.dumps({'query': {'exists': {'field': field_name}}}) t.add('query', query) t.add('auto_gen', True) queries.add(query_name, t) doc["queries"] = queries with open(args.output, 'w') as f: f.write(tomlkit.dumps(doc))
def dumps(self, reqs, project: RootDependency, content=None) -> str: doc = document() deps = [] for req in reqs: deps.append(self._format_req(req=req)) doc['build-system']['requires'] = deps return dumps(doc)
def test_remove_also_deactivates(tmp_dir, config, mocker): config.add_property("settings.virtualenvs.path", str(tmp_dir)) venv_name = EnvManager.generate_env_name("simple_project", str(CWD)) (Path(tmp_dir) / "{}-py3.7".format(venv_name)).mkdir() (Path(tmp_dir) / "{}-py3.6".format(venv_name)).mkdir() mocker.patch( "poetry.utils._compat.subprocess.check_output", side_effect=check_output_wrapper(Version.parse("3.6.6")), ) envs_file = TomlFile(Path(tmp_dir) / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": "3.6", "patch": "3.6.6"} envs_file.write(doc) manager = EnvManager(config) venv = manager.remove("python3.6", CWD) assert (Path(tmp_dir) / "{}-py3.6".format(venv_name)) == venv.path assert not (Path(tmp_dir) / "{}-py3.6".format(venv_name)).exists() envs = envs_file.read() assert venv_name not in envs
def test_remove_also_deactivates(tmp_dir, manager, poetry, config, mocker): config.merge({"virtualenvs": {"path": str(tmp_dir)}}) venv_name = manager.generate_env_name("simple-project", str(poetry.file.parent)) (Path(tmp_dir) / "{}-py3.7".format(venv_name)).mkdir() (Path(tmp_dir) / "{}-py3.6".format(venv_name)).mkdir() mocker.patch( "subprocess.check_output", side_effect=check_output_wrapper(Version.parse("3.6.6")), ) envs_file = TOMLFile(Path(tmp_dir) / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": "3.6", "patch": "3.6.6"} envs_file.write(doc) venv = manager.remove("python3.6") assert (Path(tmp_dir) / "{}-py3.6".format(venv_name)) == venv.path assert not (Path(tmp_dir) / "{}-py3.6".format(venv_name)).exists() envs = envs_file.read() assert venv_name not in envs
def set_lock_data(self, root, packages): # type: () -> bool hashes = {} packages = self._lock_packages(packages) # Retrieving hashes for package in packages: if package["name"] not in hashes: hashes[package["name"]] = [] hashes[package["name"]] += package["hashes"] del package["hashes"] lock = document() lock["package"] = packages if root.extras: lock["extras"] = { extra: [dep.pretty_name for dep in deps] for extra, deps in root.extras.items() } lock["metadata"] = { "python-versions": root.python_versions, "content-hash": self._content_hash, "hashes": hashes, } if not self.is_locked() or lock != self.lock_data: self._write_lock_data(lock) return True return False
def format_lockfile( mapping: Dict[str, Candidate], fetched_dependencies: Dict[str, List[Requirement]], summary_collection: Dict[str, str], ) -> Dict: """Format lock file from a dict of resolved candidates, a mapping of dependencies and a collection of package summaries. """ packages = tomlkit.aot() file_hashes = tomlkit.table() for k, v in sorted(mapping.items()): base = tomlkit.table() base.update(v.as_lockfile_entry()) base.add("summary", summary_collection[strip_extras(k)[0]]) deps = make_array([r.as_line() for r in fetched_dependencies[k]], True) if len(deps) > 0: base.add("dependencies", deps) packages.append(base) if v.hashes: key = f"{k} {v.version}" array = tomlkit.array() array.multiline(True) for filename, hash_value in v.hashes.items(): inline = make_inline_table({"file": filename, "hash": hash_value}) array.append(inline) if array: file_hashes.add(key, array) doc = tomlkit.document() doc.add("package", packages) metadata = tomlkit.table() metadata.add("files", file_hashes) doc.add("metadata", metadata) return doc
def write_toml(self, filename: Union[str, Path]) -> PhantomConfig: """Write config to TOML file. Parameters ---------- filename The name of the TOML output file. """ # TODO: writing to TOML does not preserve the comments. document = tomlkit.document() if self.header is not None: for line in self.header: document.add(tomlkit.comment(line)) document.add(tomlkit.nl()) d = self.to_dict() for block_key, block_val in d.items(): block = tomlkit.table() if isinstance(block_val, dict): for name, item in block_val.items(): value, comment = item if isinstance(value, datetime.timedelta): value = _convert_timedelta_to_str(value) block.add(tomlkit.nl()) if comment is not None: block.add(tomlkit.comment(comment)) block.add(name, value) document.add(block_key, block) with open(filename, 'w') as fp: fp.write(tomlkit.dumps(document)) return self
def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" if path is None: path = self.pipfile_location data = convert_toml_outline_tables(data) try: formatted_data = tomlkit.dumps(data).rstrip() except Exception: document = tomlkit.document() for section in ("packages", "dev-packages"): document[section] = tomlkit.container.Table() # Convert things to inline tables — fancy :) for package in data.get(section, {}): if hasattr(data[section][package], "keys"): table = tomlkit.inline_table() table.update(data[section][package]) document[section][package] = table else: document[section][package] = tomlkit.string(data[section][package]) formatted_data = tomlkit.dumps(document).rstrip() if ( vistir.compat.Path(path).absolute() == vistir.compat.Path(self.pipfile_location).absolute() ): newlines = self._pipfile_newlines else: newlines = DEFAULT_NEWLINES formatted_data = cleanup_toml(formatted_data) with io.open(path, "w", newline=newlines) as f: f.write(formatted_data) # pipfile is mutated! self.clear_pipfile_cache()
def test_get_prefers_explicitly_activated_virtualenvs_over_env_var( tester: CommandTester, current_python: tuple[int, int, int], venv_cache: Path, venv_name: str, venvs_in_cache_config: None, ): os.environ["VIRTUAL_ENV"] = "/environment/prefix" python_minor = ".".join(str(v) for v in current_python[:2]) python_patch = ".".join(str(v) for v in current_python[:3]) venv_dir = venv_cache / f"{venv_name}-py{python_minor}" venv_dir.mkdir(parents=True, exist_ok=True) envs_file = TOMLFile(venv_cache / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": python_minor, "patch": python_patch} envs_file.write(doc) tester.execute(python_minor) expected = f"""\ Using virtualenv: {venv_dir} """ assert tester.io.fetch_output() == expected
def test_get_prefers_explicitly_activated_virtualenvs_over_env_var( tmp_dir, manager, poetry, config, mocker): os.environ["VIRTUAL_ENV"] = "/environment/prefix" venv_name = manager.generate_env_name("simple-project", str(poetry.file.parent)) config.merge({"virtualenvs": {"path": str(tmp_dir)}}) (Path(tmp_dir) / "{}-py3.7".format(venv_name)).mkdir() envs_file = TOMLFile(Path(tmp_dir) / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": "3.7", "patch": "3.7.0"} envs_file.write(doc) mocker.patch( "subprocess.check_output", side_effect=check_output_wrapper(), ) mocker.patch( "subprocess.Popen.communicate", side_effect=[("/prefix", None)], ) env = manager.get() assert env.path == Path(tmp_dir) / "{}-py3.7".format(venv_name) assert env.base == Path("/prefix")
def test_activate_activates_recreates_for_different_patch( tmp_dir, manager, poetry, config, mocker): if "VIRTUAL_ENV" in os.environ: del os.environ["VIRTUAL_ENV"] venv_name = manager.generate_env_name("simple-project", str(poetry.file.parent)) envs_file = TOMLFile(Path(tmp_dir) / "envs.toml") doc = tomlkit.document() doc[venv_name] = {"minor": "3.7", "patch": "3.7.0"} envs_file.write(doc) os.mkdir(os.path.join(tmp_dir, "{}-py3.7".format(venv_name))) config.merge({"virtualenvs": {"path": str(tmp_dir)}}) mocker.patch( "subprocess.check_output", side_effect=check_output_wrapper(), ) mocker.patch( "subprocess.Popen.communicate", side_effect=[ ("/prefix", None), ('{"version_info": [3, 7, 0]}', None), ("/prefix", None), ("/prefix", None), ("/prefix", None), ], ) build_venv_m = mocker.patch("poetry.utils.env.EnvManager.build_venv", side_effect=build_venv) remove_venv_m = mocker.patch("poetry.utils.env.EnvManager.remove_venv", side_effect=EnvManager.remove_venv) env = manager.activate("python3.7", NullIO()) build_venv_m.assert_called_with( Path(tmp_dir) / "{}-py3.7".format(venv_name), executable="python3.7", flags={ "always-copy": False, "system-site-packages": False }, with_pip=True, with_setuptools=True, with_wheel=True, ) remove_venv_m.assert_called_with( Path(tmp_dir) / "{}-py3.7".format(venv_name)) assert envs_file.exists() envs = envs_file.read() assert envs[venv_name]["minor"] == "3.7" assert envs[venv_name]["patch"] == "3.7.1" assert env.path == Path(tmp_dir) / "{}-py3.7".format(venv_name) assert env.base == Path("/prefix") assert (Path(tmp_dir) / "{}-py3.7".format(venv_name)).exists()
def toml_doc_sorted(self, original: TOMLDocument) -> TOMLDocument: """Sort a TOMLDocument.""" sorted_document = tomlkit.document() if not self.no_header: write_header_comment(original, sorted_document) for key, value in self.sorted_children_table(original): sorted_document[key] = self.toml_elements_sorted(value) return sorted_document
def test_add_dotted_key(): doc = tomlkit.document() doc.add(tomlkit.key(["foo", "bar"]), 1) assert doc.as_string() == "foo.bar = 1\n" table = tomlkit.table() table.add(tomlkit.key(["foo", "bar"]), 1) assert table.as_string() == "foo.bar = 1\n"
def dump(self, configs, path): doc = tomlkit.document() root_comment = getattr(configs, '__doc__', '') def add_comment(sec, comment): for line in textwrap.wrap(comment.strip()): sec.add(tomlkit.comment(line)) def add_value(sec, k, v): is_none = v is None if is_none: sec.add(tomlkit.comment(f'{k} = # Uncomment to use')) else: sec.add(k, v) return not is_none if root_comment: add_comment(doc, root_comment.strip()) doc.add(tomlkit.nl()) for p, value, prop in configs.get_prop_paths(): section = doc key = p if '.' in p: parts = p.split('.') key = parts[-1] for part in parts[:-1]: section = section[part] if isinstance(value, Nestable): # Just add a table for those. table = tomlkit.table() section.add(key, table) if prop.comment is not None: add_comment(table, prop.comment) table.add(tomlkit.nl()) else: if prop.comment is not None: if len(prop.comment) > 40: # Only short comments are inlined. section.add(tomlkit.nl()) add_comment(section, prop.comment) add_value(section, key, value) else: good = add_value(section, key, value) if good: if isinstance(value, bool): item = section.item(key) else: item = section[key] item.comment(prop.comment) else: add_value(section, key, value) with open(path, 'w') as file: file.write(tomlkit.dumps(doc))
def convert_json_config_to_toml(json_config): name = json_config[0]["name"] tool = json_config[0]["tool"] toml_config = document() toml_config.add("name", name) toml_config.add("tool", tool) return dumps(toml_config)
def create_config_file(config: Config) -> None: doc = tomlkit.document() doc.add("theme", config.theme) sitemeta = tomlkit.table() sitemeta.add("title", config.sitemeta.title) sitemeta.add("author", config.sitemeta.author) sitemeta.add("language_code", config.sitemeta.language_code) sitemeta.add("year_of_publication", config.sitemeta.year_of_publication) doc.add("sitemeta", sitemeta) write_file("config.toml", doc.as_string())
def __init__(self, file='config.toml', cfg_builder=None): # type: (str, ConfigBuilder) -> None if cfg_builder is None: cfg_builder = self.builder # Copy builder config into the new Config object, with NEW value references self._config = cfg_builder.build() self._modified = False self._file = file self._toml = tomlkit.document()
def __init__(self, file): # type: (TomlFile) -> None self._file = file if not self._file.exists(): self._content = document() else: self._content = file.read()