def _parse_toml(self, config_file: Path, parser: configparser.ConfigParser) -> None: """Parse and handle errors of a toml configuration file.""" with open(config_file, mode="rb") as fp: content = tomllib.load(fp) try: sections_values = content["tool"]["pylint"] except KeyError: return for section, values in sections_values.items(): section_name = section.upper() # TOML has rich types, convert values to # strings as ConfigParser expects. if not isinstance(values, dict): # This class is a mixin: add_message comes from the `PyLinter` class self.add_message( # type: ignore[attr-defined] "bad-configuration-section", line=0, args=(section, values)) continue for option, value in values.items(): if isinstance(value, bool): values[option] = "yes" if value else "no" elif isinstance(value, list): values[option] = ",".join(value) else: values[option] = str(value) for option, value in values.items(): try: parser.set(section_name, option, value=value) except configparser.NoSectionError: parser.add_section(section_name) parser.set(section_name, option, value=value)
def _parse_toml_file(self, file_path: Path) -> Tuple[Dict[str, str], List[str]]: """Parse and handle errors of a toml configuration file.""" try: with open(file_path, mode="rb") as fp: content = tomllib.load(fp) except tomllib.TOMLDecodeError as e: self.linter.add_message("config-parse-error", line=0, args=str(e)) return {}, [] try: sections_values = content["tool"]["pylint"] except KeyError: return {}, [] config_content: Dict[str, str] = {} options: List[str] = [] for opt, values in sections_values.items(): if isinstance(values, dict): for config, value in values.items(): value = _parse_rich_type_value(value) config_content[config] = value options += [f"--{config}", value] else: values = _parse_rich_type_value(values) config_content[opt] = values options += [f"--{opt}", values] return config_content, options
def test_defaults(platform, intercepted_build_args): main() build_options: BuildOptions = intercepted_build_args.args[0].build_options( identifier=None) defaults_config_path = resources_dir / "defaults.toml" with defaults_config_path.open("rb") as f: defaults_toml = tomllib.load(f) root_defaults = defaults_toml["tool"]["cibuildwheel"] platform_defaults = defaults_toml["tool"]["cibuildwheel"][platform] defaults = {} defaults.update(root_defaults) defaults.update(platform_defaults) # test a few options assert build_options.before_all == defaults["before-all"] repair_wheel_default = defaults["repair-wheel-command"] if isinstance(repair_wheel_default, list): repair_wheel_default = " && ".join(repair_wheel_default) assert build_options.repair_command == repair_wheel_default assert build_options.build_frontend == defaults["build-frontend"] if platform == "linux": assert build_options.manylinux_images pinned_images = _get_pinned_container_images() default_x86_64_image = pinned_images["x86_64"][ defaults["manylinux-x86_64-image"]] assert build_options.manylinux_images["x86_64"] == default_x86_64_image
def read_python_configs(config: PlatformName) -> list[dict[str, str]]: input_file = resources_dir / "build-platforms.toml" with input_file.open("rb") as f: loaded_file = tomllib.load(f) results: list[dict[str, str]] = list( loaded_file[config]["python_configurations"]) return results
def _parse_toml(config_file: Path, parser: configparser.ConfigParser) -> None: """DEPRECATED: Parse and handle errors of a toml configuration file. TODO: Remove after read_config_file has been removed. """ with open(config_file, mode="rb") as fp: content = tomllib.load(fp) try: sections_values = content["tool"]["pylint"] except KeyError: return for section, values in sections_values.items(): section_name = section.upper() # TOML has rich types, convert values to # strings as ConfigParser expects. if not isinstance(values, dict): continue for option, value in values.items(): if isinstance(value, bool): values[option] = "yes" if value else "no" elif isinstance(value, list): values[option] = ",".join(value) else: values[option] = str(value) for option, value in values.items(): try: parser.set(section_name, option, value=value) except configparser.NoSectionError: parser.add_section(section_name) parser.set(section_name, option, value=value)
def load_config(self, config=None): """ Load the configuration files and append it to local dictionary. It's stored in self.configuration with the content of already loaded options. """ if config: # just add the new config at the end of the list, someone injected # config file to us for path in config: if path not in self.conf_files and path.exists(): self.conf_files.append(path) cfg = {} # sort self.conf_files as we print list of loaded configuration files self.conf_files = sorted(self.conf_files, key=self._sort_config_files) for cf in self.conf_files: try: with open(cf, 'rb') as f: toml_config = tomllib.load(f) self._merge_dictionaries(cfg, toml_config, self._is_override_config(cf)) except tomllib.TOMLDecodeError as terr: print_warning( f'(none): E: fatal error while parsing configuration file {cf}: {terr}' ) sys.exit(4) self.configuration = cfg
def get_requires_python_str(package_dir: Path) -> str | None: """Return the python requires string from the most canonical source available, or None""" # Read in from pyproject.toml:project.requires-python try: with (package_dir / "pyproject.toml").open("rb") as f1: info = tomllib.load(f1) return str(info["project"]["requires-python"]) except (FileNotFoundError, KeyError, IndexError, TypeError): pass # Read in from setup.cfg:options.python_requires try: config = ConfigParser() config.read(package_dir / "setup.cfg") return str(config["options"]["python_requires"]) except (FileNotFoundError, KeyError, IndexError, TypeError): pass try: with (package_dir / "setup.py").open(encoding="utf8") as f2: return setup_py_python_requires(f2.read()) except FileNotFoundError: pass return None
def _toml_has_config(path: Union[Path, str]) -> bool: with open(path, mode="rb") as toml_handle: try: content = tomllib.load(toml_handle) except tomllib.TOMLDecodeError as error: print(f"Failed to load '{path}': {error}") return False return "pylint" in content.get("tool", [])
def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: """Parse a pyproject toml file, pulling out relevant parts for Black If parsing fails, will raise a tomllib.TOMLDecodeError """ with open(path_config, "rb") as f: pyproject_toml = tomllib.load(f) config = pyproject_toml.get("tool", {}).get("black", {}) return {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
def _load_path_index(self, app: BaseConfig): """Load the path index from the index file provided by the app template. :param app: The config object for the app :return: The contents of the application path index. """ with (self.bundle_path(app) / "briefcase.toml").open("rb") as f: self._path_index[app] = tomllib.load(f)["paths"] return self._path_index[app]
def _load_file(self, filename: Path) -> tuple[dict[str, Any], dict[str, Any]]: """ Load a toml file, returns global and platform as separate dicts. """ with filename.open("rb") as f: config = tomllib.load(f) global_options = config.get("tool", {}).get("cibuildwheel", {}) platform_options = global_options.get(self.platform, {}) return global_options, platform_options
def _ensure_virtualenv() -> Path: input_file = resources_dir / "virtualenv.toml" with input_file.open("rb") as f: loaded_file = tomllib.load(f) version = str(loaded_file["version"]) url = str(loaded_file["url"]) path = CIBW_CACHE_PATH / f"virtualenv-{version}.pyz" with FileLock(str(path) + ".lock"): if not path.exists(): download(url, path) return path
def test_compare_configs(): with open(resources_dir / "build-platforms.toml") as f1: txt = f1.read() with open(resources_dir / "build-platforms.toml", "rb") as f2: dict_txt = tomllib.load(f2) new_txt = dump_python_configurations(dict_txt) print(new_txt) assert new_txt == txt
def __init__(self, config_file=None): """Attempt to initialize a config dictionary from a yaml file. Error out if loading the yaml file fails for any reason. :param config_file: The Bandit yaml config file :raises bandit.utils.ConfigError: If the config is invalid or unreadable. """ self.config_file = config_file self._config = {} if config_file: try: f = open(config_file, "rb") except OSError: raise utils.ConfigError("Could not read config file.", config_file) if config_file.endswith(".toml"): if tomllib is None: raise utils.ConfigError( "toml parser not available, reinstall with toml extra", config_file, ) try: with f: self._config = tomllib.load(f)["tool"]["bandit"] except tomllib.TOMLDecodeError as err: LOG.error(err) raise utils.ConfigError("Error parsing file.", config_file) else: try: with f: self._config = yaml.safe_load(f) except yaml.YAMLError as err: LOG.error(err) raise utils.ConfigError("Error parsing file.", config_file) self.validate(config_file) # valid config must be a dict if not isinstance(self._config, dict): raise utils.ConfigError("Error parsing file.", config_file) self.convert_legacy_config() else: # use sane defaults self._config["plugin_name_pattern"] = "*.py" self._config["include"] = ["*.py", "*.pyw"] self._init_settings()
def _is_cargo_project(cargo_toml: pathlib.Path, module_name: str) -> bool: with contextlib.suppress(FileNotFoundError): with open(cargo_toml, "rb") as f: cargo = tomllib.load(f) package_name = cargo.get("package", {}).get("name") if ( package_name == module_name or package_name.replace("-", "_") == module_name ): return True return False
def reload(self): """Loading a TOML file produces a single dict. Use its keys as column headings, and populate a single row. """ self.columns = [] self.rows = [] data = tomllib.load(self.source.open_bytes()) for k, v in data.items(): self.addColumn(ColumnItem(k, type=deduceType(v))) self.addRow(data)
def load_config(path: str, config_name: str) -> Tuple[dict, str]: """Load the configuration TOML file for a test at the given path. Return the configuration data itself and the containing directory. """ for dirpath in ancestors(path): config_path = os.path.join(dirpath, config_name) if os.path.isfile(config_path): with open(config_path, 'rb') as f: return tomllib.load(f), dirpath # No configuration; use defaults and embedded options only. return {}, os.path.dirname(os.path.abspath(path))
def update_pythons(force: bool, level: str) -> None: logging.basicConfig( level="INFO", format="%(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True, markup=True)], ) log.setLevel(level) all_versions = AllVersions() toml_file_path = RESOURCES_DIR / "build-platforms.toml" original_toml = toml_file_path.read_text() with toml_file_path.open("rb") as f: configs = tomllib.load(f) for config in configs["windows"]["python_configurations"]: all_versions.update_config(config) for config in configs["macos"]["python_configurations"]: all_versions.update_config(config) result_toml = dump_python_configurations(configs) rich.print() # spacer if original_toml == result_toml: rich.print("[green]Check complete, Python configurations unchanged.") return rich.print("Python configurations updated.") rich.print("Changes:") rich.print() toml_relpath = toml_file_path.relative_to(DIR).as_posix() diff_lines = difflib.unified_diff( original_toml.splitlines(keepends=True), result_toml.splitlines(keepends=True), fromfile=toml_relpath, tofile=toml_relpath, ) rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light")) rich.print() if force: toml_file_path.write_text(result_toml) rich.print("[green]TOML file updated.") else: rich.print("[yellow]File left unchanged. Use --force flag to update.")
def parse_pyproj(pyproject_file): """Parse all required and optional dependencies from pyproject file.""" with open(os.path.join(os.getcwd(), pyproject_file), "rb") as f: pyproj = tomllib.load(f).get("project", {}) deps = pyproj.get("dependencies", []).copy() for opt_deps in pyproj.get('optional-dependencies', {}).values(): deps += opt_deps # Replace any double quotes in dependencies with single quotes so we don't # break the Dockerfile deps = [d.replace('"', "'") for d in deps] return '", "'.join(deps)
def update_virtualenv(force: bool, level: str) -> None: logging.basicConfig( level="INFO", format="%(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True, markup=True)], ) log.setLevel(level) toml_file_path = RESOURCES_DIR / "virtualenv.toml" original_toml = toml_file_path.read_text() with toml_file_path.open("rb") as f: loaded_file = tomllib.load(f) version = str(loaded_file["version"]) versions = git_ls_remote_versions(GET_VIRTUALENV_GITHUB) if versions[0].version > Version(version): version = versions[0].version_string result_toml = ( f'version = "{version}"\n' f'url = "{GET_VIRTUALENV_URL_TEMPLATE.format(version=version)}"\n') rich.print() # spacer if original_toml == result_toml: rich.print("[green]Check complete, virtualenv version unchanged.") return rich.print("virtualenv version updated.") rich.print("Changes:") rich.print() toml_relpath = toml_file_path.relative_to(DIR).as_posix() diff_lines = difflib.unified_diff( original_toml.splitlines(keepends=True), result_toml.splitlines(keepends=True), fromfile=toml_relpath, tofile=toml_relpath, ) rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light")) rich.print() if force: toml_file_path.write_text(result_toml) rich.print("[green]TOML file updated.") else: rich.print("[yellow]File left unchanged. Use --force flag to update.")
def _is_compatible_stub_package(self, stub_dir: str) -> bool: """Does a stub package support the target Python version? Stub packages may contain a metadata file which specifies whether the stubs are compatible with Python 2 and 3. """ metadata_fnam = os.path.join(stub_dir, 'METADATA.toml') if os.path.isfile(metadata_fnam): with open(metadata_fnam, "rb") as f: metadata = tomllib.load(f) if self.python_major_ver == 2: return bool(metadata.get('python2', False)) else: return bool(metadata.get('python3', True)) return True
def main() -> None: project_root, _ = find_project_root((str(ROOT), )) with project_root.joinpath("pyproject.toml").open("rb") as fp: PYPROJECT = tomllib.load(fp) section: dict[str, Any] = PYPROJECT.get("tool", {}).get("blacken-docs", {}) line_length = section.get("line-length", DEFAULT_LINE_LENGTH) target_version = { mode.TargetVersion[val.upper()] for val in section.get("target-version", ()) } file_mode = mode.Mode(target_version, line_length) for file in project_root.rglob("**/*"): if INCLUDES_RE.search(str(file)) and not EXCLUDES_RE.search(str(file)): blacken_docs.format_file(str(file), file_mode, skip_errors=False)
def parse_manifest(file): """Parse the given file (iterable of lines) to a Manifest""" manifest = Manifest() data = tomllib.load(file) for kind, itemclass in itemclasses.items(): for name, item_data in data[kind].items(): try: item = itemclass(name=name, kind=kind, **item_data) manifest.add(item) except BaseException as exc: exc.add_note(f'in {kind} {name}') raise return manifest
def read(self, file_names): file_names_read = [] for fname in always_iterable(file_names): if not os.path.exists(fname): continue metadata = {"source": f"file: {fname}"} try: with open(fname, "rb") as fh: data = tomllib.load(fh) except tomllib.TOMLDecodeError as exc: warnings.warn( f"Could not load configuration file {fname} (invalid TOML: {exc})" ) else: self.update(data, metadata=metadata) file_names_read.append(fname) return file_names_read
def _load_descriptions(): """ Load rpmlint error/warning description texts from toml files. Detailed description for every rpmlint error/warning is stored in descriptions/<check_name>.toml file. Returns: A dictionary mapping error/warning/info names to their descriptions. """ descriptions = {} descr_folder = Path(__file__).parent / 'descriptions' try: for description_file in sorted(descr_folder.glob('*.toml')): with open(description_file, 'rb') as f: descriptions.update(tomllib.load(f)) except tomllib.TOMLDecodeError as terr: print_warning( f'(none): W: unable to parse description files: {terr}') return descriptions
def check_dump(manifest, filename): """Check that manifest.dump() corresponds to the data. Mainly useful when debugging this script. """ dumped = tomllib.loads('\n'.join(manifest.dump())) with filename.open('rb') as file: from_file = tomllib.load(file) if dumped != from_file: print(f'Dump differs from loaded data!', file=sys.stderr) diff = difflib.unified_diff( pprint.pformat(dumped).splitlines(), pprint.pformat(from_file).splitlines(), '<dumped>', str(filename), lineterm='', ) for line in diff: print(line, file=sys.stderr) return False else: return True
def validate(integrations: dict[str, Integration], config: Config) -> None: """Validate project metadata keys.""" metadata_path = config.root / "pyproject.toml" with open(metadata_path, "rb") as fp: data = tomllib.load(fp) try: if data["project"]["version"] != __version__: config.add_error( "metadata", f"'project.version' value does not match '{__version__}'") except KeyError: config.add_error("metadata", "No 'metadata.version' key found!") required_py_version = f">={'.'.join(map(str, REQUIRED_PYTHON_VER))}" try: if data["project"]["requires-python"] != required_py_version: config.add_error( "metadata", f"'project.requires-python' value doesn't match '{required_py_version}", ) except KeyError: config.add_error("metadata", "No 'options.python_requires' key found!")
import pathlib import re import subprocess from typing import TYPE_CHECKING try: import tomllib # type: ignore except ImportError: import tomli as tomllib if TYPE_CHECKING: from typing_extensions import TypeAlias ROOT = pathlib.Path(".").resolve() PYPROJECT = tomllib.load(open(ROOT / "pyproject.toml", "rb")) try: VERSION: str = PYPROJECT["tool"]["poetry"]["version"] except KeyError: raise RuntimeError("Version is not set") from None RELEASE_LEVELS = { "a": "alpha", "b": "beta", "rc": "candidate", } try: end_char: str = re.findall(r"\d+.\d+.\d+([^\d]+).*", VERSION)[0] except IndexError: release_level = "final"
def core_requirements(): """Gather core requirements out of pyproject.toml.""" with open("pyproject.toml", "rb") as fp: data = tomllib.load(fp) return data["project"]["dependencies"]
def get_config() -> Dict[str, str]: with open("pyproject.toml", "rb") as fp: pyproject_toml = tomllib.load(fp) return pyproject_toml.get("tool", {}).get("maturin", {})