def _parse_toml(self, config_file: Path,
parser: configparser.ConfigParser) -> None:
"""Parse and handle errors of a toml configuration file."""
with open(config_file, mode="rb") as fp:
content = tomllib.load(fp)
try:
sections_values = content["tool"]["pylint"]
except KeyError:
return
for section, values in sections_values.items():
section_name = section.upper()
# TOML has rich types, convert values to
# strings as ConfigParser expects.
if not isinstance(values, dict):
# This class is a mixin: add_message comes from the `PyLinter` class
self.add_message( # type: ignore[attr-defined]
"bad-configuration-section",
line=0,
args=(section, values))
continue
for option, value in values.items():
if isinstance(value, bool):
values[option] = "yes" if value else "no"
elif isinstance(value, list):
values[option] = ",".join(value)
else:
values[option] = str(value)
for option, value in values.items():
try:
parser.set(section_name, option, value=value)
except configparser.NoSectionError:
parser.add_section(section_name)
parser.set(section_name, option, value=value)
def _parse_toml_file(self,
file_path: Path) -> Tuple[Dict[str, str], List[str]]:
"""Parse and handle errors of a toml configuration file."""
try:
with open(file_path, mode="rb") as fp:
content = tomllib.load(fp)
except tomllib.TOMLDecodeError as e:
self.linter.add_message("config-parse-error", line=0, args=str(e))
return {}, []
try:
sections_values = content["tool"]["pylint"]
except KeyError:
return {}, []
config_content: Dict[str, str] = {}
options: List[str] = []
for opt, values in sections_values.items():
if isinstance(values, dict):
for config, value in values.items():
value = _parse_rich_type_value(value)
config_content[config] = value
options += [f"--{config}", value]
else:
values = _parse_rich_type_value(values)
config_content[opt] = values
options += [f"--{opt}", values]
return config_content, options
def test_defaults(platform, intercepted_build_args):
main()
build_options: BuildOptions = intercepted_build_args.args[0].build_options(
identifier=None)
defaults_config_path = resources_dir / "defaults.toml"
with defaults_config_path.open("rb") as f:
defaults_toml = tomllib.load(f)
root_defaults = defaults_toml["tool"]["cibuildwheel"]
platform_defaults = defaults_toml["tool"]["cibuildwheel"][platform]
defaults = {}
defaults.update(root_defaults)
defaults.update(platform_defaults)
# test a few options
assert build_options.before_all == defaults["before-all"]
repair_wheel_default = defaults["repair-wheel-command"]
if isinstance(repair_wheel_default, list):
repair_wheel_default = " && ".join(repair_wheel_default)
assert build_options.repair_command == repair_wheel_default
assert build_options.build_frontend == defaults["build-frontend"]
if platform == "linux":
assert build_options.manylinux_images
pinned_images = _get_pinned_container_images()
default_x86_64_image = pinned_images["x86_64"][
defaults["manylinux-x86_64-image"]]
assert build_options.manylinux_images["x86_64"] == default_x86_64_image
def read_python_configs(config: PlatformName) -> list[dict[str, str]]:
input_file = resources_dir / "build-platforms.toml"
with input_file.open("rb") as f:
loaded_file = tomllib.load(f)
results: list[dict[str, str]] = list(
loaded_file[config]["python_configurations"])
return results
def _parse_toml(config_file: Path, parser: configparser.ConfigParser) -> None:
"""DEPRECATED: Parse and handle errors of a toml configuration file.
TODO: Remove after read_config_file has been removed.
"""
with open(config_file, mode="rb") as fp:
content = tomllib.load(fp)
try:
sections_values = content["tool"]["pylint"]
except KeyError:
return
for section, values in sections_values.items():
section_name = section.upper()
# TOML has rich types, convert values to
# strings as ConfigParser expects.
if not isinstance(values, dict):
continue
for option, value in values.items():
if isinstance(value, bool):
values[option] = "yes" if value else "no"
elif isinstance(value, list):
values[option] = ",".join(value)
else:
values[option] = str(value)
for option, value in values.items():
try:
parser.set(section_name, option, value=value)
except configparser.NoSectionError:
parser.add_section(section_name)
parser.set(section_name, option, value=value)
def load_config(self, config=None):
"""
Load the configuration files and append it to local dictionary.
It's stored in self.configuration with the content of already loaded
options.
"""
if config:
# just add the new config at the end of the list, someone injected
# config file to us
for path in config:
if path not in self.conf_files and path.exists():
self.conf_files.append(path)
cfg = {}
# sort self.conf_files as we print list of loaded configuration files
self.conf_files = sorted(self.conf_files, key=self._sort_config_files)
for cf in self.conf_files:
try:
with open(cf, 'rb') as f:
toml_config = tomllib.load(f)
self._merge_dictionaries(cfg, toml_config,
self._is_override_config(cf))
except tomllib.TOMLDecodeError as terr:
print_warning(
f'(none): E: fatal error while parsing configuration file {cf}: {terr}'
)
sys.exit(4)
self.configuration = cfg
def get_requires_python_str(package_dir: Path) -> str | None:
"""Return the python requires string from the most canonical source available, or None"""
# Read in from pyproject.toml:project.requires-python
try:
with (package_dir / "pyproject.toml").open("rb") as f1:
info = tomllib.load(f1)
return str(info["project"]["requires-python"])
except (FileNotFoundError, KeyError, IndexError, TypeError):
pass
# Read in from setup.cfg:options.python_requires
try:
config = ConfigParser()
config.read(package_dir / "setup.cfg")
return str(config["options"]["python_requires"])
except (FileNotFoundError, KeyError, IndexError, TypeError):
pass
try:
with (package_dir / "setup.py").open(encoding="utf8") as f2:
return setup_py_python_requires(f2.read())
except FileNotFoundError:
pass
return None
def _toml_has_config(path: Union[Path, str]) -> bool:
with open(path, mode="rb") as toml_handle:
try:
content = tomllib.load(toml_handle)
except tomllib.TOMLDecodeError as error:
print(f"Failed to load '{path}': {error}")
return False
return "pylint" in content.get("tool", [])
def parse_pyproject_toml(path_config: str) -> Dict[str, Any]:
"""Parse a pyproject toml file, pulling out relevant parts for Black
If parsing fails, will raise a tomllib.TOMLDecodeError
"""
with open(path_config, "rb") as f:
pyproject_toml = tomllib.load(f)
config = pyproject_toml.get("tool", {}).get("black", {})
return {k.replace("--", "").replace("-", "_"): v for k, v in config.items()}
def _load_path_index(self, app: BaseConfig):
"""Load the path index from the index file provided by the app
template.
:param app: The config object for the app
:return: The contents of the application path index.
"""
with (self.bundle_path(app) / "briefcase.toml").open("rb") as f:
self._path_index[app] = tomllib.load(f)["paths"]
return self._path_index[app]
def _load_file(self, filename: Path) -> tuple[dict[str, Any], dict[str, Any]]:
"""
Load a toml file, returns global and platform as separate dicts.
"""
with filename.open("rb") as f:
config = tomllib.load(f)
global_options = config.get("tool", {}).get("cibuildwheel", {})
platform_options = global_options.get(self.platform, {})
return global_options, platform_options
def _ensure_virtualenv() -> Path:
input_file = resources_dir / "virtualenv.toml"
with input_file.open("rb") as f:
loaded_file = tomllib.load(f)
version = str(loaded_file["version"])
url = str(loaded_file["url"])
path = CIBW_CACHE_PATH / f"virtualenv-{version}.pyz"
with FileLock(str(path) + ".lock"):
if not path.exists():
download(url, path)
return path
def test_compare_configs():
with open(resources_dir / "build-platforms.toml") as f1:
txt = f1.read()
with open(resources_dir / "build-platforms.toml", "rb") as f2:
dict_txt = tomllib.load(f2)
new_txt = dump_python_configurations(dict_txt)
print(new_txt)
assert new_txt == txt
def __init__(self, config_file=None):
"""Attempt to initialize a config dictionary from a yaml file.
Error out if loading the yaml file fails for any reason.
:param config_file: The Bandit yaml config file
:raises bandit.utils.ConfigError: If the config is invalid or
unreadable.
"""
self.config_file = config_file
self._config = {}
if config_file:
try:
f = open(config_file, "rb")
except OSError:
raise utils.ConfigError("Could not read config file.",
config_file)
if config_file.endswith(".toml"):
if tomllib is None:
raise utils.ConfigError(
"toml parser not available, reinstall with toml extra",
config_file,
)
try:
with f:
self._config = tomllib.load(f)["tool"]["bandit"]
except tomllib.TOMLDecodeError as err:
LOG.error(err)
raise utils.ConfigError("Error parsing file.", config_file)
else:
try:
with f:
self._config = yaml.safe_load(f)
except yaml.YAMLError as err:
LOG.error(err)
raise utils.ConfigError("Error parsing file.", config_file)
self.validate(config_file)
# valid config must be a dict
if not isinstance(self._config, dict):
raise utils.ConfigError("Error parsing file.", config_file)
self.convert_legacy_config()
else:
# use sane defaults
self._config["plugin_name_pattern"] = "*.py"
self._config["include"] = ["*.py", "*.pyw"]
self._init_settings()
def _is_cargo_project(cargo_toml: pathlib.Path, module_name: str) -> bool:
with contextlib.suppress(FileNotFoundError):
with open(cargo_toml, "rb") as f:
cargo = tomllib.load(f)
package_name = cargo.get("package", {}).get("name")
if (
package_name == module_name
or package_name.replace("-", "_") == module_name
):
return True
return False
def reload(self):
"""Loading a TOML file produces a single dict. Use
its keys as column headings, and populate a single
row.
"""
self.columns = []
self.rows = []
data = tomllib.load(self.source.open_bytes())
for k, v in data.items():
self.addColumn(ColumnItem(k, type=deduceType(v)))
self.addRow(data)
def load_config(path: str, config_name: str) -> Tuple[dict, str]:
"""Load the configuration TOML file for a test at the given path.
Return the configuration data itself and the containing directory.
"""
for dirpath in ancestors(path):
config_path = os.path.join(dirpath, config_name)
if os.path.isfile(config_path):
with open(config_path, 'rb') as f:
return tomllib.load(f), dirpath
# No configuration; use defaults and embedded options only.
return {}, os.path.dirname(os.path.abspath(path))
def update_pythons(force: bool, level: str) -> None:
logging.basicConfig(
level="INFO",
format="%(message)s",
datefmt="[%X]",
handlers=[RichHandler(rich_tracebacks=True, markup=True)],
)
log.setLevel(level)
all_versions = AllVersions()
toml_file_path = RESOURCES_DIR / "build-platforms.toml"
original_toml = toml_file_path.read_text()
with toml_file_path.open("rb") as f:
configs = tomllib.load(f)
for config in configs["windows"]["python_configurations"]:
all_versions.update_config(config)
for config in configs["macos"]["python_configurations"]:
all_versions.update_config(config)
result_toml = dump_python_configurations(configs)
rich.print() # spacer
if original_toml == result_toml:
rich.print("[green]Check complete, Python configurations unchanged.")
return
rich.print("Python configurations updated.")
rich.print("Changes:")
rich.print()
toml_relpath = toml_file_path.relative_to(DIR).as_posix()
diff_lines = difflib.unified_diff(
original_toml.splitlines(keepends=True),
result_toml.splitlines(keepends=True),
fromfile=toml_relpath,
tofile=toml_relpath,
)
rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light"))
rich.print()
if force:
toml_file_path.write_text(result_toml)
rich.print("[green]TOML file updated.")
else:
rich.print("[yellow]File left unchanged. Use --force flag to update.")
def parse_pyproj(pyproject_file):
"""Parse all required and optional dependencies from pyproject file."""
with open(os.path.join(os.getcwd(), pyproject_file), "rb") as f:
pyproj = tomllib.load(f).get("project", {})
deps = pyproj.get("dependencies", []).copy()
for opt_deps in pyproj.get('optional-dependencies', {}).values():
deps += opt_deps
# Replace any double quotes in dependencies with single quotes so we don't
# break the Dockerfile
deps = [d.replace('"', "'") for d in deps]
return '", "'.join(deps)
def update_virtualenv(force: bool, level: str) -> None:
logging.basicConfig(
level="INFO",
format="%(message)s",
datefmt="[%X]",
handlers=[RichHandler(rich_tracebacks=True, markup=True)],
)
log.setLevel(level)
toml_file_path = RESOURCES_DIR / "virtualenv.toml"
original_toml = toml_file_path.read_text()
with toml_file_path.open("rb") as f:
loaded_file = tomllib.load(f)
version = str(loaded_file["version"])
versions = git_ls_remote_versions(GET_VIRTUALENV_GITHUB)
if versions[0].version > Version(version):
version = versions[0].version_string
result_toml = (
f'version = "{version}"\n'
f'url = "{GET_VIRTUALENV_URL_TEMPLATE.format(version=version)}"\n')
rich.print() # spacer
if original_toml == result_toml:
rich.print("[green]Check complete, virtualenv version unchanged.")
return
rich.print("virtualenv version updated.")
rich.print("Changes:")
rich.print()
toml_relpath = toml_file_path.relative_to(DIR).as_posix()
diff_lines = difflib.unified_diff(
original_toml.splitlines(keepends=True),
result_toml.splitlines(keepends=True),
fromfile=toml_relpath,
tofile=toml_relpath,
)
rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light"))
rich.print()
if force:
toml_file_path.write_text(result_toml)
rich.print("[green]TOML file updated.")
else:
rich.print("[yellow]File left unchanged. Use --force flag to update.")
def _is_compatible_stub_package(self, stub_dir: str) -> bool:
"""Does a stub package support the target Python version?
Stub packages may contain a metadata file which specifies
whether the stubs are compatible with Python 2 and 3.
"""
metadata_fnam = os.path.join(stub_dir, 'METADATA.toml')
if os.path.isfile(metadata_fnam):
with open(metadata_fnam, "rb") as f:
metadata = tomllib.load(f)
if self.python_major_ver == 2:
return bool(metadata.get('python2', False))
else:
return bool(metadata.get('python3', True))
return True
def main() -> None:
project_root, _ = find_project_root((str(ROOT), ))
with project_root.joinpath("pyproject.toml").open("rb") as fp:
PYPROJECT = tomllib.load(fp)
section: dict[str, Any] = PYPROJECT.get("tool", {}).get("blacken-docs", {})
line_length = section.get("line-length", DEFAULT_LINE_LENGTH)
target_version = {
mode.TargetVersion[val.upper()]
for val in section.get("target-version", ())
}
file_mode = mode.Mode(target_version, line_length)
for file in project_root.rglob("**/*"):
if INCLUDES_RE.search(str(file)) and not EXCLUDES_RE.search(str(file)):
blacken_docs.format_file(str(file), file_mode, skip_errors=False)
def parse_manifest(file):
"""Parse the given file (iterable of lines) to a Manifest"""
manifest = Manifest()
data = tomllib.load(file)
for kind, itemclass in itemclasses.items():
for name, item_data in data[kind].items():
try:
item = itemclass(name=name, kind=kind, **item_data)
manifest.add(item)
except BaseException as exc:
exc.add_note(f'in {kind} {name}')
raise
return manifest
def read(self, file_names):
file_names_read = []
for fname in always_iterable(file_names):
if not os.path.exists(fname):
continue
metadata = {"source": f"file: {fname}"}
try:
with open(fname, "rb") as fh:
data = tomllib.load(fh)
except tomllib.TOMLDecodeError as exc:
warnings.warn(
f"Could not load configuration file {fname} (invalid TOML: {exc})"
)
else:
self.update(data, metadata=metadata)
file_names_read.append(fname)
return file_names_read
def _load_descriptions():
"""
Load rpmlint error/warning description texts from toml files.
Detailed description for every rpmlint error/warning is stored in
descriptions/<check_name>.toml file.
Returns:
A dictionary mapping error/warning/info names to their
descriptions.
"""
descriptions = {}
descr_folder = Path(__file__).parent / 'descriptions'
try:
for description_file in sorted(descr_folder.glob('*.toml')):
with open(description_file, 'rb') as f:
descriptions.update(tomllib.load(f))
except tomllib.TOMLDecodeError as terr:
print_warning(
f'(none): W: unable to parse description files: {terr}')
return descriptions
def check_dump(manifest, filename):
"""Check that manifest.dump() corresponds to the data.
Mainly useful when debugging this script.
"""
dumped = tomllib.loads('\n'.join(manifest.dump()))
with filename.open('rb') as file:
from_file = tomllib.load(file)
if dumped != from_file:
print(f'Dump differs from loaded data!', file=sys.stderr)
diff = difflib.unified_diff(
pprint.pformat(dumped).splitlines(),
pprint.pformat(from_file).splitlines(),
'<dumped>',
str(filename),
lineterm='',
)
for line in diff:
print(line, file=sys.stderr)
return False
else:
return True
def validate(integrations: dict[str, Integration], config: Config) -> None:
"""Validate project metadata keys."""
metadata_path = config.root / "pyproject.toml"
with open(metadata_path, "rb") as fp:
data = tomllib.load(fp)
try:
if data["project"]["version"] != __version__:
config.add_error(
"metadata",
f"'project.version' value does not match '{__version__}'")
except KeyError:
config.add_error("metadata", "No 'metadata.version' key found!")
required_py_version = f">={'.'.join(map(str, REQUIRED_PYTHON_VER))}"
try:
if data["project"]["requires-python"] != required_py_version:
config.add_error(
"metadata",
f"'project.requires-python' value doesn't match '{required_py_version}",
)
except KeyError:
config.add_error("metadata", "No 'options.python_requires' key found!")
import pathlib
import re
import subprocess
from typing import TYPE_CHECKING
try:
import tomllib # type: ignore
except ImportError:
import tomli as tomllib
if TYPE_CHECKING:
from typing_extensions import TypeAlias
ROOT = pathlib.Path(".").resolve()
PYPROJECT = tomllib.load(open(ROOT / "pyproject.toml", "rb"))
try:
VERSION: str = PYPROJECT["tool"]["poetry"]["version"]
except KeyError:
raise RuntimeError("Version is not set") from None
RELEASE_LEVELS = {
"a": "alpha",
"b": "beta",
"rc": "candidate",
}
try:
end_char: str = re.findall(r"\d+.\d+.\d+([^\d]+).*", VERSION)[0]
except IndexError:
release_level = "final"
def core_requirements():
"""Gather core requirements out of pyproject.toml."""
with open("pyproject.toml", "rb") as fp:
data = tomllib.load(fp)
return data["project"]["dependencies"]
def get_config() -> Dict[str, str]:
with open("pyproject.toml", "rb") as fp:
pyproject_toml = tomllib.load(fp)
return pyproject_toml.get("tool", {}).get("maturin", {})
