def test_add_detached_section_option_objects():
updater = ConfigUpdater()
updater.read_string(test24_cfg_in)
sec1 = updater["sec1"]
sec2 = updater["sec2"]
assert sec2.container is updater
sec2.detach()
assert not updater.has_section("sec2")
assert not sec2.has_container()
with pytest.raises(NotAttachedError):
sec2.container
new_sec2 = Section("new-sec2")
new_opt = Option(key="new-key", value="new-value")
new_sec2.add_option(new_opt)
with pytest.raises(AlreadyAttachedError):
sec1.add_option(new_opt)
updater.add_section(new_sec2)
assert updater.has_section("new-sec2")
assert updater["new-sec2"]["new-key"].value == "new-value"
new_sec3 = Section("new-sec3")
new_opt2 = Option(key="new-key", value="new-value")
updater["new-sec3"] = new_sec3
new_sec3["new-key"] = new_opt2
def add_pyscaffold(config: ConfigUpdater, opts: ScaffoldOpts) -> ConfigUpdater:
"""Add PyScaffold section to a ``setup.cfg``-like file + PyScaffold's version +
extensions and their associated options.
"""
if "pyscaffold" not in config:
config.add_section("pyscaffold")
pyscaffold = config["pyscaffold"]
pyscaffold["version"] = pyscaffold_version
# Add the new extensions alongside the existing ones
extensions = {
ext.name
for ext in opts.get("extensions", []) if ext.persist
}
old = pyscaffold.pop("extensions", "")
old = parse_extensions(getattr(old, "value",
old)) # coerce configupdater return
pyscaffold.set("extensions")
pyscaffold["extensions"].set_values(sorted(old | extensions))
# Add extension-related opts, i.e. opts which start with an extension name
allowed = {
k: v
for k, v in opts.items() if any(map(k.startswith, extensions))
}
pyscaffold.update(allowed)
return config
def update_setup_cfg(setupcfg: ConfigUpdater, opts: ScaffoldOpts):
"""Update `pyscaffold` in setupcfg and ensure some values are there as expected"""
if "options" not in setupcfg:
setupcfg["metadata"].add_after.section("options")
if "pyscaffold" not in setupcfg:
setupcfg.add_section("pyscaffold")
setupcfg["pyscaffold"]["version"] = pyscaffold_version
return setupcfg, opts
class ProdUpdater(Updater):
"""Production dependency (setup.cfg) updater."""
def __init__(self):
self.file = Path("setup.cfg")
self.file_py = self.file.with_suffix(".py")
self.config = ConfigUpdater()
self.indent = " " # default to 2 spaces indent?
def read(self):
try:
text = self.file.read_text("utf8")
except OSError:
return
self.indent = detect_indent(text) or self.indent
self.config.read_string(text)
def get_requirements(self):
self.read()
try:
return self.config.get("options", "install_requires").value
except configparser.Error:
pass
return ""
def get_name(self):
self.read()
return self.config.get("metadata", "name").value
def get_spec(self, name, version):
if not version.startswith("0."):
version = re.match("\d+\.\d+", version).group()
return "{}~={}".format(name, version)
def write_requirements(self, lines):
if "options" not in self.config:
self.config.add_section("options")
self.config.set("options", "install_requires",
"".join("\n" + self.indent + l for l in lines))
self.file.write_text(str(self.config).replace("\r", ""),
encoding="utf8")
if not self.file_py.exists():
self.file_py.write_text("\n".join(
["from setuptools import setup", "setup()"]),
encoding="utf8")
def save(struct: "Structure", opts: "ScaffoldOpts") -> "ActionParams":
"""Save the given opts as preferences in a PyScaffold's config file."""
config = ConfigUpdater()
if not opts.get("save_config"):
file = info.config_file()
else:
file = Path(opts["save_config"])
if file.exists():
config.read(file, encoding="utf-8")
else:
config.read_string(
"# PyScaffold's configuration file, see:\n"
"# https://pyscaffold.org/en/latest/configuration.html\n#\n"
"# Accepted in `metadata`: author, author_email and license.\n"
"# Accepted in `pyscaffold`: extensions (and associated opts).\n"
)
if "metadata" not in config:
config.add_section("metadata")
# We will add metadata just if they are not the default ones
metadata = config["metadata"]
defaults = [
("author", opts["author"], info.username()),
("author_email", opts["email"], info.email()),
("license", opts["license"], api.DEFAULT_OPTIONS["license"]),
]
metadata.update({k: v for k, v, default in defaults if v != default})
templates.add_pyscaffold(config, opts)
operations.create(file, str(config), opts) # operations provide logging and pretend
return struct, opts
def test_add_section():
updater = ConfigUpdater()
updater.read_string(test7_cfg_in)
with pytest.raises(DuplicateSectionError):
updater.add_section('section1')
updater.add_section('section2')
updater['section2']['key1'] = 1
assert str(updater) == test7_cfg_out
with pytest.raises(ValueError):
updater.add_section(updater['section2']['key1'])
def test_add_section():
updater = ConfigUpdater()
updater.read_string(test7_cfg_in)
with pytest.raises(DuplicateSectionError):
updater.add_section("section1")
updater.add_section("section2")
updater["section2"]["key1"] = 1
assert str(updater) == test7_cfg_out
with pytest.raises(ValueError):
updater.add_section(updater["section2"]["key1"])
class IniPlugin(NitpickPlugin):
"""Enforce configurations and autofix INI files.
Examples of ``.ini`` files handled by this plugin:
- `setup.cfg <https://docs.python.org/3/distutils/configfile.html>`_
- `.editorconfig <https://editorconfig.org/>`_
- `tox.ini <https://github.com/tox-dev/tox>`_
- `.pylintrc <https://pylint.readthedocs.io/en/latest/user_guide/run.html#command-line-options>`_
Style examples enforcing values on INI files: :gitref:`flake8 configuration
<src/nitpick/resources/python/flake8.toml>`.
"""
fixable = True
identify_tags = {"ini", "editorconfig"}
violation_base_code = 320
updater: ConfigUpdater
comma_separated_values: set[str]
def post_init(self):
"""Post initialization after the instance was created."""
self.updater = ConfigUpdater()
self.comma_separated_values = set(self.nitpick_file_dict.get(COMMA_SEPARATED_VALUES, []))
if not self.needs_top_section:
return
if all(isinstance(v, dict) for v in self.expected_config.values()):
return
new_config = dict({TOP_SECTION: {}})
for key, value in self.expected_config.items():
if isinstance(value, dict):
new_config[key] = value
continue
new_config[TOP_SECTION][key] = value
self.expected_config = new_config
@property
def needs_top_section(self) -> bool:
"""Return True if this .ini file needs a top section (e.g.: .editorconfig)."""
return "editorconfig" in self.info.tags
@property
def current_sections(self) -> set[str]:
"""Current sections of the .ini file, including updated sections."""
return set(self.updater.sections())
@property
def initial_contents(self) -> str:
"""Suggest the initial content for this missing file."""
return self.get_missing_output()
@property
def expected_sections(self) -> set[str]:
"""Expected sections (from the style config)."""
return set(self.expected_config.keys())
@property
def missing_sections(self) -> set[str]:
"""Missing sections."""
return self.expected_sections - self.current_sections
def write_file(self, file_exists: bool) -> Fuss | None:
"""Write the new file."""
try:
if self.needs_top_section:
self.file_path.write_text(self.contents_without_top_section(str(self.updater)))
return None
if file_exists:
self.updater.update_file()
else:
self.updater.write(self.file_path.open("w"))
except ParsingError as err:
return self.reporter.make_fuss(Violations.PARSING_ERROR, cls=err.__class__.__name__, msg=err)
return None
@staticmethod
def contents_without_top_section(multiline_text: str) -> str:
"""Remove the temporary top section from multiline text, and keep the newline at the end of the file."""
return "\n".join(line for line in multiline_text.splitlines() if TOP_SECTION not in line) + "\n"
def get_missing_output(self) -> str:
"""Get a missing output string example from the missing sections in an INI file."""
missing = self.missing_sections
if not missing:
return ""
parser = ConfigParser()
for section in sorted(missing, key=lambda s: "0" if s == TOP_SECTION else f"1{s}"):
expected_config: dict = self.expected_config[section]
if self.autofix:
if self.updater.last_block:
self.updater.last_block.add_after.space(1)
self.updater.add_section(section)
self.updater[section].update(expected_config)
self.dirty = True
parser[section] = expected_config
return self.contents_without_top_section(self.get_example_cfg(parser))
# TODO: refactor: convert the contents to dict (with IniConfig().sections?) and mimic other plugins doing dict diffs
def enforce_rules(self) -> Iterator[Fuss]:
"""Enforce rules on missing sections and missing key/value pairs in an INI file."""
try:
yield from self._read_file()
except Error:
return
yield from self.enforce_missing_sections()
csv_sections = {v.split(SECTION_SEPARATOR)[0] for v in self.comma_separated_values}
missing_csv = csv_sections.difference(self.current_sections)
if missing_csv:
yield self.reporter.make_fuss(
Violations.INVALID_COMMA_SEPARATED_VALUES_SECTION, ", ".join(sorted(missing_csv))
)
# Don't continue if the comma-separated values are invalid
return
for section in self.expected_sections.intersection(self.current_sections) - self.missing_sections:
yield from self.enforce_section(section)
def _read_file(self) -> Iterator[Fuss]:
"""Read the .ini file or special files like .editorconfig."""
parsing_err: Error | None = None
try:
self.updater.read(str(self.file_path))
except MissingSectionHeaderError as err:
if self.needs_top_section:
original_contents = self.file_path.read_text()
self.updater.read_string(f"[{TOP_SECTION}]\n{original_contents}")
return
# If this is not an .editorconfig file, report this as a regular parsing error
parsing_err = err
except DuplicateOptionError as err:
parsing_err = err
if not parsing_err:
return
# Don't change the file if there was a parsing error
self.autofix = False
yield self.reporter.make_fuss(Violations.PARSING_ERROR, cls=parsing_err.__class__.__name__, msg=parsing_err)
raise Error
def enforce_missing_sections(self) -> Iterator[Fuss]:
"""Enforce missing sections."""
missing = self.get_missing_output()
if missing:
yield self.reporter.make_fuss(Violations.MISSING_SECTIONS, missing, self.autofix)
def enforce_section(self, section: str) -> Iterator[Fuss]:
"""Enforce rules for a section."""
expected_dict = self.expected_config[section]
actual_dict = {k: v.value for k, v in self.updater[section].items()}
# TODO: refactor: add a class Ini(BaseDoc) and move this dictdiffer code there
for diff_type, key, values in dictdiffer.diff(actual_dict, expected_dict):
if diff_type == dictdiffer.CHANGE:
if f"{section}.{key}" in self.comma_separated_values:
yield from self.enforce_comma_separated_values(section, key, values[0], values[1])
else:
yield from self.compare_different_keys(section, key, values[0], values[1])
elif diff_type == dictdiffer.ADD:
yield from self.show_missing_keys(section, values)
def enforce_comma_separated_values(self, section, key, raw_actual: Any, raw_expected: Any) -> Iterator[Fuss]:
"""Enforce sections and keys with comma-separated values. The values might contain spaces."""
actual_set = {s.strip() for s in raw_actual.split(",")}
expected_set = {s.strip() for s in raw_expected.split(",")}
missing = expected_set - actual_set
if not missing:
return
joined_values = ",".join(sorted(missing))
value_to_append = f",{joined_values}"
if self.autofix:
self.updater[section][key].value += value_to_append
self.dirty = True
section_header = "" if section == TOP_SECTION else f"[{section}]\n"
# TODO: test: top section with separated values in https://github.com/andreoliwa/nitpick/issues/271
yield self.reporter.make_fuss(
Violations.MISSING_VALUES_IN_LIST,
f"{section_header}{key} = (...){value_to_append}",
key=key,
fixed=self.autofix,
)
def compare_different_keys(self, section, key, raw_actual: Any, raw_expected: Any) -> Iterator[Fuss]:
"""Compare different keys, with special treatment when they are lists or numeric."""
if isinstance(raw_actual, (int, float, bool)) or isinstance(raw_expected, (int, float, bool)):
# A boolean "True" or "true" has the same effect on ConfigParser files.
actual = str(raw_actual).lower()
expected = str(raw_expected).lower()
else:
actual = raw_actual
expected = raw_expected
if actual == expected:
return
if self.autofix:
self.updater[section][key].value = expected
self.dirty = True
if section == TOP_SECTION:
yield self.reporter.make_fuss(
Violations.TOP_SECTION_HAS_DIFFERENT_VALUE,
f"{key} = {raw_expected}",
key=key,
actual=raw_actual,
fixed=self.autofix,
)
else:
yield self.reporter.make_fuss(
Violations.OPTION_HAS_DIFFERENT_VALUE,
f"[{section}]\n{key} = {raw_expected}",
section=section,
key=key,
actual=raw_actual,
fixed=self.autofix,
)
def show_missing_keys(self, section: str, values: list[tuple[str, Any]]) -> Iterator[Fuss]:
"""Show the keys that are not present in a section."""
parser = ConfigParser()
missing_dict = dict(values)
parser[section] = missing_dict
output = self.get_example_cfg(parser)
self.add_options_before_space(section, missing_dict)
if section == TOP_SECTION:
yield self.reporter.make_fuss(
Violations.TOP_SECTION_MISSING_OPTION, self.contents_without_top_section(output), self.autofix
)
else:
yield self.reporter.make_fuss(Violations.MISSING_OPTION, output, self.autofix, section=section)
def add_options_before_space(self, section: str, options: dict) -> None:
"""Add new options before a blank line in the end of the section."""
if not self.autofix:
return
space_removed = False
while isinstance(self.updater[section].last_block, Space):
space_removed = True
self.updater[section].last_block.detach()
self.updater[section].update(options)
self.dirty = True
if space_removed:
self.updater[section].last_block.add_after.space(1)
@staticmethod
def get_example_cfg(parser: ConfigParser) -> str:
"""Print an example of a config parser in a string instead of a file."""
string_stream = StringIO()
parser.write(string_stream)
output = string_stream.getvalue().strip()
return output
class CCFile:
def __init__(self, filename):
self.filename = filename
self.updater = ConfigUpdater()
self.updater.read(filename)
for section in self.updater.sections_blocks():
validate(section, filename)
self._readonly = None
self._last_modified = time.time()
self.slm = {}
def __iter__(self) -> CCNode:
for section in self.updater.sections_blocks():
yield CCNode(self, section)
@property
def readonly(self) -> bool:
file = None
try:
file = open(self.filename, 'w')
except:
readonly = True
else:
readonly = False
finally:
if file is not None:
file.close()
return readonly
def write(self):
with contextlib.suppress(Exception):
self.updater.update_file()
def move_after(self, section: Section, reference: Section = None):
self.updater.remove_section(section.name)
if reference is None:
sections = self.updater.sections()
if len(sections) > 0:
reference = self.updater[sections[0]]
reference.add_before.section(section)
else:
self.updater.add_section(section)
else:
reference.add_after.section(section)
self.tick()
def add_node(self) -> Optional[CCNode]:
if self.readonly is True:
return None
while 1:
id = uuid.uuid4().hex
try:
self.updater.add_section(id)
except DuplicateSectionError:
continue
except:
return None
else:
break
return CCNode(self, self.updater[id])
def remove_node(self, section: Section) -> bool:
if self.readonly is True:
return False
self.updater.remove_section(section.name)
return True
@property
def last_modified(self):
return int(self._last_modified)
def tick(self, last_modified=None):
self.write()
if last_modified is None:
last_modified = time.time()
self._last_modified = last_modified
def get_section_last_modified(self, id) -> int:
return self.slm.get(id, 0)
def set_section_last_modified(self, id, last_modified=None):
if last_modified is None:
last_modified = time.time()
self.slm[id] = last_modified
class EncryptedConfigFile(object):
"""Wrap encrypted config files.
Manages keys based on the data in the configuration. Also allows
management of additional files with the same keys.
"""
def __init__(self,
encrypted_file,
add_files_for_env: Optional[str] = None,
write_lock=False,
quiet=False):
self.add_files_for_env = add_files_for_env
self.write_lock = write_lock
self.quiet = quiet
self.files = {}
self.main_file = self.add_file(encrypted_file)
# Add all existing files to the session
if self.add_files_for_env:
for path in iter_other_secrets(self.add_files_for_env):
self.add_file(path)
def add_file(self, filename):
# Ensure compatibility with pathlib.
filename = str(filename)
if filename not in self.files:
self.files[filename] = f = EncryptedFile(filename, self.write_lock,
self.quiet)
f.read()
return self.files[filename]
def __enter__(self):
self.main_file.__enter__()
# Ensure `self.config`
self.read()
return self
def __exit__(self, _exc_type=None, _exc_value=None, _traceback=None):
self.main_file.__exit__()
if not self.get_members():
os.unlink(self.main_file.encrypted_filename)
def read(self):
self.main_file.read()
if not self.main_file.cleartext:
self.main_file.cleartext = NEW_FILE_TEMPLATE
self.config = ConfigUpdater()
self.config.read_string(self.main_file.cleartext)
self.set_members(self.get_members())
def write(self):
s = io.StringIO()
self.config.write(s)
self.main_file.cleartext = s.getvalue()
for file in self.files.values():
file.recipients = self.get_members()
file.write()
def get_members(self):
if 'batou' not in self.config:
self.config.add_section('batou')
try:
members = self.config.get("batou", "members").value.split(",")
except Exception:
return []
members = [x.strip() for x in members]
members = [_f for _f in members if _f]
members.sort()
return members
def set_members(self, members):
# The whitespace here is exactly what
# "members = " looks like in the config file so we get
# proper indentation.
members = ",\n ".join(members)
self.config.set("batou", "members", members)