def parse_config_and_write_with_configupdater():
ini = ConfigUpdater()
ini.read("logman.ini", encoding='windows-1251')
print_ini_updater(ini)
option = ini['REM_Log_Tolyan_10']['daysold']
val = float(option.value)
print(f'[{option.section.name}].{option.key}: {type(val)} = "{val}"')
print(delimiter)
section_name = 'Interface'
interface = ini[section_name]
par_name = 'iconblinkinterval'
val = interface[par_name].value
print(f'{section_name}.{par_name} = {val}')
par_name = 'dummy_int'
val = interface.get(par_name, default=Option(par_name, '0')).value
val = val if val else 0
print(f'{section_name}.{par_name} = {val}')
par_name = 'HotKey'
val = interface[par_name].value
val = val if val else 0
print(f'{section_name}.{par_name} = {val}')
print(delimiter)
# Writing count demo...
section_name = 'System'
system = ini[section_name]
par_name = 'multiline'
# try:
# # WTF! In this place exception configparser.DuplicateOptionError not raised!
# # It raised only during ini.update_file() !
# # So at this place we have duplicate option 'multiline' without even any warnings
# system.add_option(system.create_option(
# key=par_name,
# value="""
# 'Ok, it's'
# ' a multiline with quotation'
# ' and indentation test value!'
# """
# ))
# except Exception as e:
# # configparser.DuplicateOptionError
# # print(e)
# pass
if not system.get(par_name):
system.add_option(
system.create_option(key=par_name,
value="""
'Ok, it's'
' a multiline with quotation'
' and indentation test value!'
"""))
# Fallback/default values does not work for me for a while...
# par_name = 'Count'
# # count = system[par_name].value
# # system[par_name].value raises exception if no such option, but system.get return None
# count = system.get(key=par_name)
# count = count if count else 0
# count += 1
# system[par_name].value = str(count)
with open('logman.ini', 'w', encoding='windows-1251') as f:
ini.write(f)
class IniPlugin(NitpickPlugin):
"""Enforce configurations and autofix INI files.
Examples of ``.ini`` files handled by this plugin:
- `setup.cfg <https://docs.python.org/3/distutils/configfile.html>`_
- `.editorconfig <https://editorconfig.org/>`_
- `tox.ini <https://github.com/tox-dev/tox>`_
- `.pylintrc <https://pylint.readthedocs.io/en/latest/user_guide/run.html#command-line-options>`_
Style examples enforcing values on INI files: :gitref:`flake8 configuration
<src/nitpick/resources/python/flake8.toml>`.
"""
fixable = True
identify_tags = {"ini", "editorconfig"}
violation_base_code = 320
updater: ConfigUpdater
comma_separated_values: set[str]
def post_init(self):
"""Post initialization after the instance was created."""
self.updater = ConfigUpdater()
self.comma_separated_values = set(self.nitpick_file_dict.get(COMMA_SEPARATED_VALUES, []))
if not self.needs_top_section:
return
if all(isinstance(v, dict) for v in self.expected_config.values()):
return
new_config = dict({TOP_SECTION: {}})
for key, value in self.expected_config.items():
if isinstance(value, dict):
new_config[key] = value
continue
new_config[TOP_SECTION][key] = value
self.expected_config = new_config
@property
def needs_top_section(self) -> bool:
"""Return True if this .ini file needs a top section (e.g.: .editorconfig)."""
return "editorconfig" in self.info.tags
@property
def current_sections(self) -> set[str]:
"""Current sections of the .ini file, including updated sections."""
return set(self.updater.sections())
@property
def initial_contents(self) -> str:
"""Suggest the initial content for this missing file."""
return self.get_missing_output()
@property
def expected_sections(self) -> set[str]:
"""Expected sections (from the style config)."""
return set(self.expected_config.keys())
@property
def missing_sections(self) -> set[str]:
"""Missing sections."""
return self.expected_sections - self.current_sections
def write_file(self, file_exists: bool) -> Fuss | None:
"""Write the new file."""
try:
if self.needs_top_section:
self.file_path.write_text(self.contents_without_top_section(str(self.updater)))
return None
if file_exists:
self.updater.update_file()
else:
self.updater.write(self.file_path.open("w"))
except ParsingError as err:
return self.reporter.make_fuss(Violations.PARSING_ERROR, cls=err.__class__.__name__, msg=err)
return None
@staticmethod
def contents_without_top_section(multiline_text: str) -> str:
"""Remove the temporary top section from multiline text, and keep the newline at the end of the file."""
return "\n".join(line for line in multiline_text.splitlines() if TOP_SECTION not in line) + "\n"
def get_missing_output(self) -> str:
"""Get a missing output string example from the missing sections in an INI file."""
missing = self.missing_sections
if not missing:
return ""
parser = ConfigParser()
for section in sorted(missing, key=lambda s: "0" if s == TOP_SECTION else f"1{s}"):
expected_config: dict = self.expected_config[section]
if self.autofix:
if self.updater.last_block:
self.updater.last_block.add_after.space(1)
self.updater.add_section(section)
self.updater[section].update(expected_config)
self.dirty = True
parser[section] = expected_config
return self.contents_without_top_section(self.get_example_cfg(parser))
# TODO: refactor: convert the contents to dict (with IniConfig().sections?) and mimic other plugins doing dict diffs
def enforce_rules(self) -> Iterator[Fuss]:
"""Enforce rules on missing sections and missing key/value pairs in an INI file."""
try:
yield from self._read_file()
except Error:
return
yield from self.enforce_missing_sections()
csv_sections = {v.split(SECTION_SEPARATOR)[0] for v in self.comma_separated_values}
missing_csv = csv_sections.difference(self.current_sections)
if missing_csv:
yield self.reporter.make_fuss(
Violations.INVALID_COMMA_SEPARATED_VALUES_SECTION, ", ".join(sorted(missing_csv))
)
# Don't continue if the comma-separated values are invalid
return
for section in self.expected_sections.intersection(self.current_sections) - self.missing_sections:
yield from self.enforce_section(section)
def _read_file(self) -> Iterator[Fuss]:
"""Read the .ini file or special files like .editorconfig."""
parsing_err: Error | None = None
try:
self.updater.read(str(self.file_path))
except MissingSectionHeaderError as err:
if self.needs_top_section:
original_contents = self.file_path.read_text()
self.updater.read_string(f"[{TOP_SECTION}]\n{original_contents}")
return
# If this is not an .editorconfig file, report this as a regular parsing error
parsing_err = err
except DuplicateOptionError as err:
parsing_err = err
if not parsing_err:
return
# Don't change the file if there was a parsing error
self.autofix = False
yield self.reporter.make_fuss(Violations.PARSING_ERROR, cls=parsing_err.__class__.__name__, msg=parsing_err)
raise Error
def enforce_missing_sections(self) -> Iterator[Fuss]:
"""Enforce missing sections."""
missing = self.get_missing_output()
if missing:
yield self.reporter.make_fuss(Violations.MISSING_SECTIONS, missing, self.autofix)
def enforce_section(self, section: str) -> Iterator[Fuss]:
"""Enforce rules for a section."""
expected_dict = self.expected_config[section]
actual_dict = {k: v.value for k, v in self.updater[section].items()}
# TODO: refactor: add a class Ini(BaseDoc) and move this dictdiffer code there
for diff_type, key, values in dictdiffer.diff(actual_dict, expected_dict):
if diff_type == dictdiffer.CHANGE:
if f"{section}.{key}" in self.comma_separated_values:
yield from self.enforce_comma_separated_values(section, key, values[0], values[1])
else:
yield from self.compare_different_keys(section, key, values[0], values[1])
elif diff_type == dictdiffer.ADD:
yield from self.show_missing_keys(section, values)
def enforce_comma_separated_values(self, section, key, raw_actual: Any, raw_expected: Any) -> Iterator[Fuss]:
"""Enforce sections and keys with comma-separated values. The values might contain spaces."""
actual_set = {s.strip() for s in raw_actual.split(",")}
expected_set = {s.strip() for s in raw_expected.split(",")}
missing = expected_set - actual_set
if not missing:
return
joined_values = ",".join(sorted(missing))
value_to_append = f",{joined_values}"
if self.autofix:
self.updater[section][key].value += value_to_append
self.dirty = True
section_header = "" if section == TOP_SECTION else f"[{section}]\n"
# TODO: test: top section with separated values in https://github.com/andreoliwa/nitpick/issues/271
yield self.reporter.make_fuss(
Violations.MISSING_VALUES_IN_LIST,
f"{section_header}{key} = (...){value_to_append}",
key=key,
fixed=self.autofix,
)
def compare_different_keys(self, section, key, raw_actual: Any, raw_expected: Any) -> Iterator[Fuss]:
"""Compare different keys, with special treatment when they are lists or numeric."""
if isinstance(raw_actual, (int, float, bool)) or isinstance(raw_expected, (int, float, bool)):
# A boolean "True" or "true" has the same effect on ConfigParser files.
actual = str(raw_actual).lower()
expected = str(raw_expected).lower()
else:
actual = raw_actual
expected = raw_expected
if actual == expected:
return
if self.autofix:
self.updater[section][key].value = expected
self.dirty = True
if section == TOP_SECTION:
yield self.reporter.make_fuss(
Violations.TOP_SECTION_HAS_DIFFERENT_VALUE,
f"{key} = {raw_expected}",
key=key,
actual=raw_actual,
fixed=self.autofix,
)
else:
yield self.reporter.make_fuss(
Violations.OPTION_HAS_DIFFERENT_VALUE,
f"[{section}]\n{key} = {raw_expected}",
section=section,
key=key,
actual=raw_actual,
fixed=self.autofix,
)
def show_missing_keys(self, section: str, values: list[tuple[str, Any]]) -> Iterator[Fuss]:
"""Show the keys that are not present in a section."""
parser = ConfigParser()
missing_dict = dict(values)
parser[section] = missing_dict
output = self.get_example_cfg(parser)
self.add_options_before_space(section, missing_dict)
if section == TOP_SECTION:
yield self.reporter.make_fuss(
Violations.TOP_SECTION_MISSING_OPTION, self.contents_without_top_section(output), self.autofix
)
else:
yield self.reporter.make_fuss(Violations.MISSING_OPTION, output, self.autofix, section=section)
def add_options_before_space(self, section: str, options: dict) -> None:
"""Add new options before a blank line in the end of the section."""
if not self.autofix:
return
space_removed = False
while isinstance(self.updater[section].last_block, Space):
space_removed = True
self.updater[section].last_block.detach()
self.updater[section].update(options)
self.dirty = True
if space_removed:
self.updater[section].last_block.add_after.space(1)
@staticmethod
def get_example_cfg(parser: ConfigParser) -> str:
"""Print an example of a config parser in a string instead of a file."""
string_stream = StringIO()
parser.write(string_stream)
output = string_stream.getvalue().strip()
return output
class EncryptedConfigFile(object):
"""Wrap encrypted config files.
Manages keys based on the data in the configuration. Also allows
management of additional files with the same keys.
"""
def __init__(self,
encrypted_file,
add_files_for_env: Optional[str] = None,
write_lock=False,
quiet=False):
self.add_files_for_env = add_files_for_env
self.write_lock = write_lock
self.quiet = quiet
self.files = {}
self.main_file = self.add_file(encrypted_file)
# Add all existing files to the session
if self.add_files_for_env:
for path in iter_other_secrets(self.add_files_for_env):
self.add_file(path)
def add_file(self, filename):
# Ensure compatibility with pathlib.
filename = str(filename)
if filename not in self.files:
self.files[filename] = f = EncryptedFile(filename, self.write_lock,
self.quiet)
f.read()
return self.files[filename]
def __enter__(self):
self.main_file.__enter__()
# Ensure `self.config`
self.read()
return self
def __exit__(self, _exc_type=None, _exc_value=None, _traceback=None):
self.main_file.__exit__()
if not self.get_members():
os.unlink(self.main_file.encrypted_filename)
def read(self):
self.main_file.read()
if not self.main_file.cleartext:
self.main_file.cleartext = NEW_FILE_TEMPLATE
self.config = ConfigUpdater()
self.config.read_string(self.main_file.cleartext)
self.set_members(self.get_members())
def write(self):
s = io.StringIO()
self.config.write(s)
self.main_file.cleartext = s.getvalue()
for file in self.files.values():
file.recipients = self.get_members()
file.write()
def get_members(self):
if 'batou' not in self.config:
self.config.add_section('batou')
try:
members = self.config.get("batou", "members").value.split(",")
except Exception:
return []
members = [x.strip() for x in members]
members = [_f for _f in members if _f]
members.sort()
return members
def set_members(self, members):
# The whitespace here is exactly what
# "members = " looks like in the config file so we get
# proper indentation.
members = ",\n ".join(members)
self.config.set("batou", "members", members)
class EncryptedConfigFile(object):
"""Wrap encrypted config files."""
lockfd = None
_cleartext = None
# Additional GPG parameters. Used for testing.
gpg_opts = ""
GPG_BINARY_CANDIDATES = ["gpg", "gpg2"]
def __init__(self, encrypted_file, write_lock=False, quiet=False):
"""Context manager that opens an encrypted file.
Use the read() and write() methods in the subordinate "with"
block to manipulate cleartext content. If the cleartext content
has been replaced, the encrypted file is updated.
`write_lock` must be set True if a modification of the file is
intended.
"""
self.encrypted_file = encrypted_file
self.write_lock = write_lock
self.quiet = quiet
def __enter__(self):
self._lock()
return self
def __exit__(self, _exc_type, _exc_value, _traceback):
self.lockfd.close()
def gpg(self, cmdline):
null = tempfile.TemporaryFile()
for gpg in self.GPG_BINARY_CANDIDATES:
try:
subprocess.check_call([gpg, "--version"],
stdout=null,
stderr=null)
except (subprocess.CalledProcessError, OSError):
pass
else:
return "{} {}".format(gpg, cmdline)
raise RuntimeError("Could not find gpg binary."
" Is GPG installed? I tried looking for: {}".format(
", ".join("`{}`".format(x)
for x in self.GPG_BINARY_CANDIDATES)))
@property
def cleartext(self):
if self._cleartext is None:
return NEW_FILE_TEMPLATE
return self._cleartext
@cleartext.setter
def cleartext(self, value):
self.config = ConfigUpdater()
self.config.read_string(value)
self.set_members(self.get_members())
s = io.StringIO()
self.config.write(s)
self._cleartext = s.getvalue()
def read(self):
if self._cleartext is None:
if os.stat(self.encrypted_file).st_size:
self._decrypt()
return self.cleartext
def write(self, cleartext):
"""Replace encrypted file with new content."""
if not self.write_lock:
raise RuntimeError("write() needs a write lock")
self.cleartext = cleartext
self._encrypt()
def write_config(self):
s = io.StringIO()
self.config.write(s)
self.write(s.getvalue())
def _lock(self):
self.lockfd = open(self.encrypted_file, self.write_lock and "a+"
or "r+")
try:
if self.write_lock:
fcntl.lockf(self.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
else:
fcntl.lockf(self.lockfd, fcntl.LOCK_SH | fcntl.LOCK_NB)
except BlockingIOError:
raise FileLockedError(self.encrypted_file)
def _decrypt(self):
opts = self.gpg_opts
if self.quiet:
opts += " -q --no-tty --batch"
self.cleartext = subprocess.check_output(
[self.gpg("{} --decrypt {}".format(opts, self.encrypted_file))],
stderr=NULL,
shell=True,
).decode("utf-8")
def get_members(self):
members = self.config.get("batou", "members").value.split(",")
members = [x.strip() for x in members]
members = [_f for _f in members if _f]
members.sort()
return members
def set_members(self, members):
# The whitespace here is exactly what
# "members = " looks like in the config file so we get
# proper indentation.
members = ",\n ".join(members)
self.config.set("batou", "members", members)
def _encrypt(self):
recipients = self.get_members()
if not recipients:
raise ValueError("Need at least one recipient.")
self.set_members(self.get_members())
recipients = " ".join(
["-r {}".format(shlex.quote(r.strip())) for r in recipients])
os.rename(self.encrypted_file, self.encrypted_file + ".old")
try:
gpg = subprocess.Popen(
[
self.gpg("{} --encrypt {} -o {}".format(
self.gpg_opts, recipients, self.encrypted_file))
],
stdin=subprocess.PIPE,
shell=True,
)
gpg.communicate(self.cleartext.encode("utf-8"))
if gpg.returncode != 0:
raise RuntimeError("GPG returned non-zero exit code.")
except Exception:
os.rename(self.encrypted_file + ".old", self.encrypted_file)
raise
else:
os.unlink(self.encrypted_file + ".old")
else:
logger.debug('setting [{}]->{}={}'.format(sec, key,
values[sec][key]))
updater[sec][key] = values[sec][key]
if values.get('_global_'):
for key in values.get('_global_'):
for secname in updater.sections():
if updater.has_option(secname, key):
if values['_global_'][key] == 'comment_out' and updater[
secname].get(key):
logger.debug('commenting out [{}]->{}={}'.format(
secname, key, updater[secname][key].value))
updater[secname][key].key = '#{}'.format(key)
else:
updater[secname][key] = values['_global_'][key]
logger.debug('{} found in [{}] setting to {}'.format(
key, secname, values['_global_'][key]))
if args.get('output'):
logger.debug('writing output: {}'.format(args.get('output')))
output_file_handle = open(args['output'], 'w')
else:
logger.debug('writing output: stdout')
output_file_handle = sys.stdout
updater.write(output_file_handle)
if output_file_handle is not sys.stdout:
output_file_handle.close()