def set_load_config(database_path = None, cluster_cookie=None):
"""
Help to set the mission data loading.
Parameters
----------
database_path : str
Absolute path where to save all the downloaded files.
It not, the default path is ~/heliopy/
cluster_cookie : str
Cookie from ESA to download cluster data.
"""
#Get the path of heliopyrc file
config_path = get_config_file()
# Read the preexisting config file using configupdater
config = ConfigUpdater()
config.read(config_path)
# Set new data download directory if passed
if database_path:
config['DEFAULT']['download_dir'].value = database_path
# Set new cluster cookie if passed
if cluster_cookie:
config['DEFAULT']['cluster_cookie'].value = cluster_cookie
# Update the config file with new entries
config.update_file()
def test_update_no_changes(setup_cfg_path):
updater = ConfigUpdater()
updater.read(setup_cfg_path)
old_mtime = os.path.getmtime(setup_cfg_path)
updater.update_file()
new_mtime = os.path.getmtime(setup_cfg_path)
assert old_mtime != new_mtime
def add_new_SFUI(df_final):
updater = ConfigUpdater()
updater.read('setup.cfg')
# Subset into assigned and unassigned
df = df_final[df_final['SFUI'] == '']
df_final = df_final[df_final['SFUI'] != '']
if df.empty:
return df_final
else:
# Sort by SF
df = df.sort_values(by=['SF'])
df = df.reset_index(drop=True)
# Assign SFUI
assignment = int(updater['metadata']['sfui_last_assignment'].value) + 1
for index, row in df.iterrows():
if index == 0:
df['SFUI'].iat[index] = assignment
elif df['SF'].at[index] == df['SF'].at[index - 1]:
df['SFUI'].iat[index] = assignment
else:
assignment += 1
df['SFUI'].iat[index] = assignment
# Format SFUI
df['SFUI'] = 'S' + (df.SFUI.map('{:06}'.format))
# Add back newly assigned
df_final = pd.concat([df_final, df])
df_final = df_final.reset_index(drop=True)
# Update config file
updater['metadata']['sfui_last_assignment'].value = assignment
updater.update_file()
# Return dataframe
return df_final
def change_language(new_language: str) -> None:
"""
Updates the language in the config file
:param new_language: The new language code. Can be any of 'en', 'ro' and 'fr'
"""
updater = ConfigUpdater()
updater.read('settings.config')
updater['APPLICATION_SETTINGS']['language'].value = new_language
updater.update_file()
def test_update_file_issue_68(tmp_path):
# allow_no_value = False
file = tmp_path / "file.cfg"
file.write_text("[section]")
cfg = ConfigUpdater(allow_no_value=False).read_file(open(file))
cfg.set("section", "option")
with pytest.warns(NoneValueDisallowed):
# A warning is issued, but the method will not fail
cfg.update_file()
assert file.read_text().strip() == "[section]"
# allow_no_value = True
file.write_text("[section]")
cfg = ConfigUpdater(allow_no_value=True).read_file(open(file))
cfg.set("section", "option")
cfg.update_file()
assert file.read_text().strip() == "[section]\noption"
def test_update_no_cfg():
updater = ConfigUpdater()
with pytest.raises(NoConfigFileReadError):
updater.update_file()
class IniPlugin(NitpickPlugin):
"""Enforce configurations and autofix INI files.
Examples of ``.ini`` files handled by this plugin:
- `setup.cfg <https://docs.python.org/3/distutils/configfile.html>`_
- `.editorconfig <https://editorconfig.org/>`_
- `tox.ini <https://github.com/tox-dev/tox>`_
- `.pylintrc <https://pylint.readthedocs.io/en/latest/user_guide/run.html#command-line-options>`_
Style examples enforcing values on INI files: :gitref:`flake8 configuration
<src/nitpick/resources/python/flake8.toml>`.
"""
fixable = True
identify_tags = {"ini", "editorconfig"}
violation_base_code = 320
updater: ConfigUpdater
comma_separated_values: set[str]
def post_init(self):
"""Post initialization after the instance was created."""
self.updater = ConfigUpdater()
self.comma_separated_values = set(self.nitpick_file_dict.get(COMMA_SEPARATED_VALUES, []))
if not self.needs_top_section:
return
if all(isinstance(v, dict) for v in self.expected_config.values()):
return
new_config = dict({TOP_SECTION: {}})
for key, value in self.expected_config.items():
if isinstance(value, dict):
new_config[key] = value
continue
new_config[TOP_SECTION][key] = value
self.expected_config = new_config
@property
def needs_top_section(self) -> bool:
"""Return True if this .ini file needs a top section (e.g.: .editorconfig)."""
return "editorconfig" in self.info.tags
@property
def current_sections(self) -> set[str]:
"""Current sections of the .ini file, including updated sections."""
return set(self.updater.sections())
@property
def initial_contents(self) -> str:
"""Suggest the initial content for this missing file."""
return self.get_missing_output()
@property
def expected_sections(self) -> set[str]:
"""Expected sections (from the style config)."""
return set(self.expected_config.keys())
@property
def missing_sections(self) -> set[str]:
"""Missing sections."""
return self.expected_sections - self.current_sections
def write_file(self, file_exists: bool) -> Fuss | None:
"""Write the new file."""
try:
if self.needs_top_section:
self.file_path.write_text(self.contents_without_top_section(str(self.updater)))
return None
if file_exists:
self.updater.update_file()
else:
self.updater.write(self.file_path.open("w"))
except ParsingError as err:
return self.reporter.make_fuss(Violations.PARSING_ERROR, cls=err.__class__.__name__, msg=err)
return None
@staticmethod
def contents_without_top_section(multiline_text: str) -> str:
"""Remove the temporary top section from multiline text, and keep the newline at the end of the file."""
return "\n".join(line for line in multiline_text.splitlines() if TOP_SECTION not in line) + "\n"
def get_missing_output(self) -> str:
"""Get a missing output string example from the missing sections in an INI file."""
missing = self.missing_sections
if not missing:
return ""
parser = ConfigParser()
for section in sorted(missing, key=lambda s: "0" if s == TOP_SECTION else f"1{s}"):
expected_config: dict = self.expected_config[section]
if self.autofix:
if self.updater.last_block:
self.updater.last_block.add_after.space(1)
self.updater.add_section(section)
self.updater[section].update(expected_config)
self.dirty = True
parser[section] = expected_config
return self.contents_without_top_section(self.get_example_cfg(parser))
# TODO: refactor: convert the contents to dict (with IniConfig().sections?) and mimic other plugins doing dict diffs
def enforce_rules(self) -> Iterator[Fuss]:
"""Enforce rules on missing sections and missing key/value pairs in an INI file."""
try:
yield from self._read_file()
except Error:
return
yield from self.enforce_missing_sections()
csv_sections = {v.split(SECTION_SEPARATOR)[0] for v in self.comma_separated_values}
missing_csv = csv_sections.difference(self.current_sections)
if missing_csv:
yield self.reporter.make_fuss(
Violations.INVALID_COMMA_SEPARATED_VALUES_SECTION, ", ".join(sorted(missing_csv))
)
# Don't continue if the comma-separated values are invalid
return
for section in self.expected_sections.intersection(self.current_sections) - self.missing_sections:
yield from self.enforce_section(section)
def _read_file(self) -> Iterator[Fuss]:
"""Read the .ini file or special files like .editorconfig."""
parsing_err: Error | None = None
try:
self.updater.read(str(self.file_path))
except MissingSectionHeaderError as err:
if self.needs_top_section:
original_contents = self.file_path.read_text()
self.updater.read_string(f"[{TOP_SECTION}]\n{original_contents}")
return
# If this is not an .editorconfig file, report this as a regular parsing error
parsing_err = err
except DuplicateOptionError as err:
parsing_err = err
if not parsing_err:
return
# Don't change the file if there was a parsing error
self.autofix = False
yield self.reporter.make_fuss(Violations.PARSING_ERROR, cls=parsing_err.__class__.__name__, msg=parsing_err)
raise Error
def enforce_missing_sections(self) -> Iterator[Fuss]:
"""Enforce missing sections."""
missing = self.get_missing_output()
if missing:
yield self.reporter.make_fuss(Violations.MISSING_SECTIONS, missing, self.autofix)
def enforce_section(self, section: str) -> Iterator[Fuss]:
"""Enforce rules for a section."""
expected_dict = self.expected_config[section]
actual_dict = {k: v.value for k, v in self.updater[section].items()}
# TODO: refactor: add a class Ini(BaseDoc) and move this dictdiffer code there
for diff_type, key, values in dictdiffer.diff(actual_dict, expected_dict):
if diff_type == dictdiffer.CHANGE:
if f"{section}.{key}" in self.comma_separated_values:
yield from self.enforce_comma_separated_values(section, key, values[0], values[1])
else:
yield from self.compare_different_keys(section, key, values[0], values[1])
elif diff_type == dictdiffer.ADD:
yield from self.show_missing_keys(section, values)
def enforce_comma_separated_values(self, section, key, raw_actual: Any, raw_expected: Any) -> Iterator[Fuss]:
"""Enforce sections and keys with comma-separated values. The values might contain spaces."""
actual_set = {s.strip() for s in raw_actual.split(",")}
expected_set = {s.strip() for s in raw_expected.split(",")}
missing = expected_set - actual_set
if not missing:
return
joined_values = ",".join(sorted(missing))
value_to_append = f",{joined_values}"
if self.autofix:
self.updater[section][key].value += value_to_append
self.dirty = True
section_header = "" if section == TOP_SECTION else f"[{section}]\n"
# TODO: test: top section with separated values in https://github.com/andreoliwa/nitpick/issues/271
yield self.reporter.make_fuss(
Violations.MISSING_VALUES_IN_LIST,
f"{section_header}{key} = (...){value_to_append}",
key=key,
fixed=self.autofix,
)
def compare_different_keys(self, section, key, raw_actual: Any, raw_expected: Any) -> Iterator[Fuss]:
"""Compare different keys, with special treatment when they are lists or numeric."""
if isinstance(raw_actual, (int, float, bool)) or isinstance(raw_expected, (int, float, bool)):
# A boolean "True" or "true" has the same effect on ConfigParser files.
actual = str(raw_actual).lower()
expected = str(raw_expected).lower()
else:
actual = raw_actual
expected = raw_expected
if actual == expected:
return
if self.autofix:
self.updater[section][key].value = expected
self.dirty = True
if section == TOP_SECTION:
yield self.reporter.make_fuss(
Violations.TOP_SECTION_HAS_DIFFERENT_VALUE,
f"{key} = {raw_expected}",
key=key,
actual=raw_actual,
fixed=self.autofix,
)
else:
yield self.reporter.make_fuss(
Violations.OPTION_HAS_DIFFERENT_VALUE,
f"[{section}]\n{key} = {raw_expected}",
section=section,
key=key,
actual=raw_actual,
fixed=self.autofix,
)
def show_missing_keys(self, section: str, values: list[tuple[str, Any]]) -> Iterator[Fuss]:
"""Show the keys that are not present in a section."""
parser = ConfigParser()
missing_dict = dict(values)
parser[section] = missing_dict
output = self.get_example_cfg(parser)
self.add_options_before_space(section, missing_dict)
if section == TOP_SECTION:
yield self.reporter.make_fuss(
Violations.TOP_SECTION_MISSING_OPTION, self.contents_without_top_section(output), self.autofix
)
else:
yield self.reporter.make_fuss(Violations.MISSING_OPTION, output, self.autofix, section=section)
def add_options_before_space(self, section: str, options: dict) -> None:
"""Add new options before a blank line in the end of the section."""
if not self.autofix:
return
space_removed = False
while isinstance(self.updater[section].last_block, Space):
space_removed = True
self.updater[section].last_block.detach()
self.updater[section].update(options)
self.dirty = True
if space_removed:
self.updater[section].last_block.add_after.space(1)
@staticmethod
def get_example_cfg(parser: ConfigParser) -> str:
"""Print an example of a config parser in a string instead of a file."""
string_stream = StringIO()
parser.write(string_stream)
output = string_stream.getvalue().strip()
return output
class CCFile:
def __init__(self, filename):
self.filename = filename
self.updater = ConfigUpdater()
self.updater.read(filename)
for section in self.updater.sections_blocks():
validate(section, filename)
self._readonly = None
self._last_modified = time.time()
self.slm = {}
def __iter__(self) -> CCNode:
for section in self.updater.sections_blocks():
yield CCNode(self, section)
@property
def readonly(self) -> bool:
file = None
try:
file = open(self.filename, 'w')
except:
readonly = True
else:
readonly = False
finally:
if file is not None:
file.close()
return readonly
def write(self):
with contextlib.suppress(Exception):
self.updater.update_file()
def move_after(self, section: Section, reference: Section = None):
self.updater.remove_section(section.name)
if reference is None:
sections = self.updater.sections()
if len(sections) > 0:
reference = self.updater[sections[0]]
reference.add_before.section(section)
else:
self.updater.add_section(section)
else:
reference.add_after.section(section)
self.tick()
def add_node(self) -> Optional[CCNode]:
if self.readonly is True:
return None
while 1:
id = uuid.uuid4().hex
try:
self.updater.add_section(id)
except DuplicateSectionError:
continue
except:
return None
else:
break
return CCNode(self, self.updater[id])
def remove_node(self, section: Section) -> bool:
if self.readonly is True:
return False
self.updater.remove_section(section.name)
return True
@property
def last_modified(self):
return int(self._last_modified)
def tick(self, last_modified=None):
self.write()
if last_modified is None:
last_modified = time.time()
self._last_modified = last_modified
def get_section_last_modified(self, id) -> int:
return self.slm.get(id, 0)
def set_section_last_modified(self, id, last_modified=None):
if last_modified is None:
last_modified = time.time()
self.slm[id] = last_modified
run_test_content = re.sub(
r'(.*cd\s+amrex.+git checkout\s+)(.+)(\s+&&\s.*)',
r'\g<1>{}\g<3>'.format(amrex_new_branch),
run_test_content, flags = re.MULTILINE)
with open(run_test_path, "w", encoding='utf-8') as f:
f.write(run_test_content)
if ConfigUpdater is not None:
# WarpX-tests.ini
tests_ini_path = str(REPO_DIR.joinpath("Regression/WarpX-tests.ini"))
cp = ConfigUpdater()
cp.optionxform = str
cp.read(tests_ini_path)
cp['AMReX']['branch'].value = amrex_new_branch
cp.update_file()
# WarpX-GPU-tests.ini
tests_gpu_ini_path = str(REPO_DIR.joinpath("Regression/WarpX-GPU-tests.ini"))
cp = ConfigUpdater()
cp.optionxform = str
cp.read(tests_gpu_ini_path)
cp['AMReX']['branch'].value = amrex_new_branch
cp.update_file()
# WarpX references to AMReX: cmake/dependencies/AMReX.cmake
with open(amrex_cmake_path, encoding='utf-8') as f:
amrex_cmake_content = f.read()
# branch/commit/tag (git fetcher) version
# set(WarpX_amrex_branch "development" ...
def save_setting(property, value, section='STORE', filename='config.ini'):
updater = ConfigUpdater()
updater.read(filename)
updater[section][property].value = value
updater.update_file()
