def __init__(self,
arg_parser=default_arg_parser,
key_value_delimiters=['=', ':'],
comment_seperators=[],
key_delimiters=[','],
section_override_delimiters=["."]):
"""
CliParser parses arguments from the command line or a custom list of items that my look like this:
['-a', '-b', 'b1', 'b2', 'setting=value', 'section.setting=other_value', 'key1,section.key2=value2']
:param arg_parser: Instance of ArgParser() that is used to parse none-setting arguments
:param key_value_delimiters: delimiters to separate key and value in setting arguments
:param comment_seperators: allowed prefixes for comments
:param key_delimiters: delimiter to separate multiple keys of a setting argument
:param section_override_delimiters: The delimiter to delimit the section from the key name
(e.g. the '.' in section.key = value)
"""
if not isinstance(arg_parser, argparse.ArgumentParser):
raise TypeError("arg_parser must be an ArgumentParser")
SectionParser.__init__(self)
self._arg_parser = arg_parser
self._line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
{},
section_override_delimiters)
self.__reset_sections()
def __init__(self,
arg_parser=default_arg_parser,
key_value_delimiters=['=', ':'],
comment_seperators=[],
key_delimiters=[','],
section_override_delimiters=["."]):
"""
CliParser parses arguments from the command line or a custom list of items that my look like this:
['-a', '-b', 'b1', 'b2', 'setting=value', 'section.setting=other_value', 'key1,section.key2=value2']
:param arg_parser: Instance of ArgParser() that is used to parse none-setting arguments
:param key_value_delimiters: delimiters to separate key and value in setting arguments
:param comment_seperators: allowed prefixes for comments
:param key_delimiters: delimiter to separate multiple keys of a setting argument
:param section_override_delimiters: The delimiter to delimit the section from the key name
(e.g. the '.' in section.key = value)
"""
if not isinstance(arg_parser, argparse.ArgumentParser):
raise TypeError("arg_parser must be an ArgumentParser")
SectionParser.__init__(self)
self._arg_parser = arg_parser
self._line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
{},
section_override_delimiters)
self.__reset_sections()
def __init__(self,
key_value_delimiters=('=', ),
comment_seperators=('#', ),
key_delimiters=(',', ' '),
section_name_surroundings=MappingProxyType({"[": "]"}),
remove_empty_iter_elements=True):
self.line_parser = LineParser(key_value_delimiters, comment_seperators,
key_delimiters,
section_name_surroundings)
self.__remove_empty_iter_elements = remove_empty_iter_elements
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def __init__(self,
key_value_delimiters=['=', ':'],
comment_seperators=['#', ';', '//'],
key_delimiters=[',', ' '],
section_name_surroundings={'[': "]"}):
SectionParser.__init__(self)
self.line_parser = LineParser(key_value_delimiters, comment_seperators,
key_delimiters,
section_name_surroundings)
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
if sys.version_info < (3, 3): # pragma: no cover
self.FileNotFoundError = IOError
else:
self.FileNotFoundError = FileNotFoundError
def parse_cli(arg_list=None,
origin=os.getcwd(),
arg_parser=None,
key_value_delimiters=('=', ':'),
comment_seperators=(),
key_delimiters=(',',),
section_override_delimiters=(".",)):
"""
Parses the CLI arguments and creates sections out of it.
:param arg_list: The CLI argument list.
:param origin: Directory used to interpret relative
paths given as argument.
:param arg_parser: Instance of ArgParser that is used to
parse none-setting arguments.
:param key_value_delimiters: Delimiters to separate key and value
in setting arguments.
:param comment_seperators: Allowed prefixes for comments.
:param key_delimiters: Delimiter to separate multiple keys of
a setting argument.
:param section_override_delimiters: The delimiter to delimit the section
from the key name (e.g. the '.' in
sect.key = value).
:return: A dictionary holding section names
as keys and the sections themselves
as value.
"""
# Note: arg_list can also be []. Hence we cannot use
# `arg_list = arg_list or default_list`
arg_list = sys.argv[1:] if arg_list is None else arg_list
arg_parser = arg_parser or default_arg_parser()
origin += os.path.sep
sections = OrderedDict(default=Section('Default'))
line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
{},
section_override_delimiters)
for arg_key, arg_value in sorted(
vars(arg_parser.parse_args(arg_list)).items()):
if arg_key == 'settings' and arg_value is not None:
parse_custom_settings(sections,
arg_value,
origin,
line_parser)
else:
if isinstance(arg_value, list):
arg_value = ",".join([str(val) for val in arg_value])
append_to_sections(sections,
arg_key,
arg_value,
origin,
from_cli=True)
return sections
def __init__(self,
key_value_delimiters=('=',),
comment_seperators=('#', ';', '//'),
key_delimiters=(',', ' '),
section_name_surroundings=None):
section_name_surroundings = section_name_surroundings or {"[": "]"}
self.line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
section_name_surroundings)
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def __init__(self,
key_value_delimiters=('=',),
comment_seperators=('#',),
key_delimiters=(',', ' '),
section_name_surroundings=MappingProxyType({'[': ']'}),
remove_empty_iter_elements=True):
self.line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
section_name_surroundings)
self.__remove_empty_iter_elements = remove_empty_iter_elements
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def __init__(self,
key_value_delimiters=['=', ':'],
comment_seperators=['#', ';', '//'],
key_delimiters=[',', ' '],
section_name_surroundings={'[': "]"}):
SectionParser.__init__(self)
self.line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
section_name_surroundings)
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
if sys.version_info < (3, 3): # pragma: no cover
self.FileNotFoundError = IOError
else:
self.FileNotFoundError = FileNotFoundError
class ConfParser:
def __init__(self,
key_value_delimiters=('=',),
comment_seperators=('#',),
key_delimiters=(',', ' '),
section_name_surroundings=None,
remove_empty_iter_elements=True):
section_name_surroundings = section_name_surroundings or {"[": "]"}
self.line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
section_name_surroundings)
self.__remove_empty_iter_elements = remove_empty_iter_elements
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def parse(self, input_data, overwrite=False):
"""
Parses the input and adds the new data to the existing.
:param input_data: The filename to parse from.
:param overwrite: If True, wipes all existing Settings inside this
instance and adds only the newly parsed ones. If
False, adds the newly parsed data to the existing
one (and overwrites already existing keys with the
newly parsed values).
:return: A dictionary with (lowercase) section names as keys
and their Setting objects as values.
"""
if os.path.isdir(input_data):
input_data = os.path.join(input_data, Constants.default_coafile)
with open(input_data, "r", encoding='utf-8') as _file:
lines = _file.readlines()
if overwrite:
self.__init_sections()
self.__parse_lines(lines, input_data)
return self.sections
def get_section(self, name, create_if_not_exists=False):
key = self.__refine_key(name)
sec = self.sections.get(key, None)
if sec is not None:
return sec
if not create_if_not_exists:
raise IndexError
retval = self.sections[key] = Section(str(name),
self.sections["default"])
return retval
@staticmethod
def __refine_key(key):
return str(key).lower().strip()
def __add_comment(self, section, comment, origin):
key = "comment" + str(self.__rand_helper)
self.__rand_helper += 1
section.append(Setting(
key,
comment,
origin,
remove_empty_iter_elements=self.__remove_empty_iter_elements))
def __parse_lines(self, lines, origin):
current_section_name = "default"
current_section = self.get_section(current_section_name)
current_keys = []
for line in lines:
section_name, keys, value, comment = self.line_parser.parse(line)
if comment != "":
self.__add_comment(current_section, comment, origin)
if section_name != "":
current_section_name = section_name
current_section = self.get_section(current_section_name, True)
current_keys = []
continue
if comment == "" and keys == [] and value == "":
self.__add_comment(current_section, "", origin)
continue
if keys != []:
current_keys = keys
for section_override, key in current_keys:
if key == "":
continue
if section_override == "":
current_section.add_or_create_setting(
Setting(key,
value,
origin,
# Ignore PEP8Bear, it fails to format that
remove_empty_iter_elements=
self.__remove_empty_iter_elements),
allow_appending=(keys == []))
else:
self.get_section(
section_override,
True).add_or_create_setting(
Setting(key,
value,
origin,
# Ignore PEP8Bear, it fails to format that
remove_empty_iter_elements=
self.__remove_empty_iter_elements),
allow_appending=(keys == []))
def __init_sections(self):
self.sections = OrderedDict()
self.sections["default"] = Section("Default")
self.__rand_helper = 0
class ConfParser:
def __init__(self,
key_value_delimiters=('=', ),
comment_seperators=('#', ),
key_delimiters=(',', ' '),
section_name_surroundings=MappingProxyType({"[": "]"}),
remove_empty_iter_elements=True):
self.line_parser = LineParser(key_value_delimiters, comment_seperators,
key_delimiters,
section_name_surroundings)
self.__remove_empty_iter_elements = remove_empty_iter_elements
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def parse(self, input_data, overwrite=False):
"""
Parses the input and adds the new data to the existing.
:param input_data: The filename to parse from.
:param overwrite: If True, wipes all existing Settings inside this
instance and adds only the newly parsed ones. If
False, adds the newly parsed data to the existing
one (and overwrites already existing keys with the
newly parsed values).
:return: A dictionary with (lowercase) section names as keys
and their Setting objects as values.
"""
if os.path.isdir(input_data):
input_data = os.path.join(input_data, Constants.default_coafile)
with open(input_data, "r", encoding='utf-8') as _file:
lines = _file.readlines()
if overwrite:
self.__init_sections()
self.__parse_lines(lines, input_data)
return self.sections
def get_section(self, name, create_if_not_exists=False):
key = self.__refine_key(name)
sec = self.sections.get(key, None)
if sec is not None:
return sec
if not create_if_not_exists:
raise IndexError
retval = self.sections[key] = Section(str(name),
self.sections["default"])
return retval
@staticmethod
def __refine_key(key):
return str(key).lower().strip()
def __add_comment(self, section, comment, origin):
key = "comment" + str(self.__rand_helper)
self.__rand_helper += 1
section.append(
Setting(
key,
comment,
origin,
remove_empty_iter_elements=self.__remove_empty_iter_elements))
def __parse_lines(self, lines, origin):
current_section_name = "default"
current_section = self.get_section(current_section_name)
current_keys = []
for line in lines:
section_name, keys, value, comment = self.line_parser.parse(line)
if comment != "":
self.__add_comment(current_section, comment, origin)
if section_name != "":
current_section_name = section_name
current_section = self.get_section(current_section_name, True)
current_keys = []
continue
if comment == "" and keys == [] and value == "":
self.__add_comment(current_section, "", origin)
continue
if keys != []:
current_keys = keys
for section_override, key in current_keys:
if key == "":
continue
if section_override == "":
current_section.add_or_create_setting(
Setting(
key,
value,
origin,
# Ignore PEP8Bear, it fails to format that
remove_empty_iter_elements=self.
__remove_empty_iter_elements),
allow_appending=(keys == []))
else:
self.get_section(
section_override, True).add_or_create_setting(
Setting(
key,
value,
origin,
# Ignore PEP8Bear, it fails to format that
remove_empty_iter_elements=self.
__remove_empty_iter_elements),
allow_appending=(keys == []))
def __init_sections(self):
self.sections = OrderedDict()
self.sections["default"] = Section("Default")
self.__rand_helper = 0
class CliParser(SectionParser):
def __init__(self,
arg_parser=default_arg_parser,
key_value_delimiters=['=', ':'],
comment_seperators=[],
key_delimiters=[','],
section_override_delimiters=["."]):
"""
CliParser parses arguments from the command line or a custom list of items that my look like this:
['-a', '-b', 'b1', 'b2', 'setting=value', 'section.setting=other_value', 'key1,section.key2=value2']
:param arg_parser: Instance of ArgParser() that is used to parse none-setting arguments
:param key_value_delimiters: delimiters to separate key and value in setting arguments
:param comment_seperators: allowed prefixes for comments
:param key_delimiters: delimiter to separate multiple keys of a setting argument
:param section_override_delimiters: The delimiter to delimit the section from the key name
(e.g. the '.' in section.key = value)
"""
if not isinstance(arg_parser, argparse.ArgumentParser):
raise TypeError("arg_parser must be an ArgumentParser")
SectionParser.__init__(self)
self._arg_parser = arg_parser
self._line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
{},
section_override_delimiters)
self.__reset_sections()
def __reset_sections(self):
self.sections = OrderedDict(default=Section('Default'))
def _update_sections(self, section_name, key, value, origin):
if key == '' or value is None:
return
if section_name == "" or section_name is None:
section_name = "default"
if not section_name in self.sections:
self.sections[section_name] = Section(section_name)
self.sections[section_name].append(Setting(key, str(value), origin, from_cli=True))
def parse(self, arg_list=sys.argv[1:], origin=os.getcwd()):
"""
parses the input and adds the new data to the existing
:param arg_list: list of arguments.
:param origin: directory used to interpret relative paths given as argument
:return: the settings dictionary
"""
origin += os.path.sep
for arg_key, arg_value in sorted(vars(self._arg_parser.parse_args(arg_list)).items()):
if arg_key == 'settings' and arg_value is not None:
self._parse_custom_settings(arg_value, origin)
else:
if isinstance(arg_value, list):
arg_value = ",".join([str(val) for val in arg_value]) # [1,2,3] -> "1,2,3"
self._update_sections("default", arg_key, arg_value, origin)
return self.sections
def _parse_custom_settings(self, custom_settings_list, origin):
for setting_definition in custom_settings_list:
section_stub, key_touples, value, comment_stub = self._line_parser.parse(setting_definition)
for key_touple in key_touples:
self._update_sections(section_name=key_touple[0], key=key_touple[1], value=value, origin=origin)
def reparse(self, arg_list=sys.argv[1:], origin=os.getcwd()):
self.__reset_sections()
return self.parse(arg_list, origin)
def export_to_settings(self):
return self.sections
class ConfParser:
if sys.version_info < (3, 3): # pragma: no cover
FileNotFoundError = IOError
else:
FileNotFoundError = FileNotFoundError
def __init__(self,
key_value_delimiters=('=',),
comment_seperators=('#', ';', '//'),
key_delimiters=(',', ' '),
section_name_surroundings=None):
section_name_surroundings = section_name_surroundings or {"[": "]"}
self.line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
section_name_surroundings)
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def parse(self, input_data, overwrite=False):
"""
Parses the input and adds the new data to the existing. If you want to
catch the FileNotFoundError please take the FileNotFoundError member of
this object for catching for backwards compatability to python 3.2.
:param input_data: filename
:param overwrite: behaves like reparse if this is True
:return: the settings dictionary
"""
if os.path.isdir(input_data):
input_data = os.path.join(input_data, ".coafile")
with open(input_data, "r", encoding='utf-8') as _file:
lines = _file.readlines()
if overwrite:
self.__init_sections()
self.__parse_lines(lines, input_data)
return self.sections
def get_section(self, name, create_if_not_exists=False):
key = self.__refine_key(name)
sec = self.sections.get(key, None)
if sec is not None:
return sec
if not create_if_not_exists:
raise IndexError
retval = self.sections[key] = Section(str(name),
self.sections["default"])
return retval
@staticmethod
def __refine_key(key):
return str(key).lower().strip()
def __add_comment(self, section, comment, origin):
key = "comment" + str(self.__rand_helper)
self.__rand_helper += 1
section.append(Setting(key, comment, origin))
def __parse_lines(self, lines, origin):
current_section_name = "default"
current_section = self.get_section(current_section_name)
current_keys = []
for line in lines:
section_name, keys, value, comment = self.line_parser.parse(line)
if comment != "":
self.__add_comment(current_section, comment, origin)
if section_name != "":
current_section_name = section_name
current_section = self.get_section(current_section_name, True)
current_keys = []
continue
if comment == "" and keys == [] and value == "":
self.__add_comment(current_section, "", origin)
continue
if keys != []:
current_keys = keys
for section_override, key in current_keys:
if key == "":
continue
if section_override == "":
current_section.add_or_create_setting(
Setting(key, value, origin),
allow_appending=(keys == []))
else:
self.get_section(
section_override,
True).add_or_create_setting(
Setting(key, value, origin),
allow_appending=(keys == []))
def __init_sections(self):
self.sections = OrderedDict()
self.sections["default"] = Section("Default")
self.__rand_helper = 0
class ConfParser:
def __init__(self,
key_value_delimiters=('=',),
comment_seperators=('#',),
key_delimiters=(',', ' '),
section_name_surroundings=MappingProxyType({'[': ']'}),
remove_empty_iter_elements=True,
key_value_append_delimiters=('+=',)):
self.line_parser = LineParser(
key_value_delimiters,
comment_seperators,
key_delimiters,
section_name_surroundings,
key_value_append_delimiters=key_value_append_delimiters)
self.__remove_empty_iter_elements = remove_empty_iter_elements
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def parse(self, input_data, overwrite=False):
"""
Parses the input and adds the new data to the existing.
:param input_data: The filename to parse from.
:param overwrite: If True, wipes all existing Settings inside this
instance and adds only the newly parsed ones. If
False, adds the newly parsed data to the existing
one (and overwrites already existing keys with the
newly parsed values).
:return: A dictionary with (lowercase) section names as keys
and their Setting objects as values.
"""
if os.path.isdir(input_data):
input_data = os.path.join(input_data, Constants.default_coafile)
with open(input_data, 'r', encoding='utf-8') as _file:
lines = _file.readlines()
if overwrite:
self.__init_sections()
self.__parse_lines(lines, input_data)
return self.sections
def get_section(self, name, create_if_not_exists=False):
key = self.__refine_key(name)
sec = self.sections.get(key, None)
if sec is not None:
return sec
if not create_if_not_exists:
raise IndexError
retval = self.sections[key] = Section(str(name))
return retval
@staticmethod
def __refine_key(key):
return str(key).lower().strip()
def __add_comment(self, section, comment, origin):
key = 'comment' + str(self.__rand_helper)
self.__rand_helper += 1
section.append(Setting(
key,
comment,
origin,
remove_empty_iter_elements=self.__remove_empty_iter_elements))
def __parse_lines(self, lines, origin):
current_section_name = 'default'
current_section = self.get_section(current_section_name)
current_keys = []
no_section = True
for line in lines:
(section_name,
keys,
value,
append,
comment) = self.line_parser._parse(line)
if comment != '':
self.__add_comment(current_section, comment, origin)
if section_name != '':
no_section = False
current_section_name = section_name
current_section = self.get_section(current_section_name, True)
current_keys = []
continue
if comment == '' and keys == [] and value == '':
self.__add_comment(current_section, '', origin)
continue
if keys != []:
current_keys = keys
for section_override, key in current_keys:
if no_section:
logging.warning('A setting does not have a section.'
'This is a deprecated feature please '
'put this setting in a section defined'
' with `[<your-section-name]` in a '
'configuration file.')
if key == '':
continue
if section_override == '':
current_section.add_or_create_setting(
Setting(key,
value,
origin,
to_append=append,
# Start ignoring PEP8Bear, PycodestyleBear*
# they fail to resolve this
remove_empty_iter_elements=
self.__remove_empty_iter_elements),
# Stop ignoring
allow_appending=(keys == []))
else:
self.get_section(
section_override,
True).add_or_create_setting(
Setting(key,
value,
origin,
to_append=append,
# Start ignoring PEP8Bear, PycodestyleBear*
# they fail to resolve this
remove_empty_iter_elements=
self.__remove_empty_iter_elements),
# Stop ignoring
allow_appending=(keys == []))
def __init_sections(self):
self.sections = OrderedDict()
self.sections['default'] = Section('Default')
self.__rand_helper = 0
class ConfParser:
def __init__(self,
key_value_delimiters=('=', ),
comment_seperators=('#', ),
key_delimiters=(',', ' '),
section_name_surroundings=MappingProxyType({'[': ']'}),
remove_empty_iter_elements=True,
key_value_append_delimiters=('+=', )):
self.line_parser = LineParser(
key_value_delimiters,
comment_seperators,
key_delimiters,
section_name_surroundings,
key_value_append_delimiters=key_value_append_delimiters)
self.__remove_empty_iter_elements = remove_empty_iter_elements
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
def parse(self, input_data, overwrite=False):
"""
Parses the input and adds the new data to the existing.
:param input_data: The filename to parse from.
:param overwrite: If True, wipes all existing Settings inside this
instance and adds only the newly parsed ones. If
False, adds the newly parsed data to the existing
one (and overwrites already existing keys with the
newly parsed values).
:return: A dictionary with (lowercase) section names as keys
and their Setting objects as values.
"""
if os.path.isdir(input_data):
input_data = os.path.join(input_data, Constants.default_coafile)
with open(input_data, 'r', encoding='utf-8') as _file:
lines = _file.readlines()
if overwrite:
self.__init_sections()
self.__parse_lines(lines, input_data)
return self.sections
def get_section(self, name, create_if_not_exists=False):
key = self.__refine_key(name)
sec = self.sections.get(key, None)
if sec is not None:
return sec
if not create_if_not_exists:
raise IndexError
retval = self.sections[key] = Section(str(name))
return retval
@staticmethod
def __refine_key(key):
return str(key).lower().strip()
def __add_comment(self, section, comment, origin):
key = 'comment' + str(self.__rand_helper)
self.__rand_helper += 1
section.append(
Setting(
key,
comment,
origin,
remove_empty_iter_elements=self.__remove_empty_iter_elements))
def __parse_lines(self, lines, origin):
current_section_name = 'default'
current_section = self.get_section(current_section_name)
current_keys = []
no_section = True
for line in lines:
(section_name, keys, value, append,
comment) = self.line_parser._parse(line)
if comment != '':
self.__add_comment(current_section, comment, origin)
if section_name != '':
no_section = False
current_section_name = section_name
current_section = self.get_section(current_section_name, True)
current_keys = []
continue
if comment == '' and keys == [] and value == '':
self.__add_comment(current_section, '', origin)
continue
if keys != []:
current_keys = keys
for section_override, key in current_keys:
if no_section:
logging.warning('A setting does not have a section.'
'This is a deprecated feature please '
'put this setting in a section defined'
' with `[<your-section-name]` in a '
'configuration file.')
if key == '':
continue
if section_override == '':
current_section.add_or_create_setting(
Setting(
key,
value,
origin,
to_append=append,
# Start ignoring PEP8Bear, PycodestyleBear*
# they fail to resolve this
remove_empty_iter_elements=self.
__remove_empty_iter_elements),
# Stop ignoring
allow_appending=(keys == []))
else:
self.get_section(
section_override, True
).add_or_create_setting(
Setting(
key,
value,
origin,
to_append=append,
# Start ignoring PEP8Bear, PycodestyleBear*
# they fail to resolve this
remove_empty_iter_elements=self.
__remove_empty_iter_elements),
# Stop ignoring
allow_appending=(keys == []))
def __init_sections(self):
self.sections = OrderedDict()
self.sections['default'] = Section('Default')
self.__rand_helper = 0
class LineParserTest(unittest.TestCase):
def setUp(self):
self.uut = LineParser(comment_separators=('#', ';'))
def test_empty_line(self):
self.check_data_set('')
self.check_data_set('\n \n \n')
def test_comment_parsing(self):
self.check_data_set('# comment only$§\n',
output_comment='# comment only$§')
self.check_data_set(' ; comment only \n',
output_comment='; comment only')
self.check_data_set(' ; \\comment only \n',
output_comment='; comment only')
self.check_data_set('#', output_comment='#')
def test_section_override(self):
self.check_data_set(r'a.b, \a\.\b\ c=',
output_keys=[('a', 'b'), ('', r'\a.\b c')])
def test_escaping(self):
self.check_data_set("hello = world\ # yes here's a space",
output_keys=[('', 'hello')],
output_value='world\\ ',
output_comment="# yes here's a space")
def test_multi_value_parsing(self):
self.check_data_set(
'a, b\\ \\=, section.c= = :()&/ \\\\#heres a comment \n',
output_section='',
output_keys=[('', 'a'), ('', 'b ='), ('section', 'c')],
output_value='= :()&/ \\\\',
output_comment='#heres a comment')
def test_multi_line_parsing(self):
self.check_data_set(' a,b,d another value ',
output_value='a,b,d another value')
self.check_data_set(' a,b,d\\= another value ',
output_value='a,b,d\\= another value')
def test_section_name_parsing(self):
self.check_data_set(' [ a section name ] # with comment \n',
'a section name',
output_comment='# with comment')
self.check_data_set(' [ a section name] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\; ] \n',
'a section name;')
self.uut.section_name_surroundings['Section:'] = ''
self.check_data_set('[ sec]; thats a normal section',
output_section='sec',
output_comment='; thats a normal section')
self.check_data_set(' Section: sEc]\\\\; thats a new section',
output_section='sEc]\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\\\; thats a new section',
output_section='sec]\\\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\; thats a new section',
output_section='sec]\\; thats a new section')
def check_data_set(self,
line,
output_section='',
output_keys=None,
output_value='',
output_comment=''):
output_keys = output_keys or []
section_name, keys, value, comment = self.uut.parse(line)
self.assertEqual(section_name, output_section)
self.assertEqual(keys, output_keys)
self.assertEqual(value, output_value)
self.assertEqual(comment, output_comment)
def setUp(self):
self.uut = LineParser(comment_separators=('#', ';'))
def setUp(self):
self.uut = LineParser()
class CliParser(SectionParser):
def __init__(self,
arg_parser=default_arg_parser,
key_value_delimiters=['=', ':'],
comment_seperators=[],
key_delimiters=[','],
section_override_delimiters=["."]):
"""
CliParser parses arguments from the command line or a custom list of items that my look like this:
['-a', '-b', 'b1', 'b2', 'setting=value', 'section.setting=other_value', 'key1,section.key2=value2']
:param arg_parser: Instance of ArgParser() that is used to parse none-setting arguments
:param key_value_delimiters: delimiters to separate key and value in setting arguments
:param comment_seperators: allowed prefixes for comments
:param key_delimiters: delimiter to separate multiple keys of a setting argument
:param section_override_delimiters: The delimiter to delimit the section from the key name
(e.g. the '.' in section.key = value)
"""
if not isinstance(arg_parser, argparse.ArgumentParser):
raise TypeError("arg_parser must be an ArgumentParser")
SectionParser.__init__(self)
self._arg_parser = arg_parser
self._line_parser = LineParser(key_value_delimiters,
comment_seperators,
key_delimiters,
{},
section_override_delimiters)
self.__reset_sections()
def __reset_sections(self):
self.sections = OrderedDict(default=Section('Default'))
def _update_sections(self, section_name, key, value, origin):
if key == '' or value is None:
return
if section_name == "" or section_name is None:
section_name = "default"
if not section_name.lower() in self.sections:
self.sections[section_name.lower()] = Section(section_name)
self.sections[section_name.lower()].append(Setting(key,
str(value),
origin,
from_cli=True))
def parse(self, arg_list=sys.argv[1:], origin=os.getcwd()):
"""
parses the input and adds the new data to the existing
:param arg_list: list of arguments.
:param origin: directory used to interpret relative paths given as argument
:return: the settings dictionary
"""
origin += os.path.sep
for arg_key, arg_value in sorted(vars(self._arg_parser.parse_args(arg_list)).items()):
if arg_key == 'settings' and arg_value is not None:
self._parse_custom_settings(arg_value, origin)
else:
if isinstance(arg_value, list):
arg_value = ",".join([str(val) for val in arg_value]) # [1,2,3] -> "1,2,3"
self._update_sections("default", arg_key, arg_value, origin)
return self.sections
def _parse_custom_settings(self, custom_settings_list, origin):
for setting_definition in custom_settings_list:
section_stub, key_touples, value, comment_stub = self._line_parser.parse(setting_definition)
for key_touple in key_touples:
self._update_sections(section_name=key_touple[0], key=key_touple[1], value=value, origin=origin)
def reparse(self, arg_list=sys.argv[1:], origin=os.getcwd()):
self.__reset_sections()
return self.parse(arg_list, origin)
def export_to_settings(self):
return self.sections
class LineParserTest(unittest.TestCase):
def setUp(self):
self.uut = LineParser(comment_separators=('#', ';'))
def test_empty_line(self):
self.check_data_set('')
self.check_data_set('\n \n \n')
def test_comment_parsing(self):
self.check_data_set('# comment only$§\n',
output_comment='# comment only$§')
self.check_data_set(' ; comment only \n',
output_comment='; comment only')
self.check_data_set(' ; \\comment only \n',
output_comment='; comment only')
self.check_data_set('#', output_comment='#')
def test_section_override(self):
self.check_data_set(r'a.b, \a\.\b\ c=',
output_keys=[('a', 'b'), ('', r'\a.\b c')])
def test_escaping(self):
self.check_data_set("hello = world\ # yes here's a space",
output_keys=[('', 'hello')],
output_value='world\\ ',
output_comment="# yes here's a space")
def test_multi_value_parsing(self):
self.check_data_set(
'a, b\\ \\=, section.c= = :()&/ \\\\#heres a comment \n',
output_section='',
output_keys=[('', 'a'), ('', 'b ='), ('section', 'c')],
output_value='= :()&/ \\\\',
output_comment='#heres a comment')
def test_multi_line_parsing(self):
self.check_data_set(' a,b,d another value ',
output_value='a,b,d another value')
self.check_data_set(' a,b,d\\= another value ',
output_value='a,b,d\\= another value')
def test_section_name_parsing(self):
self.check_data_set(' [ a section name ] # with comment \n',
'a section name',
output_comment='# with comment')
self.check_data_set(' [ a section name] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\; ] \n',
'a section name;')
self.uut.section_name_surroundings['Section:'] = ''
self.check_data_set('[ sec]; thats a normal section',
output_section='sec',
output_comment='; thats a normal section')
self.check_data_set(' Section: sEc]\\\\; thats a new section',
output_section='sEc]\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\\\; thats a new section',
output_section='sec]\\\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\; thats a new section',
output_section='sec]\\; thats a new section')
def test_append_value_parsing(self):
self.check_data_set('a += b',
output_keys=[('', 'a')],
output_value='b',
output_append=True)
self.check_data_set('a = b',
output_keys=[('', 'a')],
output_value='b')
self.check_data_set('a \\+\\= b',
output_value='a \\+\\= b')
def check_data_set(self,
line,
output_section='',
output_keys=None,
output_value='',
output_append=False,
output_comment=''):
output_keys = output_keys or []
section_name, keys, value, append, comment = self.uut._parse(line)
self.assertEqual(section_name, output_section)
self.assertEqual(keys, output_keys)
self.assertEqual(value, output_value)
self.assertEqual(append, output_append)
self.assertEqual(comment, output_comment)
def test_deprecation(self):
logger = logging.getLogger()
with self.assertLogs(logger, 'WARNING') as cm:
self.uut.parse('')
self.assertRegex(cm.output[0], 'WARNING:root:The parse method of '
'LineParser is deprecated\.*')
def setUp(self):
self.uut = LineParser(comment_seperators=("#", ";"))
class LineParserTest(unittest.TestCase):
def setUp(self):
self.uut = LineParser(comment_seperators=("#", ";"))
def test_empty_line(self):
self.check_data_set("")
self.check_data_set("\n \n \n")
def test_comment_parsing(self):
self.check_data_set("# comment only$§\n", output_comment="# comment only$§")
self.check_data_set(" ; comment only \n", output_comment="; comment only")
self.check_data_set(" ; \\comment only \n", output_comment="; comment only")
self.check_data_set("#", output_comment="#")
def test_section_override(self):
self.check_data_set("a.b, \\a\\.\\b\\ c=", output_keys=[("a", "b"), ("", "a.b c")])
def test_multi_value_parsing(self):
self.check_data_set(
"a, b\\ \\=, section.c= = :()&/ \\\\#heres a comment \n",
output_section="",
output_keys=[("", "a"), ("", "b ="), ("section", "c")],
output_value="= :()&/ \\\\",
output_comment="#heres a comment",
)
def test_multi_line_parsing(self):
self.check_data_set(" a,b,d another value ", output_value="a,b,d another value")
self.check_data_set(" a,b,d\\= another value ", output_value="a,b,d\\= another value")
def test_section_name_parsing(self):
self.check_data_set(
" [ a section name ] # with comment \n", "a section name", output_comment="# with comment"
)
self.check_data_set(" [ a section name] ] \n", "a section name]")
self.check_data_set(" [ a section name\\] ] \n", "a section name]")
self.check_data_set(" [ a section name\\; ] \n", "a section name;")
self.uut.section_name_surroundings["Section:"] = ""
self.check_data_set(
"[ sec]; thats a normal section", output_section="sec", output_comment="; thats a normal section"
)
self.check_data_set(
" Section: sEc]\\\\; thats a new section", output_section="sEc]\\", output_comment="; thats a new section"
)
self.check_data_set(
" Section: sec]\\\\\\\\; thats a new section",
output_section="sec]\\\\",
output_comment="; thats a new section",
)
self.check_data_set(" Section: sec]\\\\\\; thats a new section", output_section="sec]\\; thats a new section")
def check_data_set(self, line, output_section="", output_keys=None, output_value="", output_comment=""):
output_keys = output_keys or []
section_name, keys, value, comment = self.uut.parse(line)
self.assertEqual(section_name, output_section)
self.assertEqual(keys, output_keys)
self.assertEqual(value, output_value)
self.assertEqual(comment, output_comment)
class LineParserTest(unittest.TestCase):
def setUp(self):
self.uut = LineParser(comment_separators=('#', ';'))
def test_empty_line(self):
with mock.patch('re.match', false_mock):
import re
self.check_data_set('')
self.check_data_set('\n \n \n')
def test_comment_parsing(self):
logger = logging.getLogger()
with mock.patch('re.match', false_mock):
import re
self.check_data_set('# comment only$§\n',
output_comment='# comment only$§')
self.check_data_set(' ; comment only \n',
output_comment='; comment only')
self.check_data_set(' ; \\comment only \n',
output_comment='; comment only')
self.assertEqual(re.match.called, True)
self.check_data_set('#', output_comment='#')
with self.assertLogs(logger, 'WARNING') as warn:
self.check_data_set('##\n', output_comment='##')
self.assertEqual(len(warn.output), 1)
self.assertEqual(
warn.output[0], 'WARNING:root:This comment does ' +
'not have whitespace before or ' + 'after # in: ' + repr('##') +
'. If you didn\'t mean to make ' +
'a comment, use a backslash for ' + 'escaping.')
with mock.patch('re.match', true_mock):
with self.assertLogs(logger, 'WARNING') as warn:
self.check_data_set('#A\n', output_comment='#A')
self.assertEqual(
warn.output[0], 'WARNING:root:This comment does ' +
'not have whitespace before or ' + 'after # in: ' +
repr('#A') + '. If you didn\'t mean to make ' +
'a comment, use a backslash for ' + 'escaping.')
def test_section_override(self):
self.check_data_set(r'a.b, \a\.\b\ c=',
output_keys=[('a', 'b'), ('', r'\a.\b c')])
def test_escaping(self):
self.check_data_set("hello = world\ # yes here's a space",
output_keys=[('', 'hello')],
output_value='world\\ ',
output_comment="# yes here's a space")
def test_multi_value_parsing(self):
self.check_data_set(
'a, b\\ \\=, section.c= = :()&/ \\\\#heres a comment \n',
output_section='',
output_keys=[('', 'a'), ('', 'b ='), ('section', 'c')],
output_value='= :()&/ \\\\',
output_comment='#heres a comment')
def test_multi_line_parsing(self):
self.check_data_set(' a,b,d another value ',
output_value='a,b,d another value')
self.check_data_set(' a,b,d\\= another value ',
output_value='a,b,d\\= another value')
def test_section_name_parsing(self):
self.check_data_set(' [ a section name ] # with comment \n',
'a section name',
output_comment='# with comment')
self.check_data_set(' [ a section name] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\; ] \n',
'a section name;')
self.uut.section_name_surroundings['Section:'] = ''
self.check_data_set('[ sec]; thats a normal section',
output_section='sec',
output_comment='; thats a normal section')
self.check_data_set(' Section: sEc]\\\\; thats a new section',
output_section='sEc]\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\\\; thats a new section',
output_section='sec]\\\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\; thats a new section',
output_section='sec]\\; thats a new section')
def test_append_value_parsing(self):
self.check_data_set('a += b',
output_keys=[('', 'a')],
output_value='b',
output_append=True)
self.check_data_set('a = b', output_keys=[('', 'a')], output_value='b')
self.check_data_set('a \\+\\= b', output_value='a \\+\\= b')
def check_data_set(self,
line,
output_section='',
output_keys=None,
output_value='',
output_append=False,
output_comment=''):
output_keys = output_keys or []
section_name, keys, value, append, comment = self.uut._parse(line)
self.assertEqual(section_name, output_section)
self.assertEqual(keys, output_keys)
self.assertEqual(value, output_value)
self.assertEqual(append, output_append)
self.assertEqual(comment, output_comment)
def test_deprecation(self):
logger = logging.getLogger()
with self.assertLogs(logger, 'WARNING') as cm:
self.uut.parse('')
self.assertRegex(
cm.output[0], 'WARNING:root:The parse method of '
'LineParser is deprecated\.*')
def setUp(self):
self.uut = LineParser()
class LineParserTest(unittest.TestCase):
def setUp(self):
self.uut = LineParser(comment_separators=('#', ';'))
def test_empty_line(self):
self.check_data_set("")
self.check_data_set("\n \n \n")
def test_comment_parsing(self):
self.check_data_set("# comment only$§\n",
output_comment="# comment only$§")
self.check_data_set(" ; comment only \n",
output_comment="; comment only")
self.check_data_set(" ; \\comment only \n",
output_comment="; comment only")
self.check_data_set("#", output_comment="#")
def test_section_override(self):
self.check_data_set(r"a.b, \a\.\b\ c=",
output_keys=[("a", "b"), ("", r"\a.\b c")])
def test_escaping(self):
self.check_data_set("hello = world\ # yes here's a space",
output_keys=[('', 'hello')],
output_value='world\\ ',
output_comment="# yes here's a space")
def test_multi_value_parsing(self):
self.check_data_set(
"a, b\\ \\=, section.c= = :()&/ \\\\#heres a comment \n",
output_section='',
output_keys=[("", 'a'), ("", 'b ='), ("section", 'c')],
output_value='= :()&/ \\\\',
output_comment='#heres a comment')
def test_multi_line_parsing(self):
self.check_data_set(" a,b,d another value ",
output_value="a,b,d another value")
self.check_data_set(" a,b,d\\= another value ",
output_value="a,b,d\\= another value")
def test_section_name_parsing(self):
self.check_data_set(" [ a section name ] # with comment \n",
'a section name',
output_comment="# with comment")
self.check_data_set(" [ a section name] ] \n",
'a section name]')
self.check_data_set(" [ a section name\\] ] \n",
'a section name]')
self.check_data_set(" [ a section name\\; ] \n",
'a section name;')
self.uut.section_name_surroundings["Section:"] = ''
self.check_data_set("[ sec]; thats a normal section",
output_section="sec",
output_comment="; thats a normal section")
self.check_data_set(" Section: sEc]\\\\; thats a new section",
output_section="sEc]\\",
output_comment="; thats a new section")
self.check_data_set(" Section: sec]\\\\\\\\; thats a new section",
output_section="sec]\\\\",
output_comment="; thats a new section")
self.check_data_set(" Section: sec]\\\\\\; thats a new section",
output_section="sec]\\; thats a new section")
def check_data_set(self,
line,
output_section="",
output_keys=None,
output_value='',
output_comment=''):
output_keys = output_keys or []
section_name, keys, value, comment = self.uut.parse(line)
self.assertEqual(section_name, output_section)
self.assertEqual(keys, output_keys)
self.assertEqual(value, output_value)
self.assertEqual(comment, output_comment)
class ConfParser(SectionParser):
def __init__(self,
key_value_delimiters=['=', ':'],
comment_seperators=['#', ';', '//'],
key_delimiters=[',', ' '],
section_name_surroundings={'[': "]"}):
SectionParser.__init__(self)
self.line_parser = LineParser(key_value_delimiters, comment_seperators,
key_delimiters,
section_name_surroundings)
# Declare it
self.sections = None
self.__rand_helper = None
self.__init_sections()
if sys.version_info < (3, 3): # pragma: no cover
self.FileNotFoundError = IOError
else:
self.FileNotFoundError = FileNotFoundError
def parse(self, input_data, overwrite=False):
"""
Parses the input and adds the new data to the existing. If you want to catch the FileNotFoundError please take
the FileNotFoundError member of this object for catching for backwards compatability to python 3.2.
:param input_data: filename
:param overwrite: behaves like reparse if this is True
:return: the settings dictionary
"""
if os.path.isdir(input_data):
input_data = os.path.join(input_data, ".coafile")
with open(input_data, "r", encoding='utf-8') as f:
lines = f.readlines()
if overwrite:
self.__init_sections()
self.__parse_lines(lines, input_data)
return self.export_to_settings()
def reparse(self, input_data):
"""
Parses the input and overwrites all existent data
:param input_data: filename
:return: the settings dictionary
"""
return self.parse(input_data, overwrite=True)
def export_to_settings(self):
"""
:return a dict of Settings objects representing the current parsed things
"""
return self.sections
def get_section(self, name, create_if_not_exists=False):
key = self.__refine_key(name)
sec = self.sections.get(key, None)
if sec is not None:
return sec
if not create_if_not_exists:
raise IndexError
retval = self.sections[key] = Section(str(name),
self.sections["default"])
return retval
@staticmethod
def __refine_key(key):
return str(key).lower().strip()
def __add_comment(self, section, comment, origin):
key = "comment" + str(self.__rand_helper)
self.__rand_helper += 1
section.append(Setting(key, comment, origin))
def __parse_lines(self, lines, origin):
current_section_name = "default"
current_section = self.get_section(current_section_name)
current_keys = []
for line in lines:
section_name, keys, value, comment = self.line_parser.parse(line)
if comment != "":
self.__add_comment(current_section, comment, origin)
if section_name != "":
current_section_name = section_name
current_section = self.get_section(current_section_name, True)
current_keys = []
continue
if comment == "" and keys == [] and value == "":
self.__add_comment(current_section, "", origin)
continue
if keys != []:
current_keys = keys
for section_override, key in current_keys:
if key == "":
continue
if section_override == "":
current_section.add_or_create_setting(
Setting(key, value, origin),
allow_appending=(keys == []))
else:
self.get_section(section_override,
True).add_or_create_setting(
Setting(key, value, origin),
allow_appending=(keys == []))
def __init_sections(self):
self.sections = OrderedDict()
self.sections["default"] = Section("Default")
self.__rand_helper = 0
def parse_cli(arg_list=None,
origin=os.getcwd(),
arg_parser=None,
args=None,
key_value_delimiters=('=', ':'),
comment_seperators=(),
key_delimiters=(',', ),
section_override_delimiters=('.', ),
key_value_append_delimiters=('+=', )):
"""
Parses the CLI arguments and creates sections out of it.
:param arg_list: The CLI argument list.
:param origin: Directory used to interpret relative
paths given as argument.
:param arg_parser: Instance of ArgParser that is used to
parse none-setting arguments.
:param args: Alternative pre-parsed CLI arguments.
:param key_value_delimiters: Delimiters to separate key and value
in setting arguments where settings are
being defined.
:param comment_seperators: Allowed prefixes for comments.
:param key_delimiters: Delimiter to separate multiple keys of
a setting argument.
:param section_override_delimiters: The delimiter to delimit the section
from the key name (e.g. the '.' in
sect.key = value).
:param key_value_append_delimiters: Delimiters to separate key and value
in setting arguments where settings are
being appended.
:return: A dictionary holding section names
as keys and the sections themselves
as value.
"""
assert not (arg_list and args), (
'Either call parse_cli() with an arg_list of CLI arguments or '
'with pre-parsed args, but not with both.')
if args is None:
arg_parser = default_arg_parser() if arg_parser is None else arg_parser
args = arg_parser.parse_args(arg_list)
origin += os.path.sep
sections = OrderedDict(cli=Section('cli'))
line_parser = LineParser(key_value_delimiters, comment_seperators,
key_delimiters, {}, section_override_delimiters,
key_value_append_delimiters)
for arg_key, arg_value in sorted(vars(args).items()):
if arg_key == 'settings' and arg_value is not None:
parse_custom_settings(sections, arg_value, origin, line_parser)
else:
if isinstance(arg_value, list):
arg_value = ','.join([str(val) for val in arg_value])
append_to_sections(sections,
arg_key,
arg_value,
origin,
section_name='cli',
from_cli=True)
return sections
class LineParserTest(unittest.TestCase):
def setUp(self):
self.uut = LineParser(comment_separators=('#', ';'))
def test_empty_line(self):
self.check_data_set('')
self.check_data_set('\n \n \n')
def test_comment_parsing(self):
self.check_data_set('# comment only$§\n',
output_comment='# comment only$§')
self.check_data_set(' ; comment only \n',
output_comment='; comment only')
self.check_data_set(' ; \\comment only \n',
output_comment='; comment only')
self.check_data_set('#', output_comment='#')
def test_section_override(self):
self.check_data_set(r'a.b, \a\.\b\ c=',
output_keys=[('a', 'b'), ('', r'\a.\b c')])
def test_escaping(self):
self.check_data_set("hello = world\ # yes here's a space",
output_keys=[('', 'hello')],
output_value='world\\ ',
output_comment="# yes here's a space")
def test_multi_value_parsing(self):
self.check_data_set(
'a, b\\ \\=, section.c= = :()&/ \\\\#heres a comment \n',
output_section='',
output_keys=[('', 'a'), ('', 'b ='), ('section', 'c')],
output_value='= :()&/ \\\\',
output_comment='#heres a comment')
def test_multi_line_parsing(self):
self.check_data_set(' a,b,d another value ',
output_value='a,b,d another value')
self.check_data_set(' a,b,d\\= another value ',
output_value='a,b,d\\= another value')
def test_section_name_parsing(self):
self.check_data_set(' [ a section name ] # with comment \n',
'a section name',
output_comment='# with comment')
self.check_data_set(' [ a section name] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\] ] \n',
'a section name]')
self.check_data_set(' [ a section name\\; ] \n',
'a section name;')
self.uut.section_name_surroundings['Section:'] = ''
self.check_data_set('[ sec]; thats a normal section',
output_section='sec',
output_comment='; thats a normal section')
self.check_data_set(' Section: sEc]\\\\; thats a new section',
output_section='sEc]\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\\\; thats a new section',
output_section='sec]\\\\',
output_comment='; thats a new section')
self.check_data_set(' Section: sec]\\\\\\; thats a new section',
output_section='sec]\\; thats a new section')
def check_data_set(self,
line,
output_section='',
output_keys=None,
output_value='',
output_comment=''):
output_keys = output_keys or []
section_name, keys, value, comment = self.uut.parse(line)
self.assertEqual(section_name, output_section)
self.assertEqual(keys, output_keys)
self.assertEqual(value, output_value)
self.assertEqual(comment, output_comment)
