def section(self: T, section: Union[str, "Section"]) -> T:
"""Creates a section block
Args:
section (str or :class:`Section`): name of section or object
Returns:
self for chaining
"""
from .document import Document
from .section import Section
container = self._container
if not isinstance(container, Document):
raise ValueError("Sections can only be added at section level!")
if isinstance(section, str):
# create a new section
section = Section(section)
elif not isinstance(section, Section):
msg = "Parameter must be a string or Section type!"
raise ValueError(msg, {"container": section})
if container.has_section(section.name):
raise DuplicateSectionError(section.name)
section.attach(container)
return self._insert(section)
def ns_sections(self, ns):
ns = ns.strip()
ns_sects = OrderedDict() # { id: str(section) }
for s in self._config.sections():
mat = _reNamespacedSection.match(s)
if mat is None:
continue
if mat.group('ns') != ns:
continue
id_ = mat.group('id')
if id_ in ns_sects:
raise DuplicateSectionError("%s:%s" % (ns, id_))
ns_sects[id_] = s
return [ (id_, s) for id_, s in ns_sects.items() ] # [ (id, str(section)) ]
def add_section(self, section, comment=None):
"""
Add a section
:param str section: Section to add
:raise DuplicateSectionError: if section already exist.
"""
self._read_sources()
if self._to_dot_key(section) in self._dot_keys:
raise DuplicateSectionError(section)
self._parser.add_section(section)
self._add_dot_key(section)
if comment:
self._set_comment(section, comment)
def add_section(self, section):
"""Create a new section in the configuration.
Raise DuplicateSectionError if a section by the specified name
already exists. Raise ValueError if name is DEFAULT or any of
its case-insensitive variants.
"""
# The default section is the only one that gets the case-insensitive
# treatment - so it is special-cased here.
if section.lower() == "default":
raise ValueError('Invalid section name: %s' % section)
if self.has_section(section):
raise DuplicateSectionError(section)
else:
self.data._new_namespace(section)
def add_section(self, section):
"""Create a new section in the configuration.
Raise DuplicateSectionError if a section by the specified name
already exists. Raise ValueError if name is DEFAULT.
Args:
section (str or :class:`Section`): name or Section type
"""
if section in self.sections():
raise DuplicateSectionError(section)
if isinstance(section, str):
# create a new section
section = Section(section, container=self)
elif not isinstance(section, Section):
raise ValueError("Parameter must be a string or Section type!")
self._structure.append(section)
def add_section(self, section: Union[str, Section]):
"""Create a new section in the configuration.
:raises: :exc:`~.DuplicateSectionError` if a section by the specified name already exists.
:raises: :exc:`ValueError` if name is ``DEFAULT``.
:param section:
"""
if section in self.sections():
raise DuplicateSectionError(section) # type: ignore
if isinstance(section, str):
# create a new section
section = Section(section, container=self)
elif not isinstance(section, Section):
raise ValueError("Parameter must be a string or Section type!")
self._structure.append(section)
def section(self, section):
"""Creates a section block
Args:
section (str or :class:`Section`): name of section or object
Returns:
self for chaining
"""
if not isinstance(self._container, ConfigUpdater):
raise ValueError("Sections can only be added at section level!")
if isinstance(section, str):
# create a new section
section = Section(section, container=self._container)
elif not isinstance(section, Section):
raise ValueError("Parameter must be a string or Section type!")
if section.name in [block.name for block in self._container
if isinstance(block, Section)]:
raise DuplicateSectionError(section.name)
self._container.structure.insert(self._idx, section)
self._idx += 1
return self
def add_section(self, section: Union[str, Section]):
"""Create a new section in the configuration.
Raise DuplicateSectionError if a section by the specified name
already exists. Raise ValueError if name is DEFAULT.
Args:
section (str or :class:`Section`): name or Section type
"""
if isinstance(section, str):
# create a new section
section_obj = Section(section)
elif isinstance(section, Section):
section_obj = section
else:
raise ValueError("Parameter must be a string or Section type!")
if self.has_section(section_obj.name):
raise DuplicateSectionError(section_obj.name)
section_obj.attach(self)
self._structure.append(section_obj)
def section(self, section: Union[str, "Section"]) -> "BlockBuilder":
"""
Creates a section block.
:param section: name of section or object.
:returns: self for chaining.
"""
if not isinstance(self._container, ConfigUpdater):
raise ValueError("Sections can only be added at section level!")
if isinstance(section, str):
# create a new section
section = Section(section, container=self._container)
elif not isinstance(section, Section):
raise ValueError("Parameter must be a string or Section type!")
if section.name in [block.name for block in self._container if isinstance(block, Section)]:
raise DuplicateSectionError(section.name)
self._container.structure.insert(self._idx, section)
self._idx += 1
return self
def add_section(self, section):
if section in self.values:
raise DuplicateSectionError(section)
self.values[section] = {}
def _read(self, fp, fpname):
"""Parse a sectioned configuration file.
Each section in a configuration file contains a header, indicated by
a name in square brackets (`[]`), plus key/value options, indicated by
`name` and `value` delimited with a specific substring (`=` or `:` by
default).
Values can span multiple lines, as long as they are indented deeper
than the first line of the value. Depending on the parser's mode, blank
lines may be treated as parts of multiline values or ignored.
Configuration files may include comments, prefixed by specific
characters (`#` and `;` by default). Comments may appear on their own
in an otherwise empty line or may be entered in lines holding values or
section names.
Note: This method was borrowed from ConfigParser and we keep this
mess here as close as possible to the original messod (pardon
this german pun) for consistency reasons and later upgrades.
"""
self._structure = []
elements_added = set()
cursect = None # None, or a dictionary
sectname = None
optname = None
lineno = 0
indent_level = 0
e = None # None, or an exception
for lineno, line in enumerate(fp, start=1):
comment_start = sys.maxsize
# strip inline comments
inline_prefixes = {p: -1 for p in self._inline_comment_prefixes}
while comment_start == sys.maxsize and inline_prefixes:
next_prefixes = {}
for prefix, index in inline_prefixes.items():
index = line.find(prefix, index + 1)
if index == -1:
continue
next_prefixes[prefix] = index
if index == 0 or (index > 0 and line[index - 1].isspace()):
comment_start = min(comment_start, index)
inline_prefixes = next_prefixes
# strip full line comments
for prefix in self._comment_prefixes:
if line.strip().startswith(prefix):
comment_start = 0
self._add_comment(line) # HOOK
break
if comment_start == sys.maxsize:
comment_start = None
value = line[:comment_start].strip()
if not value:
if self._empty_lines_in_values:
# add empty line to the value, but only if there was no
# comment on the line
if (comment_start is None and cursect is not None
and optname and cursect[optname] is not None):
cursect[optname].append('') # newlines added at join
self.last_item.last_item.add_line(line) # HOOK
else:
# empty line marks end of value
indent_level = sys.maxsize
if comment_start is None:
self._add_space(line)
continue
# continuation line?
first_nonspace = self.NONSPACECRE.search(line)
cur_indent_level = first_nonspace.start() if first_nonspace else 0
if (cursect is not None and optname
and cur_indent_level > indent_level):
cursect[optname].append(value)
self.last_item.last_item.add_line(line) # HOOK
# a section header or option header?
else:
indent_level = cur_indent_level
# is it a section header?
mo = self.SECTCRE.match(value)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
if self._strict and sectname in elements_added:
raise DuplicateSectionError(
sectname, fpname, lineno)
cursect = self._sections[sectname]
elements_added.add(sectname)
else:
cursect = self._dict()
self._sections[sectname] = cursect
elements_added.add(sectname)
# So sections can't start with a continuation line
optname = None
self._add_section(sectname, line) # HOOK
# no section header in the file?
elif cursect is None:
raise MissingSectionHeaderError(fpname, lineno, line)
# an option line?
else:
mo = self._optcre.match(value)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
if not optname:
e = self._handle_error(e, fpname, lineno, line)
optname = self.optionxform(optname.rstrip())
if (self._strict
and (sectname, optname) in elements_added):
raise DuplicateOptionError(sectname, optname,
fpname, lineno)
elements_added.add((sectname, optname))
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
optval = optval.strip()
cursect[optname] = [optval]
else:
# valueless option handling
cursect[optname] = None
self._add_option(optname, vi, optval, line) # HOOK
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
e = self._handle_error(e, fpname, lineno, line)
# if any parsing errors occurred, raise an exception
if e:
raise e
def _do_read_config(self, config_file, pommanipext):
"""Reads config for a single job defined by section."""
parser = InterpolationConfigParser()
dataset = parser.read(config_file)
if config_file not in dataset:
raise IOError("Config file %s not found." % config_file)
if parser.has_option('common', 'include'):
include = parser.get('common', 'include')
if include is not "":
sections_ = self.read_and_load(include)
for section_ in sections_:
if parser.has_section(section_):
raise DuplicateSectionError(
"The config section [%s] is existed in %s and include %s cfg file"
% (section_, config_file,
re.split("\\s+", include.strip())[1]))
parser._sections.update(sections_)
pom_manipulator_config = {}
common_section = {}
package_configs = {}
if pommanipext and pommanipext != '' and pommanipext != 'None': #TODO ref: remove none check, it is passed over cmd line in jenkins build
parse_pom_manipulator_ext(pom_manipulator_config, parser,
pommanipext)
if not parser.has_section('common'):
logging.error(
'Mandatory common section missing from configuration file.')
raise NoSectionError(
'Mandatory common section missing from configuration file.')
common_section['tag'] = parser.get('common', 'tag')
common_section['target'] = parser.get('common', 'target')
common_section['jobprefix'] = parser.get('common', 'jobprefix')
common_section['jobciprefix'] = parser.get('common', 'jobciprefix')
common_section['jobjdk'] = parser.get('common', 'jobjdk')
if parser.has_option('common', 'mvnver'):
common_section['mvnver'] = parser.get('common', 'mvnver')
if parser.has_option('common', 'skiptests'):
common_section['skiptests'] = parser.get('common', 'skiptests')
if parser.has_option('common', 'base'):
common_section['base'] = parser.get('common', 'base')
if parser.has_option('common', 'citemplate'):
common_section['citemplate'] = parser.get('common', 'citemplate')
if parser.has_option('common', 'jenkinstemplate'):
common_section['jenkinstemplate'] = parser.get(
'common', 'jenkinstemplate')
if parser.has_option('common', 'product_name'):
common_section['product_name'] = parser.get(
'common', 'product_name')
if parser.has_option('common', 'include'):
common_section['include'] = parser.get('common', 'include')
common_section['jobfailureemail'] = parser.get('common',
'jobfailureemail')
config_dir = utils.get_dir(config_file)
#Jira
if parser.has_option('common', 'shared_config') and parser.get(
'common', 'shared_config') is not "":
parse_shared_config(common_section, config_dir, parser)
common_section['jobtimeout'] = parser.getint('common', 'jobtimeout')
common_section['options'] = {}
# If the configuration file has global properties insert these into the common properties map.
# These may be overridden later by particular properties.
if parser.has_option('common', 'globalproperties'):
common_section['options']['properties'] = dict(
x.strip().split('=')
for x in parser.get('common', 'globalproperties').replace(
",\n", ",").split(','))
else:
# Always ensure properties has a valid dictionary so code below doesn't need multiple checks.
common_section['options']['properties'] = {}
# The same for global profiles
if parser.has_option('common', 'globalprofiles'):
common_section['options']['profiles'] = [
x.strip()
for x in parser.get('common', 'globalprofiles').split(',')
]
else:
# Always ensure profiles has a valid list so code below doesn't need multiple checks.
common_section['options']['profiles'] = []
if os.path.dirname(config_file):
config_path = os.path.dirname(config_file)
else:
config_path = os.getcwd()
logging.info("Configuration file is %s and path %s",
os.path.basename(config_file), config_path)
for section in parser.sections():
config_type = self.read_config_type(parser, section)
if section == 'common' or config_type == "bom-builder-meta":
logging.debug('Skipping section due to meta-type %s', section)
continue
self._do_read_section(config_path, os.path.basename(config_file),
package_configs, parser, section)
return (common_section, package_configs, pom_manipulator_config)
def sidey(name, engine, overwrite=False):
if not overwrite:
raise DuplicateSectionError('boom')
def _read(self, fp, fpname):
"""Parse a sectioned configuration file.
Each section in a configuration file contains a header, indicated by
a name in square brackets (`[]'), plus key/value options, indicated by
`name' and `value' delimited with a specific substring (`=' or `:' by
default).
Values can span multiple lines, as long as they are indented deeper
than the first line of the value. Depending on the parser's mode, blank
lines may be treated as parts of multiline values or ignored.
Configuration files may include comments, prefixed by specific
characters (`#' and `;' by default). Comments may appear on their own
in an otherwise empty line or may be entered in lines holding values or
section names.
This implementation is extended from the original to also accept
.. code:: ini
#include <file or pisa_resource>
or
.. code:: ini
#include <file or pisa_resource> as <section_name>
syntax anywhere in the file, which switches (via
:class:`MutableMultiFileIterator`) to the new file as if it were
in-lined within the original file. The latter syntax also prepends a
section header
.. code:: ini
[section_name]
before the text of the specified file or pisa_resource.
"""
elements_added = set()
cursect = None # None, or a dictionary
sectname = None
optname = None
lineno = 0
indent_level = 0
e = None # None, or an exception
file_iter = MutableMultiFileIterator(fp=fp, fpname=fpname)
self.file_iterators.append(file_iter)
for record in file_iter:
fpname = record['fpname']
lineno = record['lineno']
line = record['line']
comment_start = sys.maxsize
# strip inline comments
inline_prefixes = dict(
(p, -1) for p in self._inline_comment_prefixes)
while comment_start == sys.maxsize and inline_prefixes:
next_prefixes = {}
for prefix, index in inline_prefixes.items():
index = line.find(prefix, index + 1)
if index == -1:
continue
next_prefixes[prefix] = index
if index == 0 or (index > 0 and line[index - 1].isspace()):
comment_start = min(comment_start, index)
inline_prefixes = next_prefixes
# parse #include statement
include_info = self._get_include_info(line)
if include_info:
file_iter.switch_to_file(fpname=include_info['file'])
if include_info['as']:
as_header = '[%s]\n' % include_info['as']
file_iter.switch_to_file(
# Aaron Fienberg
# commented out as part of python3 update
# fp=StringIO(as_header.decode('utf-8'))
fp=StringIO(as_header))
continue
# strip full line comments
for prefix in self._comment_prefixes:
if line.strip().startswith(prefix):
comment_start = 0
break
if comment_start == sys.maxsize:
comment_start = None
value = line[:comment_start].strip()
if not value:
if self._empty_lines_in_values:
# add empty line to the value, but only if there was no
# comment on the line
if (comment_start is None and cursect is not None
and optname and cursect[optname] is not None):
cursect[optname].append('') # newlines added at join
else:
# empty line marks end of value
indent_level = sys.maxsize
continue
# continuation line?
first_nonspace = self.NONSPACECRE.search(line)
cur_indent_level = first_nonspace.start() if first_nonspace else 0
if (cursect is not None and optname
and cur_indent_level > indent_level):
cursect[optname].append(value)
# a section header or option header?
else:
indent_level = cur_indent_level
# is it a section header?
mo = self.SECTCRE.match(value)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
if self._strict and sectname in elements_added:
raise DuplicateSectionError(
sectname, fpname, lineno)
cursect = self._sections[sectname]
elements_added.add(sectname)
elif sectname == self.default_section:
cursect = self._defaults
else:
cursect = self._dict()
self._sections[sectname] = cursect
self._proxies[sectname] = SectionProxy(self, sectname)
elements_added.add(sectname)
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise MissingSectionHeaderError(fpname, lineno, line)
# an option line?
else:
mo = self._optcre.match(value)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value') # pylint: disable=unused-variable
if not optname:
e = self._handle_error(e, fpname, lineno, line)
optname = self.optionxform(optname.rstrip())
if (self._strict
and (sectname, optname) in elements_added):
raise DuplicateOptionError(sectname, optname,
fpname, lineno)
elements_added.add((sectname, optname))
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
optval = optval.strip()
cursect[optname] = [optval]
else:
# valueless option handling
cursect[optname] = None
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
e = self._handle_error(e, fpname, lineno, line)
# if any parsing errors occurred, raise an exception
if e:
raise e
self._join_multiline_values()
def _read(self, fp, fpname):
"""Parse a sectioned configuration file.
Each section in a configuration file contains a header, indicated by
a name in square brackets (`[]'), plus key/value options, indicated by
`name' and `value' delimited with a specific substring (`=' or `:' by
default).
Values can span multiple lines, as long as they are indented deeper
than the first line of the value. Depending on the parser's mode, blank
lines may be treated as parts of multiline values or ignored.
Configuration files may include comments, prefixed by specific
characters (`#' and `;' by default). Comments may appear on their own
in an otherwise empty line or may be entered in lines holding values or
section names.
"""
elements_added = set()
cursect = None # None, or a dictionary
sectname = None
optname = None
lineno = 0
indent_level = 0
e = None # None, or an exception
for lineno, line in enumerate(fp, start=1):
comment_start = None
# strip inline comments
for prefix in self._inline_comment_prefixes:
index = line.find(prefix)
if index == 0 or (index > 0 and line[index - 1].isspace()):
comment_start = index
break
# strip full line comments
for prefix in self._comment_prefixes:
if line.strip().startswith(prefix):
comment_start = 0
break
value = line[:comment_start].strip()
if not value:
if self._empty_lines_in_values:
# add empty line to the value, but only if there was no
# comment on the line
if (comment_start is None and cursect is not None
and optname and cursect[optname] is not None):
cursect[optname].append('') # newlines added at join
else:
# empty line marks end of value
indent_level = sys.maxsize
continue
# continuation line?
first_nonspace = self.NONSPACECRE.search(line)
cur_indent_level = first_nonspace.start() if first_nonspace else 0
if (cursect is not None and optname
and cur_indent_level > indent_level):
cursect[optname].append(value)
# a section header or option header?
else:
indent_level = cur_indent_level
# is it a section header?
mo = self.SECTCRE.match(value)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
if self._strict and sectname in elements_added:
raise DuplicateSectionError(
sectname, fpname, lineno)
cursect = self._sections[sectname]
elements_added.add(sectname)
elif sectname == self.default_section:
cursect = self._defaults
else:
cursect = self._dict()
self._sections[sectname] = cursect
self._proxies[sectname] = configparser.SectionProxy(
self, sectname)
elements_added.add(sectname)
# So sections can't start with a continuation line
optname = None
# no section header in the file?
elif cursect is None:
raise MissingSectionHeaderError(fpname, lineno, line)
# an option line?
else:
mo = self._optcre.match(value)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
if not optname:
e = self._handle_error(e, fpname, lineno, line)
optname = self.optionxform(optname.rstrip())
if (self._strict
and (sectname, optname) in elements_added):
raise configparser.DuplicateOptionError(
sectname, optname, fpname, lineno)
elements_added.add((sectname, optname))
# This check is fine because the OPTCRE cannot
# match if it would set optval to None
if optval is not None:
optval = optval.strip()
# Check if this optname already exists
if (optname in cursect) and (cursect[optname]
is not None):
# If it does, convert it to a tuple if it isn't already one
if not isinstance(cursect[optname], tuple):
cursect[optname] = tuple(cursect[optname])
cursect[optname] = cursect[optname] + tuple(
[optval])
else:
cursect[optname] = [optval]
else:
# valueless option handling
cursect[optname] = None
else:
# a non-fatal parsing error occurred. set up the
# exception but keep going. the exception will be
# raised at the end of the file and will contain a
# list of all bogus lines
e = self._handle_error(e, fpname, lineno, line)
# if any parsing errors occurred, raise an exception
if e:
raise e
self._join_multiline_values()
path = os.path.dirname(
os.path.realpath(__file__)) + "/../src/docstats/__main__.py"
exec(compile(open(path).read(), path, "exec"), {},
{"__name__": "__main__"})
def test_main_with_empty_args():
with pytest.raises(DocoptExit):
main([])
@pytest.mark.parametrize('error', [
KeyboardInterrupt,
FileNotFoundError,
OSError,
DuplicateSectionError('fake-sec'),
DuplicateOptionError('fake-sec', 'fake-opt'),
])
@patch('docstats.main.parsecli')
def test_main_with_ctrl_c(mock_parsecli, error):
mock_parsecli.side_effect = error
result = main([])
assert result
@patch('docstats.worker.work')
@patch('docstats.main.os.makedirs')
@patch('docstats.main.gettmpdir')
@patch('docstats.main.parseconfig')
@patch('docstats.main.parsecli')
def test_main_return_with_0(mock_parsecli, mock_parseconfig, mock_gettmpdir,
