def _parse_arguments_by_section(parents,
section,
args_from_config_file,
args_from_cmd,
required_section):
"""
This function first adds parameter names to a parser,
according to the section name.
Then it loads values from configuration files as tentative params.
Finally it overrides existing pairs of 'name, value' with commandline
inputs.
Commandline inputs only override system/custom parameters.
input data related parameters needs to be defined in config file.
:param parents: a list, parsers will be created as
subparsers of parents
:param section: section name to be parsed
:param args_from_config_file: loaded parameters from config file
:param args_from_cmd: dictionary commandline parameters
:return: parsed parameters of the section and unknown
commandline params.
"""
section_parser = argparse.ArgumentParser(
parents=parents,
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
if section == 'SYSTEM':
section_parser = add_application_args(section_parser)
elif section == 'NETWORK':
section_parser = add_network_args(section_parser)
elif section == 'TRAINING':
section_parser = add_training_args(section_parser)
elif section == 'INFERENCE':
section_parser = add_inference_args(section_parser)
elif section == required_section:
section_parser = add_customised_args(section_parser, section.upper())
else:
section_parser = add_input_data_args(section_parser)
# loading all parameters a config file first
if args_from_config_file is not None:
section_parser.set_defaults(**args_from_config_file)
# input command line input overrides config file
if (section in SYSTEM_SECTIONS) or (section == required_section):
section_args, unknown = section_parser.parse_known_args(args_from_cmd)
return section_args, unknown
# don't parse user cmd for input source sections
section_args, _ = section_parser.parse_known_args([])
return section_args, args_from_cmd
def _parse_arguments_by_section(parents,
section,
args_from_config_file,
args_from_cmd,
required_section):
"""
This function first adds parameter names to a parser,
according to the section name.
Then it loads values from configuration files as tentative params.
Finally it overrides existing pairs of 'name, value' with commandline
inputs.
Commandline inputs only override system/custom parameters.
input data related parameters needs to be defined in config file.
:param parents: a list, parsers will be created as
subparsers of parents
:param section: section name to be parsed
:param args_from_config_file: loaded parameters from config file
:param args_from_cmd: dictionary commandline parameters
:return: parsed parameters of the section and unknown
commandline params.
"""
section_parser = argparse.ArgumentParser(
parents=parents,
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
try:
add_args_func = SUPPORTED_DEFAULT_SECTIONS[section]
except KeyError:
if section == required_section:
add_args_func = lambda parser: add_customised_args(
parser, section.upper())
else:
# all remaining sections are defaulting to input section
add_args_func = add_input_data_args
section_parser = add_args_func(section_parser)
# loading all parameters a config file first
if args_from_config_file is not None:
section_parser.set_defaults(**args_from_config_file)
# input command line input overrides config file
if (section in SYSTEM_SECTIONS) or (section == required_section):
section_args, unknown = section_parser.parse_known_args(args_from_cmd)
return section_args, unknown
# don't parse user cmd for input source sections
section_args, _ = section_parser.parse_known_args([])
return section_args, args_from_cmd
def _parse_arguments_by_section(parents,
section,
args_from_config_file,
args_from_cmd,
required_section):
"""
This function first adds parameter names to a parser,
according to the section name.
Then it loads values from configuration files as tentative params.
Finally it overrides existing pairs of 'name, value' with commandline
inputs.
Commandline inputs only override system/custom parameters.
input data related parameters needs to be defined in config file.
:param parents: a list, parsers will be created as
subparsers of parents
:param section: section name to be parsed
:param args_from_config_file: loaded parameters from config file
:param args_from_cmd: dictionary commandline parameters
:return: parsed parameters of the section and unknown
commandline params.
"""
section_parser = argparse.ArgumentParser(
parents=parents,
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
if section == 'SYSTEM':
section_parser = add_application_args(section_parser)
elif section == 'NETWORK':
section_parser = add_network_args(section_parser)
elif section == 'TRAINING':
section_parser = add_training_args(section_parser)
elif section == 'INFERENCE':
section_parser = add_inference_args(section_parser)
elif section == required_section:
section_parser = add_customised_args(section_parser, section.upper())
else:
section_parser = add_input_data_args(section_parser)
# loading all parameters a config file first
if args_from_config_file is not None:
section_parser.set_defaults(**args_from_config_file)
# input command line input overrides config file
if (section in SYSTEM_SECTIONS) or (section == required_section):
section_args, unknown = section_parser.parse_known_args(args_from_cmd)
return section_args, unknown
# don't parse user cmd for input source sections
section_args, _ = section_parser.parse_known_args([])
return section_args, args_from_cmd
def check_keywords(config):
"""
check config files, validate keywords provided against
parsers' argument list
"""
validation_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
config_keywords = []
for section in config.sections():
validation_parser = add_application_args(validation_parser)
validation_parser = add_network_args(validation_parser)
validation_parser = add_training_args(validation_parser)
validation_parser = add_inference_args(validation_parser)
validation_parser = add_input_data_args(validation_parser)
try:
validation_parser = add_customised_args(
validation_parser, section.upper())
except (argparse.ArgumentError, NotImplementedError):
pass
if config.items(section):
config_keywords.extend(list(dict(config.items(section))))
default_keywords = []
for action in validation_parser._actions:
try:
default_keywords.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
for config_key in config_keywords:
if config_key in default_keywords:
continue
dists = {k: edit_distance(k, config_key) for k in default_keywords}
closest = min(dists, key=dists.get)
if dists[closest] <= 5:
raise ValueError(
'Unknown keywords in config file: By "{0}" '
'did you mean "{1}"?\n "{0}" is '
'not a valid option.{2}'.format(
config_key, closest, epilog_string))
raise ValueError(
'Unknown keywords in config file: [{}] -- all '
' possible choices are {}.{}'.format(
config_key, default_keywords, epilog_string))
def check_keywords(config):
"""
check config files, validate keywords provided against
parsers' argument list
"""
validation_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
config_keywords = []
for section in config.sections():
validation_parser = add_application_args(validation_parser)
validation_parser = add_network_args(validation_parser)
validation_parser = add_training_args(validation_parser)
validation_parser = add_inference_args(validation_parser)
validation_parser = add_input_data_args(validation_parser)
try:
validation_parser = add_customised_args(
validation_parser, section.upper())
except (argparse.ArgumentError, NotImplementedError):
pass
if config.items(section):
config_keywords.extend(list(dict(config.items(section))))
default_keywords = []
for action in validation_parser._actions:
try:
default_keywords.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
for config_key in config_keywords:
if config_key in default_keywords:
continue
dists = {k: edit_distance(k, config_key)
for k in default_keywords}
closest = min(dists, key=dists.get)
if dists[closest] <= 5:
raise ValueError(
'Unknown keywords in config file: By "{0}" ' \
'did you mean "{1}"?\n "{0}" is ' \
'not a valid option. '.format(config_key, closest))
else:
raise ValueError(
'Unknown keywords in config file: [{}] -- all '
' possible choices are {}'.format(
config_key, default_keywords))
def _parse_arguments_by_section(parents, section, args_from_config_file,
args_from_cmd, required_section):
"""
This function first adds parameter names to a parser,
according to the section name.
Then it loads values from configuration files as tentative params.
Finally it overrides existing pairs of 'name, value' with commandline
inputs.
Commandline inputs only override system/custom parameters.
input data related parameters needs to be defined in config file.
:param parents: a list, parsers will be created as
subparsers of parents
:param section: section name to be parsed
:param args_from_config_file: loaded parameters from config file
:param args_from_cmd: dictionary commandline parameters
:return: parsed parameters of the section and unknown
commandline params.
"""
section_parser = argparse.ArgumentParser(
parents=parents,
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
try:
add_args_func = SUPPORTED_DEFAULT_SECTIONS[section]
except KeyError:
if section == required_section:
add_args_func = lambda parser: add_customised_args(
parser, section.upper())
else:
# all remaining sections are defaulting to input section
add_args_func = add_input_data_args
section_parser = add_args_func(section_parser)
# loading all parameters a config file first
if args_from_config_file is not None:
section_parser.set_defaults(**args_from_config_file)
# input command line input overrides config file
if (section in SYSTEM_SECTIONS) or (section == required_section):
section_args, unknown = section_parser.parse_known_args(args_from_cmd)
return section_args, unknown
# don't parse user cmd for input source sections
section_args, _ = section_parser.parse_known_args([])
return section_args, args_from_cmd
def add_args_func(parser):
"""
wrapper around add_customised_args
"""
return add_customised_args(parser, section.upper())
