def available_keywords():
"""
returns a list of all possible keywords defined in the parsers
(duplicates from sections are removed.)
"""
all_key_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
for _, add_args_func in SUPPORTED_DEFAULT_SECTIONS.items():
all_key_parser = add_args_func(all_key_parser)
all_key_parser = add_input_data_args(all_key_parser)
# add keys from custom sections
for _, add_args_func in SUPPORTED_ARG_SECTIONS.items():
all_key_parser = add_args_func(all_key_parser)
default_keys = []
for action in all_key_parser._actions:
try:
default_keys.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
# remove duplicates
default_keys = list(set(default_keys))
# remove bad names
default_keys = [keyword for keyword in default_keys if keyword]
return default_keys
def available_keywords():
"""
returns a list of all possible keywords defined in the parsers
(duplicates from sections are removed.)
"""
all_key_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
for _, add_args_func in SUPPORTED_DEFAULT_SECTIONS.items():
all_key_parser = add_args_func(all_key_parser)
all_key_parser = add_input_data_args(all_key_parser)
# add keys from custom sections
for _, add_args_func in SUPPORTED_ARG_SECTIONS.items():
all_key_parser = add_args_func(all_key_parser)
default_keys = []
for action in all_key_parser._actions:
try:
default_keys.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
# remove duplicates
default_keys = list(set(default_keys))
# remove bad names
default_keys = [keyword for keyword in default_keys if keyword]
return default_keys
def _parse_arguments_by_section(parents,
section,
args_from_config_file,
args_from_cmd,
required_section):
"""
This function first adds parameter names to a parser,
according to the section name.
Then it loads values from configuration files as tentative params.
Finally it overrides existing pairs of 'name, value' with commandline
inputs.
Commandline inputs only override system/custom parameters.
input data related parameters needs to be defined in config file.
:param parents: a list, parsers will be created as
subparsers of parents
:param section: section name to be parsed
:param args_from_config_file: loaded parameters from config file
:param args_from_cmd: dictionary commandline parameters
:return: parsed parameters of the section and unknown
commandline params.
"""
section_parser = argparse.ArgumentParser(
parents=parents,
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
if section == 'SYSTEM':
section_parser = add_application_args(section_parser)
elif section == 'NETWORK':
section_parser = add_network_args(section_parser)
elif section == 'TRAINING':
section_parser = add_training_args(section_parser)
elif section == 'INFERENCE':
section_parser = add_inference_args(section_parser)
elif section == required_section:
section_parser = add_customised_args(section_parser, section.upper())
else:
section_parser = add_input_data_args(section_parser)
# loading all parameters a config file first
if args_from_config_file is not None:
section_parser.set_defaults(**args_from_config_file)
# input command line input overrides config file
if (section in SYSTEM_SECTIONS) or (section == required_section):
section_args, unknown = section_parser.parse_known_args(args_from_cmd)
return section_args, unknown
# don't parse user cmd for input source sections
section_args, _ = section_parser.parse_known_args([])
return section_args, args_from_cmd
def _parse_arguments_by_section(parents,
section,
args_from_config_file,
args_from_cmd,
required_section):
"""
This function first adds parameter names to a parser,
according to the section name.
Then it loads values from configuration files as tentative params.
Finally it overrides existing pairs of 'name, value' with commandline
inputs.
Commandline inputs only override system/custom parameters.
input data related parameters needs to be defined in config file.
:param parents: a list, parsers will be created as
subparsers of parents
:param section: section name to be parsed
:param args_from_config_file: loaded parameters from config file
:param args_from_cmd: dictionary commandline parameters
:return: parsed parameters of the section and unknown
commandline params.
"""
section_parser = argparse.ArgumentParser(
parents=parents,
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
if section == 'SYSTEM':
section_parser = add_application_args(section_parser)
elif section == 'NETWORK':
section_parser = add_network_args(section_parser)
elif section == 'TRAINING':
section_parser = add_training_args(section_parser)
elif section == 'INFERENCE':
section_parser = add_inference_args(section_parser)
elif section == required_section:
section_parser = add_customised_args(section_parser, section.upper())
else:
section_parser = add_input_data_args(section_parser)
# loading all parameters a config file first
if args_from_config_file is not None:
section_parser.set_defaults(**args_from_config_file)
# input command line input overrides config file
if (section in SYSTEM_SECTIONS) or (section == required_section):
section_args, unknown = section_parser.parse_known_args(args_from_cmd)
return section_args, unknown
# don't parse user cmd for input source sections
section_args, _ = section_parser.parse_known_args([])
return section_args, args_from_cmd
def check_keywords(config):
"""
check config files, validate keywords provided against
parsers' argument list
"""
validation_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
config_keywords = []
for section in config.sections():
validation_parser = add_application_args(validation_parser)
validation_parser = add_network_args(validation_parser)
validation_parser = add_training_args(validation_parser)
validation_parser = add_inference_args(validation_parser)
validation_parser = add_input_data_args(validation_parser)
try:
validation_parser = add_customised_args(
validation_parser, section.upper())
except (argparse.ArgumentError, NotImplementedError):
pass
if config.items(section):
config_keywords.extend(list(dict(config.items(section))))
default_keywords = []
for action in validation_parser._actions:
try:
default_keywords.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
for config_key in config_keywords:
if config_key in default_keywords:
continue
dists = {k: edit_distance(k, config_key) for k in default_keywords}
closest = min(dists, key=dists.get)
if dists[closest] <= 5:
raise ValueError(
'Unknown keywords in config file: By "{0}" '
'did you mean "{1}"?\n "{0}" is '
'not a valid option.{2}'.format(
config_key, closest, epilog_string))
raise ValueError(
'Unknown keywords in config file: [{}] -- all '
' possible choices are {}.{}'.format(
config_key, default_keywords, epilog_string))
def check_keywords(config):
"""
check config files, validate keywords provided against
parsers' argument list
"""
validation_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
config_keywords = []
for section in config.sections():
validation_parser = add_application_args(validation_parser)
validation_parser = add_network_args(validation_parser)
validation_parser = add_training_args(validation_parser)
validation_parser = add_inference_args(validation_parser)
validation_parser = add_input_data_args(validation_parser)
try:
validation_parser = add_customised_args(
validation_parser, section.upper())
except (argparse.ArgumentError, NotImplementedError):
pass
if config.items(section):
config_keywords.extend(list(dict(config.items(section))))
default_keywords = []
for action in validation_parser._actions:
try:
default_keywords.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
for config_key in config_keywords:
if config_key in default_keywords:
continue
dists = {k: edit_distance(k, config_key)
for k in default_keywords}
closest = min(dists, key=dists.get)
if dists[closest] <= 5:
raise ValueError(
'Unknown keywords in config file: By "{0}" ' \
'did you mean "{1}"?\n "{0}" is ' \
'not a valid option. '.format(config_key, closest))
else:
raise ValueError(
'Unknown keywords in config file: [{}] -- all '
' possible choices are {}'.format(
config_key, default_keywords))