def available_keywords(): """ returns a list of all possible keywords defined in the parsers (duplicates from sections are removed.) """ all_key_parser = argparse.ArgumentParser( parents=[], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, conflict_handler='resolve') for _, add_args_func in SUPPORTED_DEFAULT_SECTIONS.items(): all_key_parser = add_args_func(all_key_parser) all_key_parser = add_input_data_args(all_key_parser) # add keys from custom sections for _, add_args_func in SUPPORTED_ARG_SECTIONS.items(): all_key_parser = add_args_func(all_key_parser) default_keys = [] for action in all_key_parser._actions: try: default_keys.append(action.option_strings[0][2:]) except (IndexError, AttributeError, ValueError): pass # remove duplicates default_keys = list(set(default_keys)) # remove bad names default_keys = [keyword for keyword in default_keys if keyword] return default_keys
def available_keywords(): """ returns a list of all possible keywords defined in the parsers (duplicates from sections are removed.) """ all_key_parser = argparse.ArgumentParser( parents=[], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, conflict_handler='resolve') for _, add_args_func in SUPPORTED_DEFAULT_SECTIONS.items(): all_key_parser = add_args_func(all_key_parser) all_key_parser = add_input_data_args(all_key_parser) # add keys from custom sections for _, add_args_func in SUPPORTED_ARG_SECTIONS.items(): all_key_parser = add_args_func(all_key_parser) default_keys = [] for action in all_key_parser._actions: try: default_keys.append(action.option_strings[0][2:]) except (IndexError, AttributeError, ValueError): pass # remove duplicates default_keys = list(set(default_keys)) # remove bad names default_keys = [keyword for keyword in default_keys if keyword] return default_keys
def _parse_arguments_by_section(parents, section, args_from_config_file, args_from_cmd, required_section): """ This function first adds parameter names to a parser, according to the section name. Then it loads values from configuration files as tentative params. Finally it overrides existing pairs of 'name, value' with commandline inputs. Commandline inputs only override system/custom parameters. input data related parameters needs to be defined in config file. :param parents: a list, parsers will be created as subparsers of parents :param section: section name to be parsed :param args_from_config_file: loaded parameters from config file :param args_from_cmd: dictionary commandline parameters :return: parsed parameters of the section and unknown commandline params. """ section_parser = argparse.ArgumentParser( parents=parents, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) if section == 'SYSTEM': section_parser = add_application_args(section_parser) elif section == 'NETWORK': section_parser = add_network_args(section_parser) elif section == 'TRAINING': section_parser = add_training_args(section_parser) elif section == 'INFERENCE': section_parser = add_inference_args(section_parser) elif section == required_section: section_parser = add_customised_args(section_parser, section.upper()) else: section_parser = add_input_data_args(section_parser) # loading all parameters a config file first if args_from_config_file is not None: section_parser.set_defaults(**args_from_config_file) # input command line input overrides config file if (section in SYSTEM_SECTIONS) or (section == required_section): section_args, unknown = section_parser.parse_known_args(args_from_cmd) return section_args, unknown # don't parse user cmd for input source sections section_args, _ = section_parser.parse_known_args([]) return section_args, args_from_cmd
def _parse_arguments_by_section(parents, section, args_from_config_file, args_from_cmd, required_section): """ This function first adds parameter names to a parser, according to the section name. Then it loads values from configuration files as tentative params. Finally it overrides existing pairs of 'name, value' with commandline inputs. Commandline inputs only override system/custom parameters. input data related parameters needs to be defined in config file. :param parents: a list, parsers will be created as subparsers of parents :param section: section name to be parsed :param args_from_config_file: loaded parameters from config file :param args_from_cmd: dictionary commandline parameters :return: parsed parameters of the section and unknown commandline params. """ section_parser = argparse.ArgumentParser( parents=parents, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) if section == 'SYSTEM': section_parser = add_application_args(section_parser) elif section == 'NETWORK': section_parser = add_network_args(section_parser) elif section == 'TRAINING': section_parser = add_training_args(section_parser) elif section == 'INFERENCE': section_parser = add_inference_args(section_parser) elif section == required_section: section_parser = add_customised_args(section_parser, section.upper()) else: section_parser = add_input_data_args(section_parser) # loading all parameters a config file first if args_from_config_file is not None: section_parser.set_defaults(**args_from_config_file) # input command line input overrides config file if (section in SYSTEM_SECTIONS) or (section == required_section): section_args, unknown = section_parser.parse_known_args(args_from_cmd) return section_args, unknown # don't parse user cmd for input source sections section_args, _ = section_parser.parse_known_args([]) return section_args, args_from_cmd
def check_keywords(config): """ check config files, validate keywords provided against parsers' argument list """ validation_parser = argparse.ArgumentParser( parents=[], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, conflict_handler='resolve') config_keywords = [] for section in config.sections(): validation_parser = add_application_args(validation_parser) validation_parser = add_network_args(validation_parser) validation_parser = add_training_args(validation_parser) validation_parser = add_inference_args(validation_parser) validation_parser = add_input_data_args(validation_parser) try: validation_parser = add_customised_args( validation_parser, section.upper()) except (argparse.ArgumentError, NotImplementedError): pass if config.items(section): config_keywords.extend(list(dict(config.items(section)))) default_keywords = [] for action in validation_parser._actions: try: default_keywords.append(action.option_strings[0][2:]) except (IndexError, AttributeError, ValueError): pass for config_key in config_keywords: if config_key in default_keywords: continue dists = {k: edit_distance(k, config_key) for k in default_keywords} closest = min(dists, key=dists.get) if dists[closest] <= 5: raise ValueError( 'Unknown keywords in config file: By "{0}" ' 'did you mean "{1}"?\n "{0}" is ' 'not a valid option.{2}'.format( config_key, closest, epilog_string)) raise ValueError( 'Unknown keywords in config file: [{}] -- all ' ' possible choices are {}.{}'.format( config_key, default_keywords, epilog_string))
def check_keywords(config): """ check config files, validate keywords provided against parsers' argument list """ validation_parser = argparse.ArgumentParser( parents=[], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, conflict_handler='resolve') config_keywords = [] for section in config.sections(): validation_parser = add_application_args(validation_parser) validation_parser = add_network_args(validation_parser) validation_parser = add_training_args(validation_parser) validation_parser = add_inference_args(validation_parser) validation_parser = add_input_data_args(validation_parser) try: validation_parser = add_customised_args( validation_parser, section.upper()) except (argparse.ArgumentError, NotImplementedError): pass if config.items(section): config_keywords.extend(list(dict(config.items(section)))) default_keywords = [] for action in validation_parser._actions: try: default_keywords.append(action.option_strings[0][2:]) except (IndexError, AttributeError, ValueError): pass for config_key in config_keywords: if config_key in default_keywords: continue dists = {k: edit_distance(k, config_key) for k in default_keywords} closest = min(dists, key=dists.get) if dists[closest] <= 5: raise ValueError( 'Unknown keywords in config file: By "{0}" ' \ 'did you mean "{1}"?\n "{0}" is ' \ 'not a valid option. '.format(config_key, closest)) else: raise ValueError( 'Unknown keywords in config file: [{}] -- all ' ' possible choices are {}'.format( config_key, default_keywords))