def select_module(module_name, type_str, lookup_table=None): """ This function first tries to find the absolute module name by matching the static dictionary items, if not found, it tries to import the module by splitting the input ``module_name`` as module name and class name to be imported. :param module_name: string that matches the keys defined in lookup_table or an absolute class name: module.name.ClassName :param type_str: type of the module (used for better error display) :param lookup_table: defines a set of shorthands for absolute class name """ lookup_table = lookup_table or {} module_name = '{}'.format(module_name) is_external = True if module_name in lookup_table: module_name = lookup_table[module_name] is_external = False module_str, class_name = None, None try: module_str, class_name = module_name.rsplit('.', 1) the_module = importlib.import_module(module_str) the_class = getattr(the_module, class_name) if is_external: # print location of external module tf.logging.info('Import [%s] from %s.', class_name, os.path.abspath(the_module.__file__)) print(the_class) return the_class except (AttributeError, ValueError, ImportError) as not_imported: tf.logging.fatal(repr(not_imported)) if '.' not in module_name: err = 'Could not import {}: ' \ 'Incorrect module name "{}"; ' \ 'expected "module.object".'.format(type_str, module_name) else: err = '{}: Could not import object' \ '"{}" from "{}"'.format(type_str, class_name, module_str) tf.logging.fatal(err) if not lookup_table: # no further guess raise ValueError(err) dists = dict( (k, edit_distance(k, module_name)) for k in list(lookup_table)) closest = min(dists, key=dists.get) if dists[closest] <= 3: err = 'Could not import {2}: By "{0}", ' \ 'did you mean "{1}"?\n "{0}" is ' \ 'not a valid option. '.format(module_name, closest, type_str) tf.logging.fatal(err) raise ValueError(err)
def _raises_bad_keys(keys, error_info='config file'): """ raises value error if keys is not in the system key set. `error_info` is used to customise the error message. """ for key in list(keys): if key in KEYWORDS: continue dists = {k: edit_distance(k, key) for k in KEYWORDS} closest = min(dists, key=dists.get) raise ValueError('Unknown keywords in {3}: By "{0}" ' 'did you mean "{1}"?\n "{0}" is ' 'not a valid option.{2}'.format( key, closest, EPILOG_STRING, error_info))
def select_module(module_name, type_str, lookup_table): """ This function first tries to find the absolute module name by matching the static dictionary items, if not found, it tries to import the module by splitting the input ``module_name`` as module name and class name to be imported. :param module_name: string that matches the keys defined in lookup_table or an absolute class name: module.name.ClassName :param type_str: type of the module (used for better error display) :param lookup_table: defines a set of shorthands for absolute class name """ module_name = '{}'.format(module_name) if module_name in lookup_table: module_name = lookup_table[module_name] module_str, class_name = None, None try: module_str, class_name = module_name.rsplit('.', 1) the_module = importlib.import_module(module_str) the_class = getattr(the_module, class_name) tf.logging.info('Import [%s] from %s.', class_name, os.path.abspath(the_module.__file__)) return the_class except (AttributeError, ValueError, ImportError) as not_imported: # print sys.path tf.logging.fatal(repr(not_imported)) # Two possibilities: a typo for a lookup table entry # or a non-existing module dists = dict((k, edit_distance(k, module_name)) for k in list(lookup_table)) closest = min(dists, key=dists.get) if dists[closest] <= 3: err = 'Could not import {2}: By "{0}", ' \ 'did you mean "{1}"?\n "{0}" is ' \ 'not a valid option. '.format(module_name, closest, type_str) tf.logging.fatal(err) raise ValueError(err) else: if '.' not in module_name: err = 'Could not import {}: ' \ 'Incorrect module name format {}. ' \ 'Expected "module.object".'.format(type_str, module_name) tf.logging.fatal(err) raise ValueError(err) err = '{}: Could not import object' \ '"{}" from "{}"'.format(type_str, class_name, module_str) tf.logging.fatal(err) raise ValueError(err)
def check_keywords(config): """ check config files, validate keywords provided against parsers' argument list """ validation_parser = argparse.ArgumentParser( parents=[], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, conflict_handler='resolve') config_keywords = [] for section in config.sections(): validation_parser = add_application_args(validation_parser) validation_parser = add_network_args(validation_parser) validation_parser = add_training_args(validation_parser) validation_parser = add_inference_args(validation_parser) validation_parser = add_input_data_args(validation_parser) try: validation_parser = add_customised_args( validation_parser, section.upper()) except (argparse.ArgumentError, NotImplementedError): pass if config.items(section): config_keywords.extend(list(dict(config.items(section)))) default_keywords = [] for action in validation_parser._actions: try: default_keywords.append(action.option_strings[0][2:]) except (IndexError, AttributeError, ValueError): pass for config_key in config_keywords: if config_key in default_keywords: continue dists = {k: edit_distance(k, config_key) for k in default_keywords} closest = min(dists, key=dists.get) if dists[closest] <= 5: raise ValueError( 'Unknown keywords in config file: By "{0}" ' 'did you mean "{1}"?\n "{0}" is ' 'not a valid option.{2}'.format( config_key, closest, epilog_string)) raise ValueError( 'Unknown keywords in config file: [{}] -- all ' ' possible choices are {}.{}'.format( config_key, default_keywords, epilog_string))
def check_keywords(config): """ check config files, validate keywords provided against parsers' argument list """ validation_parser = argparse.ArgumentParser( parents=[], description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, conflict_handler='resolve') config_keywords = [] for section in config.sections(): validation_parser = add_application_args(validation_parser) validation_parser = add_network_args(validation_parser) validation_parser = add_training_args(validation_parser) validation_parser = add_inference_args(validation_parser) validation_parser = add_input_data_args(validation_parser) try: validation_parser = add_customised_args( validation_parser, section.upper()) except (argparse.ArgumentError, NotImplementedError): pass if config.items(section): config_keywords.extend(list(dict(config.items(section)))) default_keywords = [] for action in validation_parser._actions: try: default_keywords.append(action.option_strings[0][2:]) except (IndexError, AttributeError, ValueError): pass for config_key in config_keywords: if config_key in default_keywords: continue dists = {k: edit_distance(k, config_key) for k in default_keywords} closest = min(dists, key=dists.get) if dists[closest] <= 5: raise ValueError( 'Unknown keywords in config file: By "{0}" ' \ 'did you mean "{1}"?\n "{0}" is ' \ 'not a valid option. '.format(config_key, closest)) else: raise ValueError( 'Unknown keywords in config file: [{}] -- all ' ' possible choices are {}'.format( config_key, default_keywords))
def _raises_bad_keys(keys, error_info='config file'): """ raises value error if keys is not in the system key set. `error_info` is used to customise the error message. """ for key in list(keys): if key in KEYWORDS: continue dists = {k: edit_distance(k, key) for k in KEYWORDS} closest = min(dists, key=dists.get) raise ValueError( 'Unknown keywords in {3}: By "{0}" ' 'did you mean "{1}"?\n "{0}" is ' 'not a valid option.{2}'.format( key, closest, EPILOG_STRING, error_info)) return
def select_module(module_name, type_str, lookup_table): """ This function first tries to find the absolute module name by matching the static dictionary items, if not found, it tries to import the module by splitting the input module_name as module name and class name to be imported. :param moduel_name: string that matches the keys defined in lookup_table or an absolute class name: module.name.ClassName :type_str: type of the module (currently used for better error display) :lookup_table: defines a set of shorthands for absolute class name """ module_name = '{}'.format(module_name) if module_name in lookup_table: module_name = lookup_table[module_name] module, class_name = None, None try: module, class_name = module_name.rsplit('.', 1) the_module = getattr(importlib.import_module(module), class_name) return the_module except (AttributeError, ValueError, ImportError) as not_imported: # print sys.path tf.logging.fatal(repr(not_imported)) # Two possibilities: a typo for a lookup table entry # or a non-existing module dists = {k: edit_distance(k, module_name) for k in lookup_table.keys()} closest = min(dists, key=dists.get) if dists[closest] <= 3: err = 'Could not import {2}: By "{0}", ' \ 'did you mean "{1}"?\n "{0}" is ' \ 'not a valid option. '.format(module_name, closest, type_str) tf.logging.fatal(err) raise ValueError(err) else: if '.' not in module_name: err = 'Could not import {}: ' \ 'Incorrect module name format {}. ' \ 'Expected "module.object".'.format(type_str, module_name) tf.logging.fatal(err) raise ValueError(err) err = '{}: Could not import object' \ '"{}" from "{}"'.format(type_str, class_name, module) tf.logging.fatal(err) raise ValueError(err)