def select_module(module_name, type_str, lookup_table=None):
"""
This function first tries to find the absolute module name
by matching the static dictionary items, if not found, it
tries to import the module by splitting the input ``module_name``
as module name and class name to be imported.
:param module_name: string that matches the keys defined in lookup_table
or an absolute class name: module.name.ClassName
:param type_str: type of the module (used for better error display)
:param lookup_table: defines a set of shorthands for absolute class name
"""
lookup_table = lookup_table or {}
module_name = '{}'.format(module_name)
is_external = True
if module_name in lookup_table:
module_name = lookup_table[module_name]
is_external = False
module_str, class_name = None, None
try:
module_str, class_name = module_name.rsplit('.', 1)
the_module = importlib.import_module(module_str)
the_class = getattr(the_module, class_name)
if is_external:
# print location of external module
tf.logging.info('Import [%s] from %s.', class_name,
os.path.abspath(the_module.__file__))
print(the_class)
return the_class
except (AttributeError, ValueError, ImportError) as not_imported:
tf.logging.fatal(repr(not_imported))
if '.' not in module_name:
err = 'Could not import {}: ' \
'Incorrect module name "{}"; ' \
'expected "module.object".'.format(type_str, module_name)
else:
err = '{}: Could not import object' \
'"{}" from "{}"'.format(type_str, class_name, module_str)
tf.logging.fatal(err)
if not lookup_table:
# no further guess
raise ValueError(err)
dists = dict(
(k, edit_distance(k, module_name)) for k in list(lookup_table))
closest = min(dists, key=dists.get)
if dists[closest] <= 3:
err = 'Could not import {2}: By "{0}", ' \
'did you mean "{1}"?\n "{0}" is ' \
'not a valid option. '.format(module_name, closest, type_str)
tf.logging.fatal(err)
raise ValueError(err)
def _raises_bad_keys(keys, error_info='config file'):
"""
raises value error if keys is not in the system key set.
`error_info` is used to customise the error message.
"""
for key in list(keys):
if key in KEYWORDS:
continue
dists = {k: edit_distance(k, key) for k in KEYWORDS}
closest = min(dists, key=dists.get)
raise ValueError('Unknown keywords in {3}: By "{0}" '
'did you mean "{1}"?\n "{0}" is '
'not a valid option.{2}'.format(
key, closest, EPILOG_STRING, error_info))
def select_module(module_name, type_str, lookup_table):
"""
This function first tries to find the absolute module name
by matching the static dictionary items, if not found, it
tries to import the module by splitting the input ``module_name``
as module name and class name to be imported.
:param module_name: string that matches the keys defined in lookup_table
or an absolute class name: module.name.ClassName
:param type_str: type of the module (used for better error display)
:param lookup_table: defines a set of shorthands for absolute class name
"""
module_name = '{}'.format(module_name)
if module_name in lookup_table:
module_name = lookup_table[module_name]
module_str, class_name = None, None
try:
module_str, class_name = module_name.rsplit('.', 1)
the_module = importlib.import_module(module_str)
the_class = getattr(the_module, class_name)
tf.logging.info('Import [%s] from %s.',
class_name, os.path.abspath(the_module.__file__))
return the_class
except (AttributeError, ValueError, ImportError) as not_imported:
# print sys.path
tf.logging.fatal(repr(not_imported))
# Two possibilities: a typo for a lookup table entry
# or a non-existing module
dists = dict((k, edit_distance(k, module_name))
for k in list(lookup_table))
closest = min(dists, key=dists.get)
if dists[closest] <= 3:
err = 'Could not import {2}: By "{0}", ' \
'did you mean "{1}"?\n "{0}" is ' \
'not a valid option. '.format(module_name, closest, type_str)
tf.logging.fatal(err)
raise ValueError(err)
else:
if '.' not in module_name:
err = 'Could not import {}: ' \
'Incorrect module name format {}. ' \
'Expected "module.object".'.format(type_str, module_name)
tf.logging.fatal(err)
raise ValueError(err)
err = '{}: Could not import object' \
'"{}" from "{}"'.format(type_str, class_name, module_str)
tf.logging.fatal(err)
raise ValueError(err)
def check_keywords(config):
"""
check config files, validate keywords provided against
parsers' argument list
"""
validation_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
config_keywords = []
for section in config.sections():
validation_parser = add_application_args(validation_parser)
validation_parser = add_network_args(validation_parser)
validation_parser = add_training_args(validation_parser)
validation_parser = add_inference_args(validation_parser)
validation_parser = add_input_data_args(validation_parser)
try:
validation_parser = add_customised_args(
validation_parser, section.upper())
except (argparse.ArgumentError, NotImplementedError):
pass
if config.items(section):
config_keywords.extend(list(dict(config.items(section))))
default_keywords = []
for action in validation_parser._actions:
try:
default_keywords.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
for config_key in config_keywords:
if config_key in default_keywords:
continue
dists = {k: edit_distance(k, config_key) for k in default_keywords}
closest = min(dists, key=dists.get)
if dists[closest] <= 5:
raise ValueError(
'Unknown keywords in config file: By "{0}" '
'did you mean "{1}"?\n "{0}" is '
'not a valid option.{2}'.format(
config_key, closest, epilog_string))
raise ValueError(
'Unknown keywords in config file: [{}] -- all '
' possible choices are {}.{}'.format(
config_key, default_keywords, epilog_string))
def check_keywords(config):
"""
check config files, validate keywords provided against
parsers' argument list
"""
validation_parser = argparse.ArgumentParser(
parents=[],
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
conflict_handler='resolve')
config_keywords = []
for section in config.sections():
validation_parser = add_application_args(validation_parser)
validation_parser = add_network_args(validation_parser)
validation_parser = add_training_args(validation_parser)
validation_parser = add_inference_args(validation_parser)
validation_parser = add_input_data_args(validation_parser)
try:
validation_parser = add_customised_args(
validation_parser, section.upper())
except (argparse.ArgumentError, NotImplementedError):
pass
if config.items(section):
config_keywords.extend(list(dict(config.items(section))))
default_keywords = []
for action in validation_parser._actions:
try:
default_keywords.append(action.option_strings[0][2:])
except (IndexError, AttributeError, ValueError):
pass
for config_key in config_keywords:
if config_key in default_keywords:
continue
dists = {k: edit_distance(k, config_key)
for k in default_keywords}
closest = min(dists, key=dists.get)
if dists[closest] <= 5:
raise ValueError(
'Unknown keywords in config file: By "{0}" ' \
'did you mean "{1}"?\n "{0}" is ' \
'not a valid option. '.format(config_key, closest))
else:
raise ValueError(
'Unknown keywords in config file: [{}] -- all '
' possible choices are {}'.format(
config_key, default_keywords))
def _raises_bad_keys(keys, error_info='config file'):
"""
raises value error if keys is not in the system key set.
`error_info` is used to customise the error message.
"""
for key in list(keys):
if key in KEYWORDS:
continue
dists = {k: edit_distance(k, key) for k in KEYWORDS}
closest = min(dists, key=dists.get)
raise ValueError(
'Unknown keywords in {3}: By "{0}" '
'did you mean "{1}"?\n "{0}" is '
'not a valid option.{2}'.format(
key, closest, EPILOG_STRING, error_info))
return
def select_module(module_name, type_str, lookup_table):
"""
This function first tries to find the absolute module name
by matching the static dictionary items, if not found, it
tries to import the module by splitting the input module_name
as module name and class name to be imported.
:param moduel_name: string that matches the keys defined in lookup_table
or an absolute class name: module.name.ClassName
:type_str: type of the module (currently used for better error display)
:lookup_table: defines a set of shorthands for absolute class name
"""
module_name = '{}'.format(module_name)
if module_name in lookup_table:
module_name = lookup_table[module_name]
module, class_name = None, None
try:
module, class_name = module_name.rsplit('.', 1)
the_module = getattr(importlib.import_module(module), class_name)
return the_module
except (AttributeError, ValueError, ImportError) as not_imported:
# print sys.path
tf.logging.fatal(repr(not_imported))
# Two possibilities: a typo for a lookup table entry
# or a non-existing module
dists = {k: edit_distance(k, module_name) for k in lookup_table.keys()}
closest = min(dists, key=dists.get)
if dists[closest] <= 3:
err = 'Could not import {2}: By "{0}", ' \
'did you mean "{1}"?\n "{0}" is ' \
'not a valid option. '.format(module_name, closest, type_str)
tf.logging.fatal(err)
raise ValueError(err)
else:
if '.' not in module_name:
err = 'Could not import {}: ' \
'Incorrect module name format {}. ' \
'Expected "module.object".'.format(type_str, module_name)
tf.logging.fatal(err)
raise ValueError(err)
err = '{}: Could not import object' \
'"{}" from "{}"'.format(type_str, class_name, module)
tf.logging.fatal(err)
raise ValueError(err)
