def load(cls, bundle, **kwargs):
"""Load a model from a bundle.
This can be either a local model or a remote, exported model.
:returns a Service implementation
"""
# can delegate
if os.path.isdir(bundle):
directory = bundle
else:
directory = unzip_files(bundle)
model_basename = find_model_basename(directory)
vocabs = load_vocabs(directory)
vectorizers = load_vectorizers(directory)
be = normalize_backend(kwargs.get('backend', 'tf'))
remote = kwargs.get("remote", None)
name = kwargs.get("name", None)
if remote:
beam = kwargs.get('beam', 10)
model = Service._create_remote_model(directory, be, remote, name, cls.signature_name(), beam, preproc=kwargs.get('preproc', False))
return cls(vocabs, vectorizers, model)
# Currently nothing to do here
# labels = read_json(os.path.join(directory, model_basename) + '.labels')
import_user_module('baseline.{}.embeddings'.format(be))
import_user_module('baseline.{}.{}'.format(be, cls.task_name()))
model = load_model_for(cls.task_name(), model_basename, **kwargs)
return cls(vocabs, vectorizers, model)
def __init__(self, name=None, params=None, exporter=None):
"""Initialize the backend, optional with constructor args
:param name: (``str``) Name of the framework: currently one of (`tensorflow`, `pytorch`, `dynet`, `keras`)
:param params: (``dict``) A dictionary of framework-specific user-data to pass through keyword args to each sub-module
:param exporter: A framework-specific exporter to facilitate exporting to runtime deployment
"""
self.name = normalize_backend(name)
self.params = params
def __init__(self, name=None, params=None, exporter=None):
"""Initialize the backend, optional with constructor args
:param name: (``str``) Name of the framework: currently one of (`tensorflow`, `pytorch`, `dynet`, `keras`)
:param params: (``dict``) A dictionary of framework-specific user-data to pass through keyword args to each sub-module
:param exporter: A framework-specific exporter to facilitate exporting to runtime deployment
"""
self.name = normalize_backend(name)
self.params = params
def main():
parser = argparse.ArgumentParser(description='Create an Embeddings Service')
parser.add_argument('--config', help='JSON Configuration for an experiment', type=convert_path, default="$MEAD_CONFIG")
parser.add_argument('--settings', help='JSON Configuration for mead', default='config/mead-settings.json', type=convert_path)
parser.add_argument('--datasets', help='json library of dataset labels', default='config/datasets.json', type=convert_path)
parser.add_argument('--embeddings', help='json library of embeddings', default='config/embeddings.json', type=convert_path)
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--export', help='Should this create a export bundle?', default=True, type=str2bool)
parser.add_argument('--exporter_type', help="exporter type (default 'default')", default=None)
parser.add_argument('--model_version', help='model_version', default=None)
parser.add_argument('--output_dir', help="output dir (default './models')", default=None)
parser.add_argument('--project', help='Name of project, used in path first', default=None)
parser.add_argument('--name', help='Name of the model, used second in the path', default=None)
parser.add_argument('--is_remote', help='if True, separate items for remote server and client. If False bundle everything together (default True)', default=None)
args, reporting_args = parser.parse_known_args()
config_params = read_config_stream(args.config)
try:
args.settings = read_config_stream(args.settings)
except:
logger.warning('Warning: no mead-settings file was found at [{}]'.format(args.settings))
args.settings = {}
args.datasets = read_config_stream(args.datasets)
args.embeddings = read_config_stream(args.embeddings)
if args.backend is not None:
config_params['backend'] = normalize_backend(args.backend)
os.environ['CUDA_VISIBLE_DEVICES'] = ""
os.environ['NV_GPU'] = ""
if 'gpus' in config_params.get('train', {}):
del config_params['train']['gpus']
config_params['task'] = 'servable-embeddings'
task = mead.Task.get_task_specific(config_params['task'], args.settings)
task.read_config(config_params, args.datasets, reporting_args=[], config_file=deepcopy(config_params))
task.initialize(args.embeddings)
to_zip = False if args.export else True
task.train(None, zip_model=to_zip)
if args.export:
model = os.path.abspath(task.get_basedir())
output_dir, project, name, model_version, exporter_type, return_labels, is_remote = get_export_params(
config_params.get('export', {}),
args.output_dir,
args.project, args.name,
args.model_version,
args.exporter_type,
False,
args.is_remote,
)
feature_exporter_field_map = create_feature_exporter_field_map(config_params['features'])
exporter = create_exporter(task, exporter_type, return_labels=return_labels,
feature_exporter_field_map=feature_exporter_field_map)
exporter.run(model, output_dir, project, name, model_version, remote=is_remote)
def load(cls, bundle, **kwargs):
"""Load a model from a bundle.
This can be either a local model or a remote, exported model.
:returns a Service implementation
"""
# can delegate
basehead = None
if os.path.isdir(bundle):
directory = bundle
elif os.path.isfile(bundle):
directory = unzip_files(bundle)
else:
directory = os.path.dirname(bundle)
basehead = os.path.basename(bundle)
model_basename = find_model_basename(directory, basehead)
suffix = model_basename.split('-')[-1] + ".json"
vocabs = load_vocabs(directory, suffix)
be = normalize_backend(kwargs.get('backend', 'tf'))
remote = kwargs.get("remote", None)
name = kwargs.get("name", None)
if remote:
logging.debug("loading remote model")
beam = int(kwargs.get('beam', 30))
model, preproc = Service._create_remote_model(
directory,
be,
remote,
name,
cls.task_name(),
cls.signature_name(),
beam,
preproc=kwargs.get('preproc', 'client'),
version=kwargs.get('version'),
remote_type=kwargs.get('remote_type'),
)
vectorizers = load_vectorizers(directory)
return cls(vocabs, vectorizers, model, preproc)
# Currently nothing to do here
# labels = read_json(os.path.join(directory, model_basename) + '.labels')
import_user_module('baseline.{}.embeddings'.format(be))
try:
import_user_module('baseline.{}.{}'.format(be, cls.task_name()))
except:
pass
model = load_model_for(cls.task_name(), model_basename, **kwargs)
vectorizers = load_vectorizers(directory)
return cls(vocabs, vectorizers, model, 'client')
def main():
parser = argparse.ArgumentParser(description='Train a text classifier')
parser.add_argument('--config', help='JSON Configuration for an experiment', type=convert_path, default="$MEAD_CONFIG")
parser.add_argument('--settings', help='JSON Configuration for mead', default='config/mead-settings.json', type=convert_path)
parser.add_argument('--datasets', help='json library of dataset labels', default='config/datasets.json', type=convert_path)
parser.add_argument('--embeddings', help='json library of embeddings', default='config/embeddings.json', type=convert_path)
parser.add_argument('--logging', help='json file for logging', default='config/logging.json', type=convert_path)
parser.add_argument('--task', help='task to run', choices=['classify', 'tagger', 'seq2seq', 'lm'])
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--num_iters', type=int, default=5)
parser.add_argument('--max_lr', type=float, default=10)
parser.add_argument('--smooth', type=float, default=0.05)
parser.add_argument('--use_val', type=str2bool, default=False)
parser.add_argument('--log', type=str2bool, default=True)
parser.add_argument('--diverge_threshold', type=int, default=5)
args, reporting_args = parser.parse_known_args()
config_params = read_config_stream(args.config)
try:
args.settings = read_config_stream(args.settings)
except:
print('Warning: no mead-settings file was found at [{}]'.format(args.config))
args.settings = {}
args.datasets = read_config_stream(args.datasets)
args.embeddings = read_config_stream(args.embeddings)
args.logging = read_config_stream(args.logging)
if args.backend is not None:
config_params['backend'] = normalize_backend(args.backend)
config_params['reporting'] = {}
config_params['train']['fit_func'] = "lr-find"
config_params['train']['lr_scheduler_type'] = 'warmup_linear'
config_params['train']['smooth_beta'] = args.smooth
config_params['train']['use_val'] = args.use_val
config_params['train']['log_scale'] = args.log
config_params['train']['diverge_threshold'] = args.diverge_threshold
config_params['train']['be'] = config_params['backend']
task_name = config_params.get('task', 'classify') if args.task is None else args.task
print('Task: [{}]'.format(task_name))
task = mead.Task.get_task_specific(task_name, args.logging, args.settings)
task.read_config(config_params, args.datasets, reporting_args=reporting_args, config_file=deepcopy(config_params))
task.initialize(args.embeddings)
task.train()
def main():
parser = argparse.ArgumentParser(description='Train a text classifier')
parser.add_argument('--config', help='JSON Configuration for an experiment', type=convert_path, default="$MEAD_CONFIG")
parser.add_argument('--settings', help='JSON Configuration for mead', default='config/mead-settings.json', type=convert_path)
parser.add_argument('--datasets', help='json library of dataset labels', default='config/datasets.json', type=convert_path)
parser.add_argument('--embeddings', help='json library of embeddings', default='config/embeddings.json', type=convert_path)
parser.add_argument('--logging', help='json file for logging', default='config/logging.json', type=convert_path)
parser.add_argument('--task', help='task to run', choices=['classify', 'tagger', 'seq2seq', 'lm'])
parser.add_argument('--gpus', help='Number of GPUs (defaults to number available)', type=int, default=-1)
parser.add_argument('--basedir', help='Override the base directory where models are stored', type=str)
parser.add_argument('--reporting', help='reporting hooks', nargs='+')
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--checkpoint', help='Restart training from this checkpoint')
args, reporting_args = parser.parse_known_args()
args.logging = read_config_stream(args.logging)
configure_logger(args.logging)
config_params = read_config_stream(args.config)
try:
args.settings = read_config_stream(args.settings)
except:
logger.warning('Warning: no mead-settings file was found at [{}]'.format(args.settings))
args.settings = {}
args.datasets = read_config_stream(args.datasets)
args.embeddings = read_config_stream(args.embeddings)
if args.gpus is not None:
config_params['model']['gpus'] = args.gpus
if args.basedir is not None:
config_params['basedir'] = args.basedir
if args.backend is not None:
config_params['backend'] = normalize_backend(args.backend)
cmd_hooks = args.reporting if args.reporting is not None else []
config_hooks = config_params.get('reporting') if config_params.get('reporting') is not None else []
reporting = parse_extra_args(set(chain(cmd_hooks, config_hooks)), reporting_args)
config_params['reporting'] = reporting
task_name = config_params.get('task', 'classify') if args.task is None else args.task
logger.info('Task: [{}]'.format(task_name))
task = mead.Task.get_task_specific(task_name, args.settings)
task.read_config(config_params, args.datasets, reporting_args=reporting_args, config_file=deepcopy(config_params))
task.initialize(args.embeddings)
task.train(args.checkpoint)
def main():
parser = argparse.ArgumentParser(description='Train a text classifier')
parser.add_argument('--config', help='JSON Configuration for an experiment', type=convert_path, default="$MEAD_CONFIG")
parser.add_argument('--settings', help='JSON Configuration for mead', default='config/mead-settings.json', type=convert_path)
parser.add_argument('--datasets', help='json library of dataset labels', default='config/datasets.json', type=convert_path)
parser.add_argument('--embeddings', help='json library of embeddings', default='config/embeddings.json', type=convert_path)
parser.add_argument('--logging', help='json file for logging', default='config/logging.json', type=convert_path)
parser.add_argument('--task', help='task to run', choices=['classify', 'tagger', 'seq2seq', 'lm'])
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--num_iters', type=int, default=5)
parser.add_argument('--max_lr', type=float, default=10)
parser.add_argument('--smooth', type=float, default=0.05)
parser.add_argument('--use_val', type=str2bool, default=False)
parser.add_argument('--log', type=str2bool, default=True)
parser.add_argument('--diverge_threshold', type=int, default=5)
args, reporting_args = parser.parse_known_args()
config_params = read_config_stream(args.config)
try:
args.settings = read_config_stream(args.settings)
except:
print('Warning: no mead-settings file was found at [{}]'.format(args.config))
args.settings = {}
args.datasets = read_config_stream(args.datasets)
args.embeddings = read_config_stream(args.embeddings)
args.logging = read_config_stream(args.logging)
if args.backend is not None:
config_params['backend'] = normalize_backend(args.backend)
config_params['reporting'] = {}
config_params['train']['fit_func'] = "lr-find"
config_params['train']['lr_scheduler_type'] = 'warmup_linear'
config_params['train']['smooth_beta'] = args.smooth
config_params['train']['use_val'] = args.use_val
config_params['train']['log_scale'] = args.log
config_params['train']['diverge_threshold'] = args.diverge_threshold
config_params['train']['be'] = config_params['backend']
task_name = config_params.get('task', 'classify') if args.task is None else args.task
print('Task: [{}]'.format(task_name))
task = mead.Task.get_task_specific(task_name, args.logging, args.settings)
task.read_config(config_params, args.datasets, reporting_args=reporting_args, config_file=deepcopy(config_params))
task.initialize(args.embeddings)
task.train()
def load(cls, bundle, **kwargs):
"""Load a model from a bundle.
This can be either a local model or a remote, exported model.
:returns a Service implementation
"""
# can delegate
if os.path.isdir(bundle):
directory = bundle
else:
directory = unzip_files(bundle)
model_basename = find_model_basename(directory)
vocabs = load_vocabs(directory)
vectorizers = load_vectorizers(directory)
be = normalize_backend(kwargs.get('backend', 'tf'))
remote = kwargs.get("remote", None)
name = kwargs.get("name", None)
if remote:
logging.debug("loading remote model")
beam = kwargs.get('beam', 30)
model, preproc = Service._create_remote_model(
directory, be, remote, name, cls.signature_name(), beam,
preproc=kwargs.get('preproc', 'client'),
version=kwargs.get('version')
)
return cls(vocabs, vectorizers, model, preproc)
# Currently nothing to do here
# labels = read_json(os.path.join(directory, model_basename) + '.labels')
import_user_module('baseline.{}.embeddings'.format(be))
try:
import_user_module('baseline.{}.{}'.format(be, cls.task_name()))
except:
pass
model = load_model_for(cls.task_name(), model_basename, **kwargs)
return cls(vocabs, vectorizers, model, 'client')
def main():
parser = argparse.ArgumentParser(description='Train a text classifier')
parser.add_argument(
'--config',
help=
'JSON/YML Configuration for an experiment: local file or remote URL',
type=convert_path,
default="$MEAD_CONFIG")
parser.add_argument('--settings',
help='JSON/YML Configuration for mead',
default=DEFAULT_SETTINGS_LOC,
type=convert_path)
parser.add_argument('--task_modules',
help='tasks to load, must be local',
default=[],
nargs='+',
required=False)
parser.add_argument(
'--datasets',
help=
'index of dataset labels: local file, remote URL or mead-ml/hub ref',
type=convert_path)
parser.add_argument(
'--modules',
help='modules to load: local files, remote URLs or mead-ml/hub refs',
default=[],
nargs='+',
required=False)
parser.add_argument('--mod_train_file', help='override the training set')
parser.add_argument('--mod_valid_file', help='override the validation set')
parser.add_argument('--mod_test_file', help='override the test set')
parser.add_argument('--fit_func', help='override the fit function')
parser.add_argument(
'--embeddings',
help='index of embeddings: local file, remote URL or mead-ml/hub ref',
type=convert_path)
parser.add_argument(
'--vecs',
help='index of vectorizers: local file, remote URL or hub mead-ml/ref',
type=convert_path)
parser.add_argument('--logging',
help='json file for logging',
default=DEFAULT_LOGGING_LOC,
type=convert_path)
parser.add_argument('--task',
help='task to run',
choices=['classify', 'tagger', 'seq2seq', 'lm'])
parser.add_argument('--gpus',
help='Number of GPUs (defaults to number available)',
type=int,
default=-1)
parser.add_argument(
'--basedir',
help='Override the base directory where models are stored',
type=str)
parser.add_argument('--reporting', help='reporting hooks', nargs='+')
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--checkpoint',
help='Restart training from this checkpoint')
parser.add_argument(
'--prefer_eager',
help="If running in TensorFlow, should we prefer eager model",
type=str2bool)
args, overrides = parser.parse_known_args()
config_params = read_config_stream(args.config)
config_params = parse_and_merge_overrides(config_params,
overrides,
pre='x')
if args.basedir is not None:
config_params['basedir'] = args.basedir
# task_module overrides are not allowed via hub or HTTP, must be defined locally
for task in args.task_modules:
import_user_module(task)
task_name = config_params.get(
'task', 'classify') if args.task is None else args.task
args.logging = read_config_stream(args.logging)
configure_logger(args.logging,
config_params.get('basedir', './{}'.format(task_name)))
try:
args.settings = read_config_stream(args.settings)
except:
logger.warning(
'Warning: no mead-settings file was found at [{}]'.format(
args.settings))
args.settings = {}
args.datasets = args.settings.get(
'datasets', convert_path(
DEFAULT_DATASETS_LOC)) if args.datasets is None else args.datasets
args.datasets = read_config_stream(args.datasets)
if args.mod_train_file or args.mod_valid_file or args.mod_test_file:
logging.warning(
'Warning: overriding the training/valid/test data with user-specified files'
' different from what was specified in the dataset index. Creating a new key for this entry'
)
update_datasets(args.datasets, config_params, args.mod_train_file,
args.mod_valid_file, args.mod_test_file)
args.embeddings = args.settings.get(
'embeddings', convert_path(DEFAULT_EMBEDDINGS_LOC)
) if args.embeddings is None else args.embeddings
args.embeddings = read_config_stream(args.embeddings)
args.vecs = args.settings.get('vecs', convert_path(
DEFAULT_VECTORIZERS_LOC)) if args.vecs is None else args.vecs
args.vecs = read_config_stream(args.vecs)
if args.gpus:
# why does it go to model and not to train?
config_params['train']['gpus'] = args.gpus
if args.fit_func:
config_params['train']['fit_func'] = args.fit_func
if args.backend:
config_params['backend'] = normalize_backend(args.backend)
config_params['modules'] = list(
set(chain(config_params.get('modules', []), args.modules)))
cmd_hooks = args.reporting if args.reporting is not None else []
config_hooks = config_params.get('reporting') if config_params.get(
'reporting') is not None else []
reporting = parse_extra_args(set(chain(cmd_hooks, config_hooks)),
overrides)
config_params['reporting'] = reporting
logger.info('Task: [{}]'.format(task_name))
task = mead.Task.get_task_specific(task_name, args.settings)
task.read_config(config_params,
args.datasets,
args.vecs,
reporting_args=overrides,
prefer_eager=args.prefer_eager)
task.initialize(args.embeddings)
task.train(args.checkpoint)
def main():
parser = argparse.ArgumentParser(description='Train a text classifier')
parser.add_argument('--config',
help='configuration for an experiment',
type=convert_path,
default="$MEAD_CONFIG")
parser.add_argument('--settings',
help='configuration for mead',
default=DEFAULT_SETTINGS_LOC,
type=convert_path)
parser.add_argument('--datasets',
help='index of dataset labels',
type=convert_path)
parser.add_argument('--modules',
help='modules to load',
default=[],
nargs='+',
required=False)
parser.add_argument('--mod_train_file', help='override the training set')
parser.add_argument('--mod_valid_file', help='override the validation set')
parser.add_argument('--mod_test_file', help='override the test set')
parser.add_argument('--embeddings',
help='index of embeddings',
type=convert_path)
parser.add_argument('--logging',
help='config file for logging',
default=DEFAULT_LOGGING_LOC,
type=convert_path)
parser.add_argument('--task',
help='task to run',
choices=['classify', 'tagger', 'seq2seq', 'lm'])
parser.add_argument('--gpus',
help='Number of GPUs (defaults to number available)',
type=int,
default=-1)
parser.add_argument(
'--basedir',
help='Override the base directory where models are stored',
type=str)
parser.add_argument('--reporting', help='reporting hooks', nargs='+')
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--checkpoint',
help='Restart training from this checkpoint')
args, reporting_args = parser.parse_known_args()
config_params = read_config_stream(args.config)
if args.basedir is not None:
config_params['basedir'] = args.basedir
task_name = config_params.get(
'task', 'classify') if args.task is None else args.task
args.logging = read_config_stream(args.logging)
configure_logger(args.logging,
config_params.get('basedir', './{}'.format(task_name)))
try:
args.settings = read_config_stream(args.settings)
except:
logger.warning(
'Warning: no mead-settings file was found at [{}]'.format(
args.settings))
args.settings = {}
args.datasets = args.datasets if args.datasets else args.settings.get(
'datasets', convert_path(DEFAULT_DATASETS_LOC))
args.datasets = read_config_stream(args.datasets)
if args.mod_train_file or args.mod_valid_file or args.mod_test_file:
logging.warning(
'Warning: overriding the training/valid/test data with user-specified files'
' different from what was specified in the dataset index. Creating a new key for this entry'
)
update_datasets(args.datasets, config_params, args.mod_train_file,
args.mod_valid_file, args.mod_test_file)
args.embeddings = args.embeddings if args.embeddings else args.settings.get(
'embeddings', convert_path(DEFAULT_EMBEDDINGS_LOC))
args.embeddings = read_config_stream(args.embeddings)
if args.gpus is not None:
config_params['model']['gpus'] = args.gpus
if args.backend is None and 'backend' in args.settings:
args.backend = args.settings['backend']
if args.backend is not None:
config_params['backend'] = normalize_backend(args.backend)
config_params['modules'] = list(
set(chain(config_params.get('modules', []), args.modules)))
cmd_hooks = args.reporting if args.reporting is not None else []
config_hooks = config_params.get('reporting') if config_params.get(
'reporting') is not None else []
reporting = parse_extra_args(set(chain(cmd_hooks, config_hooks)),
reporting_args)
config_params['reporting'] = reporting
logger.info('Task: [{}]'.format(task_name))
task = mead.Task.get_task_specific(task_name, args.settings)
task.read_config(config_params,
args.datasets,
reporting_args=reporting_args)
task.initialize(args.embeddings)
task.train(args.checkpoint)
def main():
parser = argparse.ArgumentParser(description='Train a text classifier')
parser.add_argument('--config',
help='JSON Configuration for an experiment',
type=convert_path,
default="$MEAD_CONFIG")
parser.add_argument('--settings',
help='JSON Configuration for mead',
default='config/mead-settings.json',
type=convert_path)
parser.add_argument('--datasets',
help='json library of dataset labels',
default='config/datasets.json',
type=convert_path)
parser.add_argument('--embeddings',
help='json library of embeddings',
default='config/embeddings.json',
type=convert_path)
parser.add_argument('--logging',
help='json file for logging',
default='config/logging.json',
type=convert_path)
parser.add_argument('--task',
help='task to run',
choices=['classify', 'tagger', 'seq2seq', 'lm'])
parser.add_argument('--gpus',
help='Number of GPUs (defaults to number available)',
type=int,
default=-1)
parser.add_argument('--reporting', help='reporting hooks', nargs='+')
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--checkpoint',
help='Restart training from this checkpoint')
args, reporting_args = parser.parse_known_args()
args.logging = read_config_stream(args.logging)
configure_logger(args.logging)
config_params = read_config_stream(args.config)
try:
args.settings = read_config_stream(args.settings)
except:
logger.warning(
'Warning: no mead-settings file was found at [{}]'.format(
args.settings))
args.settings = {}
args.datasets = read_config_stream(args.datasets)
args.embeddings = read_config_stream(args.embeddings)
if args.gpus is not None:
config_params['model']['gpus'] = args.gpus
if args.backend is not None:
config_params['backend'] = normalize_backend(args.backend)
cmd_hooks = args.reporting if args.reporting is not None else []
config_hooks = config_params.get('reporting') if config_params.get(
'reporting') is not None else []
reporting = parse_extra_args(set(chain(cmd_hooks, config_hooks)),
reporting_args)
config_params['reporting'] = reporting
task_name = config_params.get(
'task', 'classify') if args.task is None else args.task
logger.info('Task: [{}]'.format(task_name))
task = mead.Task.get_task_specific(task_name, args.settings)
task.read_config(config_params,
args.datasets,
reporting_args=reporting_args,
config_file=deepcopy(config_params))
task.initialize(args.embeddings)
task.train(args.checkpoint)
def main():
parser = argparse.ArgumentParser(description='Export a model')
parser.add_argument('--config', help='configuration for an experiment', required=True, type=convert_path)
parser.add_argument('--settings', help='configuration for mead', required=False, default=DEFAULT_SETTINGS_LOC, type=convert_path)
parser.add_argument('--modules', help='modules to load', default=[], nargs='+', required=False)
parser.add_argument('--datasets', help='json library of dataset labels')
parser.add_argument('--vecs', help='index of vectorizers: local file, remote URL or hub mead-ml/ref', default='config/vecs.json', type=convert_path)
parser.add_argument('--logging', help='json file for logging', default='config/logging.json', type=convert_path)
parser.add_argument('--task', help='task to run', choices=['classify', 'tagger', 'seq2seq', 'lm'])
parser.add_argument('--exporter_type', help="exporter type (default 'default')", default=None)
parser.add_argument('--return_labels', help='if true, the exported model returns actual labels else '
'the indices for labels vocab (default False)', default=None)
parser.add_argument('--model', help='model name', required=True, type=unzip_files)
parser.add_argument('--model_version', help='model_version', default=None)
parser.add_argument('--output_dir', help="output dir (default './models')", default=None)
parser.add_argument('--project', help='Name of project, used in path first', default=None)
parser.add_argument('--name', help='Name of the model, used second in the path', default=None)
parser.add_argument('--beam', help='beam_width', default=30, type=int)
parser.add_argument('--nbest_input', help='Is the input to this model N-best', default=False, type=str2bool)
parser.add_argument('--is_remote', help='if True, separate items for remote server and client. If False bundle everything together (default True)', default=None)
parser.add_argument('--backend', help='The deep learning backend to use')
parser.add_argument('--reporting', help='reporting hooks', nargs='+')
parser.add_argument('--use_version', help='Should we use the version?', type=str2bool, default=True)
parser.add_argument('--use_all_features', help='If a feature is found via vectorizer and not in embeddings, should we include it?', type=str2bool, default=False)
parser.add_argument('--zip', help='Should we zip the results?', type=str2bool, default=False)
args, overrides = parser.parse_known_args()
configure_logger(args.logging)
config_params = read_config_stream(args.config)
config_params = parse_and_merge_overrides(config_params, overrides, pre='x')
try:
args.settings = read_config_stream(args.settings)
except Exception:
logger.warning('Warning: no mead-settings file was found at [{}]'.format(args.settings))
args.settings = {}
task_name = config_params.get('task', 'classify') if args.task is None else args.task
# Remove multigpu references
os.environ['CUDA_VISIBLE_DEVICES'] = ""
os.environ['NV_GPU'] = ""
if 'gpus' in config_params.get('train', {}):
del config_params['train']['gpus']
if task_name == 'seq2seq' and 'beam' not in config_params:
config_params['beam'] = args.beam
config_params['modules'] = config_params.get('modules', []) + args.modules
if args.backend is not None:
config_params['backend'] = normalize_backend(args.backend)
cmd_hooks = args.reporting if args.reporting is not None else []
config_hooks = config_params.get('reporting') if config_params.get('reporting') is not None else []
reporting = parse_extra_args(set(chain(cmd_hooks, config_hooks)), overrides)
config_params['reporting'] = reporting
args.vecs = read_config_stream(args.vecs)
task = mead.Task.get_task_specific(task_name, args.settings)
output_dir, project, name, model_version, exporter_type, return_labels, is_remote = get_export_params(
config_params.get('export', {}),
args.output_dir,
args.project, args.name,
args.model_version,
args.exporter_type,
args.return_labels,
args.is_remote,
)
# Here we reuse code in `.read_config` which needs a dataset index (when used with mead-train)
# but when used with mead-export it is not needed. This is a dummy dataset index that will work
# It means we don't need to pass it in
datasets = [{'label': config_params['dataset']}]
task.read_config(config_params, datasets, args.vecs, exporter_type=exporter_type)
feature_exporter_field_map = create_feature_exporter_field_map(config_params['features'])
exporter = create_exporter(task, exporter_type, return_labels=return_labels,
feature_exporter_field_map=feature_exporter_field_map,
nbest_input=args.nbest_input)
exporter.run(args.model, output_dir, project, name, model_version,
remote=is_remote, use_version=args.use_version, zip_results=args.zip, use_all_features=args.use_all_features)
