def full_experiment(model_definition,
model_definition_file=None,
data_df=None,
data_train_df=None,
data_validation_df=None,
data_test_df=None,
data_csv=None,
data_train_csv=None,
data_validation_csv=None,
data_test_csv=None,
data_hdf5=None,
data_train_hdf5=None,
data_validation_hdf5=None,
data_test_hdf5=None,
train_set_metadata_json=None,
experiment_name='experiment',
model_name='run',
model_load_path=None,
model_resume_path=None,
skip_save_training_description=False,
skip_save_training_statistics=False,
skip_save_model=False,
skip_save_progress=False,
skip_save_log=False,
skip_save_processed_input=False,
skip_save_unprocessed_output=False,
skip_save_test_predictions=False,
skip_save_test_statistics=False,
output_directory='results',
gpus=None,
gpu_memory_limit=None,
allow_parallel_threads=True,
use_horovod=None,
random_seed=default_random_seed,
debug=False,
**kwargs):
"""Trains a model on a dataset's training and validation splits and
uses it to predict on the test split.
It saves the trained model and the statistics of training and testing.
:param model_definition: Model definition which defines the different
parameters of the model, features, preprocessing and training.
:type model_definition: Dictionary
:param model_definition_file: The file that specifies the model definition.
It is a yaml file.
:type model_definition_file: filepath (str)
:param data_csv: A CSV file containing the input data which is used to
train, validate and test a model. The CSV either contains a
split column or will be split.
:type data_csv: filepath (str)
:param data_train_csv: A CSV file containing the input data which is used
to train a model.
:type data_train_csv: filepath (str)
:param data_validation_csv: A CSV file containing the input data which is used
to validate a model..
:type data_validation_csv: filepath (str)
:param data_test_csv: A CSV file containing the input data which is used
to test a model.
:type data_test_csv: filepath (str)
:param data_hdf5: If the dataset is in the hdf5 format, this is used instead
of the csv file.
:type data_hdf5: filepath (str)
:param data_train_hdf5: If the training set is in the hdf5 format, this is
used instead of the csv file.
:type data_train_hdf5: filepath (str)
:param data_validation_hdf5: If the validation set is in the hdf5 format,
this is used instead of the csv file.
:type data_validation_hdf5: filepath (str)
:param data_test_hdf5: If the test set is in the hdf5 format, this is
used instead of the csv file.
:type data_test_hdf5: filepath (str)
:param train_set_metadata_json: If the dataset is in hdf5 format, this is
the associated json file containing metadata.
:type train_set_metadata_json: filepath (str)
:param experiment_name: The name for the experiment.
:type experiment_name: Str
:param model_name: Name of the model that is being used.
:type model_name: Str
:param model_load_path: If this is specified the loaded model will be used
as initialization (useful for transfer learning).
:type model_load_path: filepath (str)
:param model_resume_path: Resumes training of the model from the path
specified. The difference with model_load_path is that also training
statistics like the current epoch and the loss and performance so
far are also resumed effectively continuing a previously interrupted
training process.
:type model_resume_path: filepath (str)
:param skip_save_training_description: Disables saving
the description JSON file.
:type skip_save_training_description: Boolean
:param skip_save_training_statistics: Disables saving
training statistics JSON file.
:type skip_save_training_statistics: Boolean
:param skip_save_model: Disables
saving model weights and hyperparameters each time the model
improves. By default Ludwig saves model weights after each epoch
the validation metric improves, but if the model is really big
that can be time consuming if you do not want to keep
the weights and just find out what performance can a model get
with a set of hyperparameters, use this parameter to skip it,
but the model will not be loadable later on.
:type skip_save_model: Boolean
:param skip_save_progress: Disables saving
progress each epoch. By default Ludwig saves weights and stats
after each epoch for enabling resuming of training, but if
the model is really big that can be time consuming and will uses
twice as much space, use this parameter to skip it, but training
cannot be resumed later on.
:type skip_save_progress: Boolean
:param skip_save_log: Disables saving TensorBoard
logs. By default Ludwig saves logs for the TensorBoard, but if it
is not needed turning it off can slightly increase the
overall speed..
:type skip_save_log: Boolean
:param skip_save_processed_input: If a CSV dataset is provided it is
preprocessed and then saved as an hdf5 and json to avoid running
the preprocessing again. If this parameter is False,
the hdf5 and json file are not saved.
:type skip_save_processed_input: Boolean
:param skip_save_unprocessed_output: By default predictions and
their probabilities are saved in both raw unprocessed numpy files
containing tensors and as postprocessed CSV files
(one for each output feature). If this parameter is True,
only the CSV ones are saved and the numpy ones are skipped.
:type skip_save_unprocessed_output: Boolean
:param skip_save_test_predictions: skips saving test predictions CSV files
:type skip_save_test_predictions: Boolean
:param skip_save_test_statistics: skips saving test statistics JSON file
:type skip_save_test_statistics: Boolean
:param output_directory: The directory that will contain the training
statistics, the saved model and the training progress files.
:type output_directory: filepath (str)
:param gpus: List of GPUs that are available for training.
:type gpus: List
:param gpu_memory_limit: maximum memory in MB to allocate per GPU device.
:type gpu_memory_limit: Integer
:param allow_parallel_threads: allow TensorFlow to use multithreading parallelism
to improve performance at the cost of determinism.
:type allow_parallel_threads: Boolean
:param use_horovod: Flag for using horovod
:type use_horovod: Boolean
:param random_seed: Random seed used for weights initialization,
splits and any other random function.
:type random_seed: Integer
:param debug: If true turns on tfdbg with inf_or_nan checks.
:type debug: Boolean
"""
set_on_master(use_horovod)
(
model,
preprocessed_data,
experiment_dir_name,
_, # train_stats
model_definition,
test_results) = experiment(
model_definition,
model_definition_file=model_definition_file,
data_df=data_df,
data_train_df=data_train_df,
data_validation_df=data_validation_df,
data_test_df=data_test_df,
data_csv=data_csv,
data_train_csv=data_train_csv,
data_validation_csv=data_validation_csv,
data_test_csv=data_test_csv,
data_hdf5=data_hdf5,
data_train_hdf5=data_train_hdf5,
data_validation_hdf5=data_validation_hdf5,
data_test_hdf5=data_test_hdf5,
train_set_metadata_json=train_set_metadata_json,
experiment_name=experiment_name,
model_name=model_name,
model_load_path=model_load_path,
model_resume_path=model_resume_path,
skip_save_training_description=skip_save_training_description,
skip_save_training_statistics=skip_save_training_statistics,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
skip_save_log=skip_save_log,
skip_save_processed_input=skip_save_processed_input,
output_directory=output_directory,
gpus=gpus,
gpu_memory_limit=gpu_memory_limit,
allow_parallel_threads=allow_parallel_threads,
use_horovod=use_horovod,
random_seed=random_seed,
debug=debug,
**kwargs)
(training_set, validation_set, test_set,
train_set_metadata) = preprocessed_data
if test_set is not None:
# check if we need to create the output dir
if is_on_master():
if not (skip_save_unprocessed_output and skip_save_test_predictions
and skip_save_test_statistics):
if not os.path.exists(experiment_dir_name):
os.makedirs(experiment_dir_name)
# postprocess
postprocessed_output = postprocess(
test_results, model_definition['output_features'],
train_set_metadata, experiment_dir_name,
skip_save_unprocessed_output or not is_on_master())
if is_on_master():
print_test_results(test_results)
if not skip_save_test_predictions:
save_prediction_outputs(postprocessed_output,
experiment_dir_name)
if not skip_save_test_statistics:
save_test_statistics(test_results, experiment_dir_name)
if is_on_master():
logger.info('\nFinished: {0}_{1}'.format(experiment_name, model_name))
logger.info('Saved to: {}'.format(experiment_dir_name))
contrib_command("experiment_save", experiment_dir_name)
return experiment_dir_name
def train_and_eval_on_split(
model_definition,
eval_split=VALIDATION,
data_df=None,
data_train_df=None,
data_validation_df=None,
data_test_df=None,
data_csv=None,
data_train_csv=None,
data_validation_csv=None,
data_test_csv=None,
data_hdf5=None,
data_train_hdf5=None,
data_validation_hdf5=None,
data_test_hdf5=None,
train_set_metadata_json=None,
experiment_name="hyperopt",
model_name="run",
# model_load_path=None,
# model_resume_path=None,
skip_save_training_description=False,
skip_save_training_statistics=False,
skip_save_model=False,
skip_save_progress=False,
skip_save_log=False,
skip_save_processed_input=False,
skip_save_unprocessed_output=False,
skip_save_test_predictions=False,
skip_save_test_statistics=False,
output_directory="results",
gpus=None,
gpu_memory_limit=None,
allow_parallel_threads=True,
use_horovod=False,
random_seed=default_random_seed,
debug=False,
**kwargs
):
# Collect training and validation losses and metrics
# & append it to `results`
# ludwig_model = LudwigModel(modified_model_definition)
(model, preprocessed_data, experiment_dir_name, train_stats,
model_definition) = full_train(
model_definition=model_definition,
data_df=data_df,
data_train_df=data_train_df,
data_validation_df=data_validation_df,
data_test_df=data_test_df,
data_csv=data_csv,
data_train_csv=data_train_csv,
data_validation_csv=data_validation_csv,
data_test_csv=data_test_csv,
data_hdf5=data_hdf5,
data_train_hdf5=data_train_hdf5,
data_validation_hdf5=data_validation_hdf5,
data_test_hdf5=data_test_hdf5,
train_set_metadata_json=train_set_metadata_json,
experiment_name=experiment_name,
model_name=model_name,
# model_load_path=model_load_path,
# model_resume_path=model_resume_path,
skip_save_training_description=skip_save_training_description,
skip_save_training_statistics=skip_save_training_statistics,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
skip_save_log=skip_save_log,
skip_save_processed_input=skip_save_processed_input,
output_directory=output_directory,
gpus=gpus,
gpu_memory_limit=gpu_memory_limit,
allow_parallel_threads=allow_parallel_threads,
use_horovod=use_horovod,
random_seed=random_seed,
debug=debug,
)
(training_set, validation_set, test_set,
train_set_metadata) = preprocessed_data
if model_definition[TRAINING]["eval_batch_size"] > 0:
batch_size = model_definition[TRAINING]["eval_batch_size"]
else:
batch_size = model_definition[TRAINING]["batch_size"]
eval_set = validation_set
if eval_split == TRAINING:
eval_set = training_set
elif eval_split == VALIDATION:
eval_set = validation_set
elif eval_split == TEST:
eval_set = test_set
test_results = predict(
eval_set,
train_set_metadata,
model,
model_definition,
batch_size,
evaluate_performance=True,
debug=debug
)
if not (
skip_save_unprocessed_output and skip_save_test_predictions and skip_save_test_statistics):
if not os.path.exists(experiment_dir_name):
os.makedirs(experiment_dir_name)
# postprocess
postprocessed_output = postprocess(
test_results,
model_definition["output_features"],
train_set_metadata,
experiment_dir_name,
skip_save_unprocessed_output,
)
print_test_results(test_results)
if not skip_save_test_predictions:
save_prediction_outputs(postprocessed_output, experiment_dir_name)
if not skip_save_test_statistics:
save_test_statistics(test_results, experiment_dir_name)
return train_stats, test_results