def kfold_cross_validate_cli(k_fold, model_definition=None, model_definition_file=None, data_csv=None, output_directory='results', random_seed=default_random_seed, skip_save_k_fold_split_indices=False, **kwargs): """Wrapper function to performs k-fold cross validation. # Inputs :param k_fold: (int) number of folds to create for the cross-validation :param model_definition: (dict, default: None) a dictionary containing information needed to build a model. Refer to the [User Guide] (http://ludwig.ai/user_guide/#model-definition) for details. :param model_definition_file: (string, optional, default: `None`) path to a YAML file containing the model definition. If available it will be used instead of the model_definition dict. :param data_csv: (string, default: None) :param output_directory: (string, default: 'results') :param random_seed: (int) Random seed used k-fold splits. :param skip_save_k_fold_split_indices: (boolean, default: False) Disables saving k-fold split indices :return: None """ model_definition = check_which_model_definition(model_definition, model_definition_file) (kfold_cv_stats, kfold_split_indices) = kfold_cross_validate( k_fold, model_definition=model_definition, data_csv=data_csv, output_directory=output_directory, random_seed=random_seed) # save k-fold cv statistics save_json(os.path.join(output_directory, 'kfold_training_statistics.json'), kfold_cv_stats) # save k-fold split indices if not skip_save_k_fold_split_indices: save_json(os.path.join(output_directory, 'kfold_split_indices.json'), kfold_split_indices)
def experiment_cli(model_definition, model_definition_file=None, dataset=None, training_set=None, validation_set=None, test_set=None, data_format=None, training_set_metadata=None, experiment_name='experiment', model_name='run', model_load_path=None, model_resume_path=None, skip_save_training_description=False, skip_save_training_statistics=False, skip_save_model=False, skip_save_progress=False, skip_save_log=False, skip_save_processed_input=False, skip_save_unprocessed_output=False, skip_save_predictions=False, skip_save_eval_stats=False, skip_collect_predictions=False, skip_collect_overall_stats=False, output_directory='results', gpus=None, gpu_memory_limit=None, allow_parallel_threads=True, use_horovod=None, random_seed=default_random_seed, debug=False, logging_level=logging.INFO, **kwargs): """Trains a model on a dataset's training and validation splits and uses it to predict on the test split. It saves the trained model and the statistics of training and testing. :param model_definition: Model definition which defines the different parameters of the model, features, preprocessing and training. :type model_definition: Dictionary :param model_definition_file: The file that specifies the model definition. It is a yaml file. :type model_definition_file: filepath (str) :param dataset: Source containing the entire dataset. If it has a split column, it will be used for splitting (0: train, 1: validation, 2: test), otherwise the dataset will be randomly split. :type dataset: Str, Dictionary, DataFrame :param training_set: Source containing training data. :type training_set: Str, Dictionary, DataFrame :param validation_set: Source containing validation data. :type validation_set: Str, Dictionary, DataFrame :param test_set: Source containing test data. :type test_set: Str, Dictionary, DataFrame :param training_set_metadata: Metadata JSON file or loaded metadata. Intermediate preprocess structure containing the mappings of the input CSV created the first time a CSV file is used in the same directory with the same name and a '.json' extension. :type training_set_metadata: Str, Dictionary :param data_format: Format to interpret data sources. Will be inferred automatically if not specified. :type data_format: Str :param experiment_name: The name for the experiment. :type experiment_name: Str :param model_name: Name of the model that is being used. :type model_name: Str :param model_load_path: If this is specified the loaded model will be used as initialization (useful for transfer learning). :type model_load_path: filepath (str) :param model_resume_path: Resumes training of the model from the path specified. The difference with model_load_path is that also training statistics like the current epoch and the loss and performance so far are also resumed effectively continuing a previously interrupted training process. :type model_resume_path: filepath (str) :param skip_save_training_description: Disables saving the description JSON file. :type skip_save_training_description: Boolean :param skip_save_training_statistics: Disables saving training statistics JSON file. :type skip_save_training_statistics: Boolean :param skip_save_model: Disables saving model weights and hyperparameters each time the model improves. By default Ludwig saves model weights after each epoch the validation metric improves, but if the model is really big that can be time consuming if you do not want to keep the weights and just find out what performance can a model get with a set of hyperparameters, use this parameter to skip it, but the model will not be loadable later on. :type skip_save_model: Boolean :param skip_save_progress: Disables saving progress each epoch. By default Ludwig saves weights and stats after each epoch for enabling resuming of training, but if the model is really big that can be time consuming and will uses twice as much space, use this parameter to skip it, but training cannot be resumed later on. :type skip_save_progress: Boolean :param skip_save_log: Disables saving TensorBoard logs. By default Ludwig saves logs for the TensorBoard, but if it is not needed turning it off can slightly increase the overall speed.. :type skip_save_log: Boolean :param skip_save_processed_input: If a CSV dataset is provided it is preprocessed and then saved as an hdf5 and json to avoid running the preprocessing again. If this parameter is False, the hdf5 and json file are not saved. :type skip_save_processed_input: Boolean :param skip_save_unprocessed_output: By default predictions and their probabilities are saved in both raw unprocessed numpy files containing tensors and as postprocessed CSV files (one for each output feature). If this parameter is True, only the CSV ones are saved and the numpy ones are skipped. :type skip_save_unprocessed_output: Boolean :param skip_save_predictions: skips saving test predictions CSV files :type skip_save_predictions: Boolean :param skip_save_eval_stats: skips saving test statistics JSON file :type skip_save_eval_stats: Boolean :param skip_collect_predictions: skips collecting post-processed predictions during eval. :type skip_collect_predictions: Boolean :param skip_collect_overall_stats: skips collecting overall stats during eval. :type skip_collect_overall_stats: Boolean :param output_directory: The directory that will contain the training statistics, the saved model and the training progress files. :type output_directory: filepath (str) :param gpus: List of GPUs that are available for training. :type gpus: List :param gpu_memory_limit: maximum memory in MB to allocate per GPU device. :type gpu_memory_limit: Integer :param allow_parallel_threads: allow TensorFlow to use multithreading parallelism to improve performance at the cost of determinism. :type allow_parallel_threads: Boolean :param use_horovod: Flag for using horovod :type use_horovod: Boolean :param random_seed: Random seed used for weights initialization, splits and any other random function. :type random_seed: Integer :param debug: If true turns on tfdbg with inf_or_nan checks. :type debug: Boolean :param logging_level: Log level to send to stderr. :type logging_level: int """ set_on_master(use_horovod) model_definition = check_which_model_definition(model_definition, model_definition_file) if model_load_path: model = LudwigModel.load(model_load_path) else: model = LudwigModel( model_definition=model_definition, logging_level=logging_level, use_horovod=use_horovod, gpus=gpus, gpu_memory_limit=gpu_memory_limit, allow_parallel_threads=allow_parallel_threads, ) (test_results, train_stats, preprocessed_data, output_directory) = model.experiment( dataset=dataset, training_set=training_set, validation_set=validation_set, test_set=test_set, training_set_metadata=training_set_metadata, data_format=data_format, experiment_name=experiment_name, model_name=model_name, model_resume_path=model_resume_path, skip_save_training_description=skip_save_training_description, skip_save_training_statistics=skip_save_training_statistics, skip_save_model=skip_save_model, skip_save_progress=skip_save_progress, skip_save_log=skip_save_log, skip_save_processed_input=skip_save_processed_input, skip_save_unprocessed_output=skip_save_unprocessed_output, skip_save_predictions=skip_save_predictions, skip_save_eval_stats=skip_save_eval_stats, skip_collect_predictions=skip_collect_predictions, skip_collect_overall_stats=skip_collect_overall_stats, output_directory=output_directory, random_seed=random_seed, debug=debug, ) return model, test_results, train_stats, preprocessed_data, output_directory
def hyperopt_cli( model_definition=None, model_definition_file=None, dataset=None, training_set=None, validation_set=None, test_set=None, training_set_metadata=None, data_format=None, experiment_name="hyperopt", model_name="run", # model_load_path=None, # model_resume_path=None, skip_save_training_description=True, skip_save_training_statistics=True, skip_save_model=True, skip_save_progress=True, skip_save_log=True, skip_save_processed_input=True, skip_save_unprocessed_output=True, skip_save_predictions=True, skip_save_eval_stats=True, skip_save_hyperopt_statistics=False, output_directory="results", gpus=None, gpu_memory_limit=None, allow_parallel_threads=True, use_horovod=None, random_seed=default_random_seed, debug=False, **kwargs, ): model_definition = check_which_model_definition(model_definition, model_definition_file) return hyperopt( model_definition=model_definition, dataset=dataset, training_set=training_set, validation_set=validation_set, test_set=test_set, training_set_metadata=training_set_metadata, data_format=data_format, experiment_name=experiment_name, model_name=model_name, # model_load_path=model_load_path, # model_resume_path=model_resume_path, skip_save_training_description=skip_save_training_description, skip_save_training_statistics=skip_save_training_statistics, skip_save_model=skip_save_model, skip_save_progress=skip_save_progress, skip_save_log=skip_save_log, skip_save_processed_input=skip_save_processed_input, skip_save_unprocessed_output=skip_save_unprocessed_output, skip_save_predictions=skip_save_predictions, skip_save_eval_stats=skip_save_eval_stats, skip_save_hyperopt_statistics=skip_save_hyperopt_statistics, output_directory=output_directory, gpus=gpus, gpu_memory_limit=gpu_memory_limit, allow_parallel_threads=allow_parallel_threads, use_horovod=use_horovod, random_seed=random_seed, debug=debug, **kwargs, )
def hyperopt_cli( model_definition: dict, model_definition_file: str = None, dataset: str = None, training_set: str = None, validation_set: str = None, test_set: str = None, training_set_metadata: str = None, data_format: str = None, experiment_name: str = 'experiment', model_name: str = 'run', # model_load_path=None, # model_resume_path=None, skip_save_training_description: bool = False, skip_save_training_statistics: bool = False, skip_save_model: bool = False, skip_save_progress: bool = False, skip_save_log: bool = False, skip_save_processed_input: bool = False, skip_save_unprocessed_output: bool = False, skip_save_predictions: bool = False, skip_save_eval_stats: bool = False, skip_save_hyperopt_statistics: bool = False, output_directory: str = 'results', gpus: Union[str, int, List[int]] = None, gpu_memory_limit: int = None, allow_parallel_threads: bool = True, use_horovod: bool = None, random_seed: int = default_random_seed, debug: bool = False, **kwargs, ): """ Searches for optimal hyperparameters. # Inputs :param model_definition: (dict) model definition which defines the different parameters of the model, features, preprocessing and training. :param model_definition_file: (str, default: `None`) the filepath string that specifies the model definition. It is a yaml file. :param dataset: (Union[str, dict, pandas.DataFrame], default: `None`) source containing the entire dataset to be used for training. If it has a split column, it will be used for splitting (0 for train, 1 for validation, 2 for test), otherwise the dataset will be randomly split. :param training_set: (Union[str, dict, pandas.DataFrame], default: `None`) source containing training data. :param validation_set: (Union[str, dict, pandas.DataFrame], default: `None`) source containing validation data. :param test_set: (Union[str, dict, pandas.DataFrame], default: `None`) source containing test data. :param training_set_metadata: (Union[str, dict], default: `None`) metadata JSON file or loaded metadata. Intermediate preprocess structure containing the mappings of the input dataset created the first time an input file is used in the same directory with the same name and a '.meta.json' extension. :param data_format: (str, default: `None`) format to interpret data sources. Will be inferred automatically if not specified. Valid formats are `'auto'`, `'csv'`, `'excel'`, `'feather'`, `'fwf'`, `'hdf5'` (cache file produced during previous training), `'html'` (file containing a single HTML `<table>`), `'json'`, `'jsonl'`, `'parquet'`, `'pickle'` (pickled Pandas DataFrame), `'sas'`, `'spss'`, `'stata'`, `'tsv'`. :param experiment_name: (str, default: `'experiment'`) name for the experiment. :param model_name: (str, default: `'run'`) name of the model that is being used. :param skip_save_training_description: (bool, default: `False`) disables saving the description JSON file. :param skip_save_training_statistics: (bool, default: `False`) disables saving training statistics JSON file. :param skip_save_model: (bool, default: `False`) disables saving model weights and hyperparameters each time the model improves. By default Ludwig saves model weights after each epoch the validation metric improves, but if the model is really big that can be time consuming if you do not want to keep the weights and just find out what performance can a model get with a set of hyperparameters, use this parameter to skip it, but the model will not be loadable later on and the returned model will have the weights obtained at the end of training, instead of the weights of the epoch with the best validation performance. :param skip_save_progress: (bool, default: `False`) disables saving progress each epoch. By default Ludwig saves weights and stats after each epoch for enabling resuming of training, but if the model is really big that can be time consuming and will uses twice as much space, use this parameter to skip it, but training cannot be resumed later on. :param skip_save_log: (bool, default: `False`) disables saving TensorBoard logs. By default Ludwig saves logs for the TensorBoard, but if it is not needed turning it off can slightly increase the overall speed. :param skip_save_processed_input: (bool, default: `False`) if input dataset is provided it is preprocessed and cached by saving an HDF5 and JSON files to avoid running the preprocessing again. If this parameter is `False`, the HDF5 and JSON file are not saved. :param skip_save_unprocessed_output: (bool, default: `False`) by default predictions and their probabilities are saved in both raw unprocessed numpy files containing tensors and as postprocessed CSV files (one for each output feature). If this parameter is True, only the CSV ones are saved and the numpy ones are skipped. :param skip_save_predictions: (bool, default: `False`) skips saving test predictions CSV files :param skip_save_eval_stats: (bool, default: `False`) skips saving test statistics JSON file :param skip_save_hyperopt_statistics: (bool, default: `False`) skips saving hyperopt stats file. :param output_directory: (str, default: `'results'`) the directory that will contain the training statistics, TensorBoard logs, the saved model and the training progress files. :param gpus: (list, default: `None`) list of GPUs that are available for training. :param gpu_memory_limit: (int, default: `None`) maximum memory in MB to allocate per GPU device. :param allow_parallel_threads: (bool, default: `True`) allow TensorFlow to use multithreading parallelism to improve performance at the cost of determinism. :param use_horovod: (bool, default: `None`) flag for using horovod. :param random_seed: (int: default: 42) random seed used for weights initialization, splits and any other random function. :param debug: (bool, default: `False) if `True` turns on `tfdbg` with `inf_or_nan` checks. **kwargs: # Return :return" (`None`) """ model_definition = check_which_model_definition(model_definition, model_definition_file) return hyperopt( model_definition=model_definition, dataset=dataset, training_set=training_set, validation_set=validation_set, test_set=test_set, training_set_metadata=training_set_metadata, data_format=data_format, experiment_name=experiment_name, model_name=model_name, # model_load_path=model_load_path, # model_resume_path=model_resume_path, skip_save_training_description=skip_save_training_description, skip_save_training_statistics=skip_save_training_statistics, skip_save_model=skip_save_model, skip_save_progress=skip_save_progress, skip_save_log=skip_save_log, skip_save_processed_input=skip_save_processed_input, skip_save_unprocessed_output=skip_save_unprocessed_output, skip_save_predictions=skip_save_predictions, skip_save_eval_stats=skip_save_eval_stats, skip_save_hyperopt_statistics=skip_save_hyperopt_statistics, output_directory=output_directory, gpus=gpus, gpu_memory_limit=gpu_memory_limit, allow_parallel_threads=allow_parallel_threads, use_horovod=use_horovod, random_seed=random_seed, debug=debug, **kwargs, )
def train_cli( model_definition=None, model_definition_file=None, dataset=None, training_set=None, validation_set=None, test_set=None, training_set_metadata=None, data_format=None, experiment_name='experiment', model_name='run', model_load_path=None, model_resume_path=None, skip_save_training_description=False, skip_save_training_statistics=False, skip_save_model=False, skip_save_progress=False, skip_save_log=False, skip_save_processed_input=False, output_directory='results', gpus=None, gpu_memory_limit=None, allow_parallel_threads=True, use_horovod=None, random_seed=default_random_seed, logging_level=logging.INFO, debug=False, **kwargs ): """*full_train* defines the entire training procedure used by Ludwig's internals. Requires most of the parameters that are taken into the model. Builds a full ludwig model and performs the training. :param data_test_df: :param data_df: :param data_train_df: :param data_validation_df: :param model_definition: Model definition which defines the different parameters of the model, features, preprocessing and training. :type model_definition: Dictionary :param model_definition_file: The file that specifies the model definition. It is a yaml file. :type model_definition_file: filepath (str) :param data_csv: A CSV file containing the input data which is used to train, validate and test a model. The CSV either contains a split column or will be split. :type data_csv: filepath (str) :param data_train_csv: A CSV file containing the input data which is used to train a model. :type data_train_csv: filepath (str) :param data_validation_csv: A CSV file containing the input data which is used to validate a model.. :type data_validation_csv: filepath (str) :param data_test_csv: A CSV file containing the input data which is used to test a model. :type data_test_csv: filepath (str) :param data_hdf5: If the dataset is in the hdf5 format, this is used instead of the csv file. :type data_hdf5: filepath (str) :param data_train_hdf5: If the training set is in the hdf5 format, this is used instead of the csv file. :type data_train_hdf5: filepath (str) :param data_validation_hdf5: If the validation set is in the hdf5 format, this is used instead of the csv file. :type data_validation_hdf5: filepath (str) :param data_test_hdf5: If the test set is in the hdf5 format, this is used instead of the csv file. :type data_test_hdf5: filepath (str) :param training_set_metadata_json: If the dataset is in hdf5 format, this is the associated json file containing metadata. :type training_set_metadata_json: filepath (str) :param experiment_name: The name for the experiment. :type experiment_name: Str :param model_name: Name of the model that is being used. :type model_name: Str :param model_load_path: If this is specified the loaded model will be used as initialization (useful for transfer learning). :type model_load_path: filepath (str) :param model_resume_path: Resumes training of the model from the path specified. The difference with model_load_path is that also training statistics like the current epoch and the loss and performance so far are also resumed effectively cotinuing a previously interrupted training process. :type model_resume_path: filepath (str) :param skip_save_training_description: Disables saving the description JSON file. :type skip_save_training_description: Boolean :param skip_save_training_statistics: Disables saving training statistics JSON file. :type skip_save_training_statistics: Boolean :param skip_save_model: Disables saving model weights and hyperparameters each time the model improves. By default Ludwig saves model weights after each epoch the validation metric improves, but if the model is really big that can be time consuming if you do not want to keep the weights and just find out what performance can a model get with a set of hyperparameters, use this parameter to skip it, but the model will not be loadable later on. :type skip_save_model: Boolean :param skip_save_progress: Disables saving progress each epoch. By default Ludwig saves weights and stats after each epoch for enabling resuming of training, but if the model is really big that can be time consuming and will uses twice as much space, use this parameter to skip it, but training cannot be resumed later on. :type skip_save_progress: Boolean :param skip_save_processed_input: If a CSV dataset is provided it is preprocessed and then saved as an hdf5 and json to avoid running the preprocessing again. If this parameter is False, the hdf5 and json file are not saved. :type skip_save_processed_input: Boolean :param skip_save_log: Disables saving TensorBoard logs. By default Ludwig saves logs for the TensorBoard, but if it is not needed turning it off can slightly increase the overall speed.. :type skip_save_progress: Boolean :param output_directory: The directory that will contain the training statistics, the saved model and the training progress files. :type output_directory: filepath (str) :param gpus: (string, default: `None`) list of GPUs to use (it uses the same syntax of CUDA_VISIBLE_DEVICES) :type gpus: str :param gpu_memory_limit: maximum memory in MB to allocate per GPU device. :type gpu_memory_limit: Integer :param allow_parallel_threads: allow TensorFlow to use multithreading parallelism to improve performance at the cost of determinism. :type allow_parallel_threads: Boolean :param random_seed: Random seed used for weights initialization, splits and any other random function. :type random_seed: Integer :param debug: If true turns on tfdbg with inf_or_nan checks. :type debug: Boolean :returns: None """ model_definition = check_which_model_definition(model_definition, model_definition_file) if model_load_path: model = LudwigModel.load(model_load_path) else: model = LudwigModel( model_definition=model_definition, logging_level=logging_level, use_horovod=use_horovod, gpus=gpus, gpu_memory_limit=gpu_memory_limit, allow_parallel_threads=allow_parallel_threads, random_seed=random_seed ) model.train( dataset=dataset, training_set=training_set, validation_set=validation_set, test_set=test_set, training_set_metadata=training_set_metadata, data_format=data_format, experiment_name=experiment_name, model_name=model_name, model_resume_path=model_resume_path, skip_save_training_description=skip_save_training_description, skip_save_training_statistics=skip_save_training_statistics, skip_save_model=skip_save_model, skip_save_progress=skip_save_progress, skip_save_log=skip_save_log, skip_save_processed_input=skip_save_processed_input, output_directory=output_directory, random_seed=random_seed, debug=debug, )