def set_logging_level(logging_level): """ :param logging_level: Set/Update the logging level. Use logging constants like `logging.DEBUG` , `logging.INFO` and `logging.ERROR`. :return: None """ logging.getLogger('ludwig').setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True)
def load(model_dir, logging_level=logging.ERROR): """This function allows for loading pretrained models # Inputs :param model_dir: (string) path to the directory containing the model. If the model was trained by the `train` or `experiment` command, the model is in `results_dir/experiment_dir/model`. :param logging_level: (int, default: `logging.ERROR`) logging level to use for logging. Use logging constants like `logging.DEBUG`, `logging.INFO` and `logging.ERROR`. By default only errors will be printed. # Return :return: (LudwigModel) a LudwigModel object # Example usage ```python ludwig_model = LudwigModel.load(model_dir) ``` """ logging.getLogger().setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True) model, model_definition = load_model_and_definition(model_dir) ludwig_model = LudwigModel(model_definition) ludwig_model.model = model ludwig_model.train_set_metadata = load_metadata( os.path.join( model_dir, TRAIN_SET_METADATA_FILE_NAME ) ) return ludwig_model
def _predict( self, data_df=None, data_csv=None, data_dict=None, return_type=pd.DataFrame, batch_size=128, gpus=None, gpu_fraction=1, only_predictions=True, logging_level=logging.ERROR, ): logging.getLogger().setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True) if (self.model is None or self.model_definition is None or self.train_set_metadata is None): raise ValueError('Model has not been trained or loaded') if data_df is None: data_df = self._read_data(data_csv, data_dict) logging.debug('Preprocessing {} datapoints'.format(len(data_df))) features_to_load = self.model_definition['input_features'] if not only_predictions: features_to_load += self.model_definition['output_features'] preprocessed_data = build_data(data_df, features_to_load, self.train_set_metadata, self.model_definition['preprocessing']) replace_text_feature_level( self.model_definition['input_features'] + ([] if only_predictions else self.model_definition['output_features']), [preprocessed_data]) dataset = Dataset(preprocessed_data, self.model_definition['input_features'], [] if only_predictions else self.model_definition['output_features'], None) logging.debug('Predicting') predict_results = self.model.predict(dataset, batch_size, only_predictions=only_predictions, gpus=gpus, gpu_fraction=gpu_fraction, session=getattr( self.model, 'session', None)) if not only_predictions: calculate_overall_stats(predict_results, self.model_definition['output_features'], dataset, self.train_set_metadata) logging.debug('Postprocessing') if (return_type == 'dict' or return_type == 'dictionary' or return_type == dict): postprocessed_predictions = postprocess( predict_results, self.model_definition['output_features'], self.train_set_metadata) elif (return_type == 'dataframe' or return_type == 'df' or return_type == pd.DataFrame): postprocessed_predictions = postprocess_df( predict_results, self.model_definition['output_features'], self.train_set_metadata) else: logging.warning('Unrecognized return_type: {}. ' 'Returning DataFrame.'.format(return_type)) postprocessed_predictions = postprocess( predict_results, self.model_definition['output_features'], self.train_set_metadata) return postprocessed_predictions, predict_results
def train_online( self, data_df=None, data_csv=None, data_dict=None, batch_size=None, learning_rate=None, regularization_lambda=None, dropout_rate=None, bucketing_field=None, gpus=None, gpu_fraction=1, logging_level=logging.ERROR, ): """This function is used to perform one epoch of training of the model on the specified dataset. # Inputs :param data_df: (DataFrame) dataframe containing data. :param data_csv: (string) input data CSV file. :param data_dict: (dict) input data dictionary. It is expected to contain one key for each field and the values have to be lists of the same length. Each index in the lists corresponds to one datapoint. For example a data set consisting of two datapoints with a text and a class may be provided as the following dict ``{'text_field_name': ['text of the first datapoint', text of the second datapoint'], 'class_filed_name': ['class_datapoints_1', 'class_datapoints_2']}`. :param batch_size: (int) the batch size to use for training. By default it's the one specified in the model definition. :param learning_rate: (float) the learning rate to use for training. By default the values is the one specified in the model definition. :param regularization_lambda: (float) the regularization lambda parameter to use for training. By default the values is the one specified in the model definition. :param dropout_rate: (float) the dropout rate to use for training. By default the values is the one specified in the model definition. :param bucketing_field: (string) the bucketing field to use for bucketing the data. By default the values is one specified in the model definition. :param gpus: (string, default: `None`) list of GPUs to use (it uses the same syntax of CUDA_VISIBLE_DEVICES) :param gpu_fraction: (float, default `1.0`) fraction of GPU memory to initialize the process with :param logging_level: (int, default: `logging.ERROR`) logging level to use for logging. Use logging constants like `logging.DEBUG`, `logging.INFO` and `logging.ERROR`. By default only errors will be printed. There are three ways to provide data: by dataframes using the `data_df` parameter, by CSV using the `data_csv` parameter and by dictionary, using the `data_dict` parameter. The DataFrame approach uses data previously obtained and put in a dataframe, the CSV approach loads data from a CSV file, while dict approach uses data organized by keys representing columns and values that are lists of the datapoints for each. For example a data set consisting of two datapoints with a text and a class may be provided as the following dict ``{'text_field_name}: ['text of the first datapoint', text of the second datapoint'], 'class_filed_name': ['class_datapoints_1', 'class_datapoints_2']}`. """ logging.getLogger().setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True) if (self.model is None or self.model_definition is None or self.train_set_metadata is None): raise ValueError('Model has not been initialized or loaded') if data_df is None: data_df = self._read_data(data_csv, data_dict) data_df.csv = data_csv if batch_size is None: batch_size = self.model_definition['training']['batch_size'] if learning_rate is None: learning_rate = self.model_definition['training']['learning_rate'] if regularization_lambda is None: regularization_lambda = self.model_definition['training'][ 'regularization_lambda'] if dropout_rate is None: dropout_rate = self.model_definition['training']['dropout_rate'], if bucketing_field is None: bucketing_field = self.model_definition['training'][ 'bucketing_field'] logging.debug('Preprocessing {} datapoints'.format(len(data_df))) features_to_load = (self.model_definition['input_features'] + self.model_definition['output_features']) preprocessed_data = build_data(data_df, features_to_load, self.train_set_metadata, self.model_definition['preprocessing']) replace_text_feature_level( self.model_definition['input_features'] + self.model_definition['output_features'], [preprocessed_data]) dataset = Dataset(preprocessed_data, self.model_definition['input_features'], self.model_definition['output_features'], None) logging.debug('Training batch') self.model.train_online(dataset, batch_size=batch_size, learning_rate=learning_rate, regularization_lambda=regularization_lambda, dropout_rate=dropout_rate, bucketing_field=bucketing_field, gpus=gpus, gpu_fraction=gpu_fraction)
def initialize_model(self, train_set_metadata=None, train_set_metadata_json=None, gpus=None, gpu_fraction=1, random_seed=default_random_seed, logging_level=logging.ERROR, debug=False, **kwargs): """This function initializes a model. It is need for performing online learning, so it has to be called before `train_online`. `train` initialize the model under the hood, so there is no need to call this function if you don't use `train_online`. # Inputs :param train_set_metadata: (dict) it contains metadata information for the input and output features the model is going to be trained on. It's the same content of the metadata json file that is created while training. :param train_set_metadata_json: (string) path to the JSON metadata file created while training. it contains metadata information for the input and output features the model is going to be trained on :param gpus: (string, default: `None`) list of GPUs to use (it uses the same syntax of CUDA_VISIBLE_DEVICES) :param gpu_fraction: (float, default `1.0`) fraction of GPU memory to initialize the process with :param random_seed: (int, default`42`) a random seed that is going to be used anywhere there is a call to a random number generator: data splitting, parameter initialization and training set shuffling :param logging_level: (int, default: `logging.ERROR`) logging level to use for logging. Use logging constants like `logging.DEBUG`, `logging.INFO` and `logging.ERROR`. By default only errors will be printed. :param debug: (bool, default: `False`) enables debugging mode """ logging.getLogger().setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True) if train_set_metadata is None and train_set_metadata_json is None: raise ValueError( 'train_set_metadata or train_set_metadata_json must not None.') if train_set_metadata_json is not None: train_set_metadata = load_metadata(train_set_metadata_json) # update model definition with metadata properties update_model_definition_with_metadata(self.model_definition, train_set_metadata) # build model model = Model(self.model_definition['input_features'], self.model_definition['output_features'], self.model_definition['combiner'], self.model_definition['training'], self.model_definition['preprocessing'], random_seed=random_seed, debug=debug) model.initialize_session(gpus=gpus, gpu_fraction=gpu_fraction) # set parameters self.model = model self.train_set_metadata = train_set_metadata
def train(self, data_df=None, data_train_df=None, data_validation_df=None, data_test_df=None, data_csv=None, data_train_csv=None, data_validation_csv=None, data_test_csv=None, data_hdf5=None, data_train_hdf5=None, data_validation_hdf5=None, data_test_hdf5=None, train_set_metadata_json=None, dataset_type='generic', model_name='run', model_load_path=None, model_resume_path=None, skip_save_model=False, skip_save_progress=False, skip_save_log=False, skip_save_processed_input=False, output_directory='results', gpus=None, gpu_fraction=1.0, random_seed=42, logging_level=logging.ERROR, debug=False, **kwargs): """This function is used to perform a full training of the model on the specified dataset. # Inputs :param data_df: (DataFrame) dataframe containing data. If it has a split column, it will be used for splitting (0: train, 1: validation, 2: test), otherwise the dataset will be randomly split :param data_train_df: (DataFrame) dataframe containing training data :param data_validation_df: (DataFrame) dataframe containing validation data :param data_test_df: (DataFrame dataframe containing test data :param data_csv: (string) input data CSV file. If it has a split column, it will be used for splitting (0: train, 1: validation, 2: test), otherwise the dataset will be randomly split :param data_train_csv: (string) input train data CSV file :param data_validation_csv: (string) input validation data CSV file :param data_test_csv: (string) input test data CSV file :param data_hdf5: (string) input data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param data_train_hdf5: (string) input train data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param data_validation_hdf5: (string) input validation data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param data_test_hdf5: (string) input test data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param train_set_metadata_json: (string) input metadata JSON file. It is an intermediate preprocess file containing the mappings of the input CSV created the first time a CSV file is used in the same directory with the same name and a json extension :param dataset_type: (string, default: `'default'`) determines the type of preprocessing will be applied to the data. Only `generic` is available at the moment :param model_name: (string) a name for the model, user for the save directory :param model_load_path: (string) path of a pretrained model to load as initialization :param model_resume_path: (string) path of a the model directory to resume training of :param skip_save_model: (bool, default: `False`) disables saving model weights and hyperparameters each time the model improves. By default Ludwig saves model weights after each epoch the validation measure imrpvoes, but if the model is really big that can be time consuming if you do not want to keep the weights and just find out what performance can a model get with a set of hyperparameters, use this parameter to skip it, but the model will not be loadable later on. :param skip_save_progress: (bool, default: `False`) disables saving progress each epoch. By default Ludwig saves weights and stats after each epoch for enabling resuming of training, but if the model is really big that can be time consuming and will uses twice as much space, use this parameter to skip it, but training cannot be resumed later on. :param skip_save_log: (bool, default: `False`) disables saving TensorBoard logs. By default Ludwig saves logs for the TensorBoard, but if it is not needed turning it off can slightly increase the overall speed. :param skip_save_processed_input: (bool, default: `False`) skips saving intermediate HDF5 and JSON files :param output_directory: (string, default: `'results'`) directory that contains the results :param gpus: (string, default: `None`) list of GPUs to use (it uses the same syntax of CUDA_VISIBLE_DEVICES) :param gpu_fraction: (float, default `1.0`) fraction of gpu memory to initialize the process with :param random_seed: (int, default`42`) a random seed that is going to be used anywhere there is a call to a random number generator: data splitting, parameter initialization and training set shuffling :param debug: (bool, default: `False`) enables debugging mode :param logging_level: (int, default: `logging.ERROR`) logging level to use for logging. Use logging constants like `logging.DEBUG`, `logging.INFO` and `logging.ERROR`. By default only errors will be printed. There are three ways to provide data: by dataframes using the `_df` parameters, by CSV using the `_csv` parameters and by HDF5 and JSON, using `_hdf5` and `_json` parameters. The DataFrame approach uses data previously obtained and put in a dataframe, the CSV approach loads data from a CSV file, while HDF5 and JSON load previously preprocessed HDF5 and JSON files (they are saved in the same directory of the CSV they are obtained from). For all three approaches either a full dataset can be provided (which will be split randomly according to the split probabilities defined in the model definition, by default 70% training, 10% validation and 20% test) or, if it contanins a plit column, it will be plit according to that column (interpreting 0 as training, 1 as validation and 2 as test). Alternatively separated dataframes / CSV / HDF5 files can beprovided for each split. During training the model and statistics will be saved in a directory `[output_dir]/[experiment_name]_[model_name]_n` where all variables are resolved to user spiecified ones and `n` is an increasing number starting from 0 used to differentiate different runs. # Return :return: (dict) a dictionary containing training statistics for each output feature containing loss and measures values for each epoch. """ logging.getLogger().setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True) # setup directories and file names experiment_dir_name = None if model_resume_path is not None: if os.path.exists(model_resume_path): experiment_dir_name = model_resume_path else: logging.info('Model resume path does not exists,' ' starting training from scratch') model_resume_path = None if model_resume_path is None: experiment_dir_name = get_experiment_dir_name( output_directory, '', model_name) description_fn, training_stats_fn, model_dir = get_file_names( experiment_dir_name) # save description description = get_experiment_description( self.model_definition, dataset_type, data_csv=data_csv, data_train_csv=data_train_csv, data_validation_csv=data_validation_csv, data_test_csv=data_test_csv, data_hdf5=data_hdf5, data_train_hdf5=data_train_hdf5, data_validation_hdf5=data_validation_hdf5, data_test_hdf5=data_test_hdf5, metadata_json=train_set_metadata_json, random_seed=random_seed) save_json(description_fn, description) # print description logging.info('Model name: {}'.format(model_name)) logging.info('Output path: {}'.format(experiment_dir_name)) logging.info('\n') for key, value in description.items(): logging.info('{0}: {1}'.format(key, pformat(value, indent=4))) logging.info('\n') # preprocess if data_df is not None or data_train_df is not None: (training_set, validation_set, test_set, train_set_metadata) = preprocess_for_training( self.model_definition, dataset_type, data_df=data_df, data_train_df=data_train_df, data_validation_df=data_validation_df, data_test_df=data_test_df, train_set_metadata_json=train_set_metadata_json, skip_save_processed_input=True, preprocessing_params=self.model_definition['preprocessing'], random_seed=random_seed) else: (training_set, validation_set, test_set, train_set_metadata) = preprocess_for_training( self.model_definition, dataset_type, data_csv=data_csv, data_train_csv=data_train_csv, data_validation_csv=data_validation_csv, data_test_csv=data_test_csv, data_hdf5=data_hdf5, data_train_hdf5=data_train_hdf5, data_validation_hdf5=data_validation_hdf5, data_test_hdf5=data_test_hdf5, train_set_metadata_json=train_set_metadata_json, skip_save_processed_input=skip_save_processed_input, preprocessing_params=self.model_definition['preprocessing'], random_seed=random_seed) logging.info('Training set: {0}'.format(training_set.size)) if validation_set is not None: logging.info('Validation set: {0}'.format(validation_set.size)) if test_set is not None: logging.info('Test set: {0}'.format(test_set.size)) # update model definition with metadata properties update_model_definition_with_metadata(self.model_definition, train_set_metadata) if not skip_save_model: os.makedirs(model_dir, exist_ok=True) train_set_metadata_path = os.path.join( model_dir, TRAIN_SET_METADATA_FILE_NAME) save_json(train_set_metadata_path, train_set_metadata) # run the experiment model, result = train(training_set=training_set, validation_set=validation_set, test_set=test_set, model_definition=self.model_definition, save_path=model_dir, model_load_path=model_load_path, resume=model_resume_path is not None, skip_save_model=skip_save_model, skip_save_progress=skip_save_progress, skip_save_log=skip_save_log, gpus=gpus, gpu_fraction=gpu_fraction, random_seed=random_seed, debug=debug) train_trainset_stats, train_valisest_stats, train_testset_stats = result train_stats = { 'train': train_trainset_stats, 'validation': train_valisest_stats, 'test': train_testset_stats } # save training and test statistics save_json(training_stats_fn, train_stats) # grab the results of the model with highest validation test performance md_training = self.model_definition['training'] validation_field = md_training['validation_field'] validation_measure = md_training['validation_measure'] validation_field_result = train_valisest_stats[validation_field] best_function = get_best_function(validation_measure) # print results of the model with highest validation test performance if validation_set is not None: # max or min depending on the measure epoch_best_vali_measure, best_vali_measure = best_function( enumerate(validation_field_result[validation_measure]), key=lambda pair: pair[1]) logging.info('Best validation model epoch: {0}'.format( epoch_best_vali_measure + 1)) logging.info( 'Best validation model {0} on validation set {1}: {2}'.format( validation_measure, validation_field, best_vali_measure)) if test_set is not None: best_vali_measure_epoch_test_measure = train_testset_stats[ validation_field][validation_measure][ epoch_best_vali_measure] logging.info( 'Best validation model {0} on test set {1}: {2}'.format( validation_measure, validation_field, best_vali_measure_epoch_test_measure)) logging.info('Finished: {0}'.format(model_name)) logging.info('Saved to {0}:'.format(experiment_dir_name)) # set parameters self.model = model self.train_set_metadata = train_set_metadata return train_stats
def train(self, data_df=None, data_train_df=None, data_validation_df=None, data_test_df=None, data_csv=None, data_train_csv=None, data_validation_csv=None, data_test_csv=None, data_hdf5=None, data_train_hdf5=None, data_validation_hdf5=None, data_test_hdf5=None, data_dict=None, data_train_dict=None, data_validation_dict=None, data_test_dict=None, train_set_metadata_json=None, experiment_name='api_experiment', model_name='run', model_load_path=None, model_resume_path=None, skip_save_model=False, skip_save_progress=False, skip_save_log=False, skip_save_processed_input=False, output_directory='results', gpus=None, gpu_fraction=1.0, use_horovod=False, random_seed=42, logging_level=logging.ERROR, debug=False, **kwargs): """This function is used to perform a full training of the model on the specified dataset. # Inputs :param data_df: (DataFrame) dataframe containing data. If it has a split column, it will be used for splitting (0: train, 1: validation, 2: test), otherwise the dataset will be randomly split :param data_train_df: (DataFrame) dataframe containing training data :param data_validation_df: (DataFrame) dataframe containing validation data :param data_test_df: (DataFrame dataframe containing test data :param data_csv: (string) input data CSV file. If it has a split column, it will be used for splitting (0: train, 1: validation, 2: test), otherwise the dataset will be randomly split :param data_train_csv: (string) input train data CSV file :param data_validation_csv: (string) input validation data CSV file :param data_test_csv: (string) input test data CSV file :param data_hdf5: (string) input data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param data_train_hdf5: (string) input train data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param data_validation_hdf5: (string) input validation data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param data_test_hdf5: (string) input test data HDF5 file. It is an intermediate preprocess version of the input CSV created the first time a CSV file is used in the same directory with the same name and a hdf5 extension :param data_dict: (dict) input data dictionary. It is expected to contain one key for each field and the values have to be lists of the same length. Each index in the lists corresponds to one datapoint. For example a data set consisting of two datapoints with a text and a class may be provided as the following dict `{'text_field_name': ['text of the first datapoint', text of the second datapoint'], 'class_filed_name': ['class_datapoints_1', 'class_datapoints_2']}`. :param data_train_dict: (dict) input training data dictionary. It is expected to contain one key for each field and the values have to be lists of the same length. Each index in the lists corresponds to one datapoint. For example a data set consisting of two datapoints with a text and a class may be provided as the following dict: `{'text_field_name': ['text of the first datapoint', 'text of the second datapoint'], 'class_field_name': ['class_datapoint_1', 'class_datapoint_2']}`. :param data_validation_dict: (dict) input validation data dictionary. It is expected to contain one key for each field and the values have to be lists of the same length. Each index in the lists corresponds to one datapoint. For example a data set consisting of two datapoints with a text and a class may be provided as the following dict: `{'text_field_name': ['text of the first datapoint', 'text of the second datapoint'], 'class_field_name': ['class_datapoint_1', 'class_datapoint_2']}`. :param data_test_dict: (dict) input test data dictionary. It is expected to contain one key for each field and the values have to be lists of the same length. Each index in the lists corresponds to one datapoint. For example a data set consisting of two datapoints with a text and a class may be provided as the following dict: `{'text_field_name': ['text of the first datapoint', 'text of the second datapoint'], 'class_field_name': ['class_datapoint_1', 'class_datapoint_2']}`. :param train_set_metadata_json: (string) input metadata JSON file. It is an intermediate preprocess file containing the mappings of the input CSV created the first time a CSV file is used in the same directory with the same name and a json extension :param experiment_name: (string) a name for the experiment, used for the save directory :param model_name: (string) a name for the model, used for the save directory :param model_load_path: (string) path of a pretrained model to load as initialization :param model_resume_path: (string) path of a the model directory to resume training of :param skip_save_model: (bool, default: `False`) disables saving model weights and hyperparameters each time the model improves. By default Ludwig saves model weights after each epoch the validation measure imrpvoes, but if the model is really big that can be time consuming if you do not want to keep the weights and just find out what performance can a model get with a set of hyperparameters, use this parameter to skip it, but the model will not be loadable later on. :param skip_save_progress: (bool, default: `False`) disables saving progress each epoch. By default Ludwig saves weights and stats after each epoch for enabling resuming of training, but if the model is really big that can be time consuming and will uses twice as much space, use this parameter to skip it, but training cannot be resumed later on. :param skip_save_log: (bool, default: `False`) disables saving TensorBoard logs. By default Ludwig saves logs for the TensorBoard, but if it is not needed turning it off can slightly increase the overall speed. :param skip_save_processed_input: (bool, default: `False`) skips saving intermediate HDF5 and JSON files :param output_directory: (string, default: `'results'`) directory that contains the results :param gpus: (string, default: `None`) list of GPUs to use (it uses the same syntax of CUDA_VISIBLE_DEVICES) :param gpu_fraction: (float, default `1.0`) fraction of gpu memory to initialize the process with :param random_seed: (int, default`42`) a random seed that is going to be used anywhere there is a call to a random number generator: data splitting, parameter initialization and training set shuffling :param debug: (bool, default: `False`) enables debugging mode :param logging_level: (int, default: `logging.ERROR`) logging level to use for logging. Use logging constants like `logging.DEBUG`, `logging.INFO` and `logging.ERROR`. By default only errors will be printed. There are three ways to provide data: by dataframes using the `_df` parameters, by CSV using the `_csv` parameters and by HDF5 and JSON, using `_hdf5` and `_json` parameters. The DataFrame approach uses data previously obtained and put in a dataframe, the CSV approach loads data from a CSV file, while HDF5 and JSON load previously preprocessed HDF5 and JSON files (they are saved in the same directory of the CSV they are obtained from). For all three approaches either a full dataset can be provided (which will be split randomly according to the split probabilities defined in the model definition, by default 70% training, 10% validation and 20% test) or, if it contanins a plit column, it will be plit according to that column (interpreting 0 as training, 1 as validation and 2 as test). Alternatively separated dataframes / CSV / HDF5 files can beprovided for each split. During training the model and statistics will be saved in a directory `[output_dir]/[experiment_name]_[model_name]_n` where all variables are resolved to user spiecified ones and `n` is an increasing number starting from 0 used to differentiate different runs. # Return :return: (dict) a dictionary containing training statistics for each output feature containing loss and measures values for each epoch. """ logging.getLogger('ludwig').setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True) if data_df is None and data_dict is not None: data_df = pd.DataFrame(data_dict) if data_train_df is None and data_train_dict is not None: data_train_df = pd.DataFrame(data_train_dict) if data_validation_df is None and data_validation_dict is not None: data_validation_df = pd.DataFrame(data_validation_dict) if data_test_df is None and data_test_dict is not None: data_test_df = pd.DataFrame(data_test_dict) (self.model, preprocessed_data, self.exp_dir_name, train_stats, self.model_definition) = full_train( self.model_definition, data_df=data_df, data_train_df=data_train_df, data_validation_df=data_validation_df, data_test_df=data_test_df, data_csv=data_csv, data_train_csv=data_train_csv, data_validation_csv=data_validation_csv, data_test_csv=data_test_csv, data_hdf5=data_hdf5, data_train_hdf5=data_train_hdf5, data_validation_hdf5=data_validation_hdf5, data_test_hdf5=data_test_hdf5, train_set_metadata_json=train_set_metadata_json, experiment_name=experiment_name, model_name=model_name, model_load_path=model_load_path, model_resume_path=model_resume_path, skip_save_model=skip_save_model, skip_save_progress=skip_save_progress, skip_save_log=skip_save_log, skip_save_processed_input=skip_save_processed_input, output_directory=output_directory, should_close_session=False, gpus=gpus, gpu_fraction=gpu_fraction, use_horovod=use_horovod, random_seed=random_seed, debug=debug, ) self.train_set_metadata = preprocessed_data[-1] return train_stats
def _predict( self, data_df=None, data_csv=None, data_dict=None, return_type=pd.DataFrame, batch_size=128, gpus=None, gpu_fraction=1, evaluate_performance=False, logging_level=logging.ERROR, ): logging.getLogger('ludwig').setLevel(logging_level) if logging_level in {logging.WARNING, logging.ERROR, logging.CRITICAL}: set_disable_progressbar(True) if (self.model is None or self.model_definition is None or self.train_set_metadata is None): raise ValueError('Model has not been trained or loaded') if data_df is None: data_df = self._read_data(data_csv, data_dict) logger.debug('Preprocessing {} datapoints'.format(len(data_df))) # Added [:] to next line, before I was just assigning, # this way I'm copying the list. If you don't do it, you are actually # modifying the input feature list when you add output features, # which you definitely don't want to do features_to_load = self.model_definition['input_features'][:] if evaluate_performance: output_features = self.model_definition['output_features'] else: output_features = [] features_to_load += output_features preprocessed_data = build_data(data_df, features_to_load, self.train_set_metadata, self.model_definition['preprocessing']) replace_text_feature_level(features_to_load, [preprocessed_data]) dataset = Dataset(preprocessed_data, self.model_definition['input_features'], output_features, None) logger.debug('Predicting') predict_results = self.model.predict( dataset, batch_size, evaluate_performance=evaluate_performance, gpus=gpus, gpu_fraction=gpu_fraction, session=getattr(self.model, 'session', None)) if evaluate_performance: calculate_overall_stats(predict_results, self.model_definition['output_features'], dataset, self.train_set_metadata) logger.debug('Postprocessing') if (return_type == 'dict' or return_type == 'dictionary' or return_type == dict): postprocessed_predictions = postprocess( predict_results, self.model_definition['output_features'], self.train_set_metadata) elif (return_type == 'dataframe' or return_type == 'df' or return_type == pd.DataFrame): postprocessed_predictions = postprocess_df( predict_results, self.model_definition['output_features'], self.train_set_metadata) else: logger.warning('Unrecognized return_type: {}. ' 'Returning DataFrame.'.format(return_type)) postprocessed_predictions = postprocess( predict_results, self.model_definition['output_features'], self.train_set_metadata) return postprocessed_predictions, predict_results