def test_datetime_split(df_engine):
nrows = 100
npartitions = 10
df = pd.DataFrame(np.random.randint(0, 100, size=(nrows, 3)),
columns=["A", "B", "C"])
def random_date(*args, **kwargs):
start = datetime.strptime("1/1/1990 1:30 PM", "%m/%d/%Y %I:%M %p")
end = datetime.strptime("1/1/2022 4:50 AM", "%m/%d/%Y %I:%M %p")
delta = end - start
int_delta = (delta.days * 24 * 60 * 60) + delta.seconds
random_second = randrange(int_delta)
return str(start + timedelta(seconds=random_second))
df["date_col"] = df["C"].map(random_date)
if isinstance(df_engine, DaskEngine):
df = df_engine.df_lib.from_pandas(df, npartitions=npartitions)
probs = (0.7, 0.1, 0.2)
split_params = {
"type": "datetime",
"column": "date_col",
"probabilities": probs,
}
splitter = get_splitter(**split_params)
backend = Mock()
backend.df_engine = df_engine
splits = splitter.split(df, backend)
assert len(splits) == 3
min_datestr = "1990-01-01 00:00:00"
for split, p in zip(splits, probs):
if isinstance(df_engine, DaskEngine):
# Dask splitting is not exact, so apply soft constraint here
split = split.compute()
assert np.isclose(len(split), int(nrows * p), atol=15)
else:
assert len(split) == int(nrows * p)
assert np.all(split["date_col"] > min_datestr)
min_datestr = split["date_col"].max()
def test_stratify_split():
nrows = 100
thresholds = [60, 80, 100]
df = pd.DataFrame(np.random.randint(0, 100, size=(nrows, 3)),
columns=["A", "B", "C"])
def get_category(v):
if v < thresholds[0]:
return 0
if thresholds[0] <= v < thresholds[1]:
return 1
return 2
df["category"] = df.index.map(get_category).astype(np.int8)
probs = (0.7, 0.1, 0.2)
split_params = {
"type": "stratify",
"column": "category",
"probabilities": probs,
}
splitter = get_splitter(**split_params)
backend = Mock()
backend.df_engine = PandasEngine()
splits = splitter.split(df, backend, random_seed=42)
assert len(splits) == 3
ratios = [60, 20, 20]
for split, p in zip(splits, probs):
for idx, r in enumerate(ratios):
actual = np.sum(split["category"] == idx)
expected = int(r * p)
assert np.isclose(actual, expected, atol=1)
# Test determinism
splits2 = splitter.split(df, backend, random_seed=7)
for s1, s2 in zip(splits, splits2):
assert not s1.equals(s2)
splits3 = splitter.split(df, backend, random_seed=42)
for s1, s3 in zip(splits, splits3):
assert s1.equals(s3)
def test_random_split(df_engine):
nrows = 100
npartitions = 10
df = pd.DataFrame(np.random.randint(0, 100, size=(nrows, 3)),
columns=["A", "B", "C"])
if isinstance(df_engine, DaskEngine):
df = df_engine.df_lib.from_pandas(df, npartitions=npartitions)
probs = (0.7, 0.1, 0.2)
split_params = {
"type": "random",
"probabilities": probs,
}
splitter = get_splitter(**split_params)
backend = Mock()
backend.df_engine = df_engine
splits = splitter.split(df, backend, random_seed=42)
assert len(splits) == 3
for split, p in zip(splits, probs):
if isinstance(df_engine, DaskEngine):
# Dask splitting is not exact, so apply soft constraint here
assert np.isclose(len(split), int(nrows * p), atol=5)
else:
assert len(split) == int(nrows * p)
# Test determinism
def compute(dfs):
return [
df.compute() if isinstance(backend.df_engine, DaskEngine) else df
for df in dfs
]
splits = compute(splits)
splits2 = compute(splitter.split(df, backend, random_seed=7))
for s1, s2 in zip(splits, splits2):
assert not s1.equals(s2)
splits3 = compute(splitter.split(df, backend, random_seed=42))
for s1, s3 in zip(splits, splits3):
assert s1.equals(s3)
def test_fixed_split(df_engine):
nrows = 100
npartitions = 10
thresholds = [60, 80, 100]
df = pd.DataFrame(np.random.randint(0, 100, size=(nrows, 3)),
columns=["A", "B", "C"])
def get_split(v):
if v < thresholds[0]:
return 0
if thresholds[0] <= v < thresholds[1]:
return 1
return 2
df["split_col"] = df["C"].map(get_split).astype(np.int8)
if isinstance(df_engine, DaskEngine):
df = df_engine.df_lib.from_pandas(df, npartitions=npartitions)
split_params = {
"type": "fixed",
"column": "split_col",
}
splitter = get_splitter(**split_params)
backend = Mock()
backend.df_engine = df_engine
splits = splitter.split(df, backend)
assert len(splits) == 3
last_t = 0
for split, t in zip(splits, thresholds):
if isinstance(df_engine, DaskEngine):
split = split.compute()
assert np.all(split["C"] < t)
assert np.all(split["C"] >= last_t)
last_t = t
def test_model_save_reload_api(tmpdir, csv_filename, tmp_path):
torch.manual_seed(1)
random.seed(1)
np.random.seed(1)
image_dest_folder = os.path.join(os.getcwd(), "generated_images")
audio_dest_folder = os.path.join(os.getcwd(), "generated_audio")
input_features = [
binary_feature(),
number_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3,
encoder="rnn",
cell_type="lstm",
num_layers=2,
bidirections=True),
vector_feature(),
image_feature(image_dest_folder),
audio_feature(audio_dest_folder, encoder="stacked_cnn"),
timeseries_feature(encoder="parallel_cnn"),
sequence_feature(vocab_size=3, encoder="stacked_parallel_cnn"),
date_feature(),
h3_feature(),
set_feature(vocab_size=3),
bag_feature(vocab_size=3),
]
output_features = [
binary_feature(),
number_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
set_feature(vocab_size=3),
vector_feature(),
]
# Generate test data
data_csv_path = generate_data(input_features,
output_features,
csv_filename,
num_examples=50)
#############
# Train model
#############
config = {
"input_features": input_features,
"output_features": output_features,
TRAINER: {
"epochs": 2
}
}
data_df = read_csv(data_csv_path)
splitter = get_splitter("random")
training_set, validation_set, test_set = splitter.split(
data_df, LocalTestBackend())
# create sub-directory to store results
results_dir = tmp_path / "results"
results_dir.mkdir()
# perform initial model training
backend = LocalTestBackend()
ludwig_model1 = LudwigModel(config, backend=backend)
_, _, output_dir = ludwig_model1.train(
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
output_directory="results", # results_dir
)
preds_1, _ = ludwig_model1.predict(dataset=validation_set)
def check_model_equal(ludwig_model2):
# Compare model predictions
preds_2, _ = ludwig_model2.predict(dataset=validation_set)
assert set(preds_1.keys()) == set(preds_2.keys())
for key in preds_1:
assert preds_1[key].dtype == preds_2[key].dtype, key
assert np.all(a == b
for a, b in zip(preds_1[key], preds_2[key])), key
# assert preds_2[key].dtype == preds_3[key].dtype, key
# assert list(preds_2[key]) == list(preds_3[key]), key
# Compare model weights
for if_name in ludwig_model1.model.input_features:
if1 = ludwig_model1.model.input_features[if_name]
if2 = ludwig_model2.model.input_features[if_name]
for if1_w, if2_w in zip(if1.encoder_obj.parameters(),
if2.encoder_obj.parameters()):
assert torch.allclose(if1_w, if2_w)
c1 = ludwig_model1.model.combiner
c2 = ludwig_model2.model.combiner
for c1_w, c2_w in zip(c1.parameters(), c2.parameters()):
assert torch.allclose(c1_w, c2_w)
for of_name in ludwig_model1.model.output_features:
of1 = ludwig_model1.model.output_features[of_name]
of2 = ludwig_model2.model.output_features[of_name]
for of1_w, of2_w in zip(of1.decoder_obj.parameters(),
of2.decoder_obj.parameters()):
assert torch.allclose(of1_w, of2_w)
ludwig_model1.save(tmpdir)
ludwig_model_loaded = LudwigModel.load(tmpdir, backend=backend)
check_model_equal(ludwig_model_loaded)
# Test loading the model from the experiment directory
ludwig_model_exp = LudwigModel.load(os.path.join(output_dir, "model"),
backend=backend)
check_model_equal(ludwig_model_exp)
def test_gbm_model_save_reload_api(tmpdir, csv_filename, tmp_path):
torch.manual_seed(1)
random.seed(1)
np.random.seed(1)
input_features = [
binary_feature(),
number_feature(),
category_feature(vocab_size=3)
]
output_features = [category_feature(vocab_size=3)]
# Generate test data
data_csv_path = generate_data(input_features, output_features,
csv_filename)
#############
# Train tree model
#############
config = {
"model_type": "gbm",
"input_features": input_features,
"output_features": output_features,
TRAINER: {
"num_boost_round": 2
},
}
data_df = read_csv(data_csv_path)
splitter = get_splitter("random")
training_set, validation_set, test_set = splitter.split(
data_df, LocalTestBackend())
# create sub-directory to store results
results_dir = tmp_path / "results"
results_dir.mkdir()
# perform initial model training
backend = LocalTestBackend()
ludwig_model1 = LudwigModel(config, backend=backend)
_, _, output_dir = ludwig_model1.train(
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
output_directory="results", # results_dir
)
preds_1, _ = ludwig_model1.predict(dataset=validation_set)
def check_model_equal(ludwig_model2):
# Compare model predictions
preds_2, _ = ludwig_model2.predict(dataset=validation_set)
assert set(preds_1.keys()) == set(preds_2.keys())
for key in preds_1:
assert preds_1[key].dtype == preds_2[key].dtype, key
assert np.all(a == b
for a, b in zip(preds_1[key], preds_2[key])), key
# Compare model weights
for if_name in ludwig_model1.model.input_features:
if1 = ludwig_model1.model.input_features[if_name]
if2 = ludwig_model2.model.input_features[if_name]
for if1_w, if2_w in zip(if1.encoder_obj.parameters(),
if2.encoder_obj.parameters()):
assert torch.allclose(if1_w, if2_w)
tree1 = ludwig_model1.model.compiled_model
tree2 = ludwig_model2.model.compiled_model
for t1_w, t2_w in zip(tree1.parameters(), tree2.parameters()):
assert torch.allclose(t1_w, t2_w)
for of_name in ludwig_model1.model.output_features:
of1 = ludwig_model1.model.output_features[of_name]
of2 = ludwig_model2.model.output_features[of_name]
for of1_w, of2_w in zip(of1.decoder_obj.parameters(),
of2.decoder_obj.parameters()):
assert torch.allclose(of1_w, of2_w)
# Test saving and loading the model explicitly
ludwig_model1.save(tmpdir)
ludwig_model_loaded = LudwigModel.load(tmpdir, backend=backend)
check_model_equal(ludwig_model_loaded)
# Test loading the model from the experiment directory
ludwig_model_exp = LudwigModel.load(os.path.join(output_dir, "model"),
backend=backend)
check_model_equal(ludwig_model_exp)
def merge_with_defaults(config: dict) -> dict: # noqa: F821
config = copy.deepcopy(config)
upgrade_deprecated_fields(config)
_perform_sanity_checks(config)
_set_feature_column(config)
_set_proc_column(config)
_merge_hyperopt_with_trainer(config)
# ===== Defaults =====
if DEFAULTS not in config:
config[DEFAULTS] = dict()
# Update defaults with the default feature specific preprocessing parameters
for feature_type, preprocessing_defaults in default_feature_specific_preprocessing_parameters.items():
# Create a new key with feature type if defaults is empty
if feature_type not in config.get(DEFAULTS):
if PREPROCESSING in preprocessing_defaults:
config[DEFAULTS][feature_type] = preprocessing_defaults
else:
config[DEFAULTS][feature_type] = {PREPROCESSING: preprocessing_defaults}
# Feature type exists but preprocessing hasn't be specified
elif PREPROCESSING not in config[DEFAULTS][feature_type]:
config[DEFAULTS][feature_type][PREPROCESSING] = preprocessing_defaults
# Preprocessing parameters exist for feature type, update defaults with parameters from config
else:
config[DEFAULTS][feature_type][PREPROCESSING].update(
merge_dict(preprocessing_defaults, config[DEFAULTS][feature_type][PREPROCESSING])
)
# ===== Preprocessing =====
config[PREPROCESSING] = merge_dict(base_preprocessing_parameters, config.get(PREPROCESSING, {}))
splitter = get_splitter(**config[PREPROCESSING].get(SPLIT, {}))
splitter.validate(config)
# ===== Model Type =====
set_default_value(config, MODEL_TYPE, default_model_type)
# ===== Training =====
# Convert config dictionary into an instance of BaseTrainerConfig.
full_trainer_config, _ = load_trainer_with_kwargs(config[MODEL_TYPE], config[TRAINER] if TRAINER in config else {})
config[TRAINER] = asdict(full_trainer_config)
set_default_value(
config[TRAINER],
"validation_metric",
output_type_registry[config[OUTPUT_FEATURES][0][TYPE]].default_validation_metric,
)
# ===== Input Features =====
for input_feature in config[INPUT_FEATURES]:
if config[MODEL_TYPE] == MODEL_GBM:
input_feature[ENCODER] = "passthrough"
remove_ecd_params(input_feature)
get_from_registry(input_feature[TYPE], input_type_registry).populate_defaults(input_feature)
# Update encoder parameters for output feature from global defaults
update_feature_from_defaults(config, input_feature, INPUT_FEATURES)
# ===== Combiner =====
set_default_value(config, COMBINER, {TYPE: default_combiner_type})
full_combiner_config, _ = load_config_with_kwargs(
combiner_registry[config[COMBINER][TYPE]].get_schema_cls(), config[COMBINER]
)
config[COMBINER].update(asdict(full_combiner_config))
# ===== Output features =====
for output_feature in config[OUTPUT_FEATURES]:
if config[MODEL_TYPE] == MODEL_GBM:
output_feature[DECODER] = "passthrough"
remove_ecd_params(output_feature)
get_from_registry(output_feature[TYPE], output_type_registry).populate_defaults(output_feature)
# By default, drop rows with missing output features
set_default_value(output_feature, PREPROCESSING, {})
set_default_value(output_feature[PREPROCESSING], "missing_value_strategy", DROP_ROW)
# Update decoder and loss related parameters for output feature from global defaults
update_feature_from_defaults(config, output_feature, OUTPUT_FEATURES)
# ===== Hyperpot =====
if HYPEROPT in config:
set_default_value(config[HYPEROPT][EXECUTOR], TYPE, RAY)
return config
def hyperopt(
config: Union[str, dict],
dataset: Union[str, dict, pd.DataFrame] = None,
training_set: Union[str, dict, pd.DataFrame] = None,
validation_set: Union[str, dict, pd.DataFrame] = None,
test_set: Union[str, dict, pd.DataFrame] = None,
training_set_metadata: Union[str, dict] = None,
data_format: str = None,
experiment_name: str = "hyperopt",
model_name: str = "run",
resume: Optional[bool] = None,
skip_save_training_description: bool = False,
skip_save_training_statistics: bool = False,
skip_save_model: bool = False,
skip_save_progress: bool = False,
skip_save_log: bool = False,
skip_save_processed_input: bool = True,
skip_save_unprocessed_output: bool = False,
skip_save_predictions: bool = False,
skip_save_eval_stats: bool = False,
skip_save_hyperopt_statistics: bool = False,
output_directory: str = "results",
gpus: Union[str, int, List[int]] = None,
gpu_memory_limit: int = None,
allow_parallel_threads: bool = True,
callbacks: List[Callback] = None,
backend: Union[Backend, str] = None,
random_seed: int = default_random_seed,
hyperopt_log_verbosity: int = 3,
**kwargs,
) -> HyperoptResults:
"""This method performs an hyperparameter optimization.
# Inputs
:param config: (Union[str, dict]) config which defines
the different parameters of the model, features, preprocessing and
training. If `str`, filepath to yaml configuration file.
:param dataset: (Union[str, dict, pandas.DataFrame], default: `None`)
source containing the entire dataset to be used in the experiment.
If it has a split column, it will be used for splitting (0 for train,
1 for validation, 2 for test), otherwise the dataset will be
randomly split.
:param training_set: (Union[str, dict, pandas.DataFrame], default: `None`)
source containing training data.
:param validation_set: (Union[str, dict, pandas.DataFrame], default: `None`)
source containing validation data.
:param test_set: (Union[str, dict, pandas.DataFrame], default: `None`)
source containing test data.
:param training_set_metadata: (Union[str, dict], default: `None`)
metadata JSON file or loaded metadata. Intermediate preprocessed
structure containing the mappings of the input
dataset created the first time an input file is used in the same
directory with the same name and a '.meta.json' extension.
:param data_format: (str, default: `None`) format to interpret data
sources. Will be inferred automatically if not specified. Valid
formats are `'auto'`, `'csv'`, `'df'`, `'dict'`, `'excel'`, `'feather'`,
`'fwf'`, `'hdf5'` (cache file produced during previous training),
`'html'` (file containing a single HTML `<table>`), `'json'`, `'jsonl'`,
`'parquet'`, `'pickle'` (pickled Pandas DataFrame), `'sas'`, `'spss'`,
`'stata'`, `'tsv'`.
:param experiment_name: (str, default: `'experiment'`) name for
the experiment.
:param model_name: (str, default: `'run'`) name of the model that is
being used.
:param resume: (bool) If true, continue hyperopt from the state of the previous
run in the output directory with the same experiment name. If false, will create
new trials, ignoring any previous state, even if they exist in the output_directory.
By default, will attempt to resume if there is already an existing experiment with
the same name, and will create new trials if not.
:param skip_save_training_description: (bool, default: `False`) disables
saving the description JSON file.
:param skip_save_training_statistics: (bool, default: `False`) disables
saving training statistics JSON file.
:param skip_save_model: (bool, default: `False`) disables
saving model weights and hyperparameters each time the model
improves. By default Ludwig saves model weights after each epoch
the validation metric improves, but if the model is really big
that can be time consuming. If you do not want to keep
the weights and just find out what performance a model can get
with a set of hyperparameters, use this parameter to skip it,
but the model will not be loadable later on and the returned model
will have the weights obtained at the end of training, instead of
the weights of the epoch with the best validation performance.
:param skip_save_progress: (bool, default: `False`) disables saving
progress each epoch. By default Ludwig saves weights and stats
after each epoch for enabling resuming of training, but if
the model is really big that can be time consuming and will uses
twice as much space, use this parameter to skip it, but training
cannot be resumed later on.
:param skip_save_log: (bool, default: `False`) disables saving
TensorBoard logs. By default Ludwig saves logs for the TensorBoard,
but if it is not needed turning it off can slightly increase the
overall speed.
:param skip_save_processed_input: (bool, default: `False`) if input
dataset is provided it is preprocessed and cached by saving an HDF5
and JSON files to avoid running the preprocessing again. If this
parameter is `False`, the HDF5 and JSON file are not saved.
:param skip_save_unprocessed_output: (bool, default: `False`) by default
predictions and their probabilities are saved in both raw
unprocessed numpy files containing tensors and as postprocessed
CSV files (one for each output feature). If this parameter is True,
only the CSV ones are saved and the numpy ones are skipped.
:param skip_save_predictions: (bool, default: `False`) skips saving test
predictions CSV files.
:param skip_save_eval_stats: (bool, default: `False`) skips saving test
statistics JSON file.
:param skip_save_hyperopt_statistics: (bool, default: `False`) skips saving
hyperopt stats file.
:param output_directory: (str, default: `'results'`) the directory that
will contain the training statistics, TensorBoard logs, the saved
model and the training progress files.
:param gpus: (list, default: `None`) list of GPUs that are available
for training.
:param gpu_memory_limit: (int, default: `None`) maximum memory in MB to
allocate per GPU device.
:param allow_parallel_threads: (bool, default: `True`) allow TensorFlow
to use multithreading parallelism to improve performance at
the cost of determinism.
:param callbacks: (list, default: `None`) a list of
`ludwig.callbacks.Callback` objects that provide hooks into the
Ludwig pipeline.
:param backend: (Union[Backend, str]) `Backend` or string name
of backend to use to execute preprocessing / training steps.
:param random_seed: (int: default: 42) random seed used for weights
initialization, splits and any other random function.
:param hyperopt_log_verbosity: (int: default: 3) controls verbosity of
ray tune log messages. Valid values: 0 = silent, 1 = only status updates,
2 = status and brief trial results, 3 = status and detailed trial results.
# Return
:return: (List[dict]) List of results for each trial, ordered by
descending performance on the target metric.
"""
from ludwig.hyperopt.execution import get_build_hyperopt_executor, RayTuneExecutor
# check if config is a path or a dict
if isinstance(config, str): # assume path
with open_file(config, "r") as def_file:
config_dict = yaml.safe_load(def_file)
else:
config_dict = config
# Get mapping of input/output features that don't have an encoder for shared parameters
features_eligible_for_shared_params = {
INPUT_FEATURES:
get_features_eligible_for_shared_params(config_dict, INPUT_FEATURES),
OUTPUT_FEATURES:
get_features_eligible_for_shared_params(config_dict, OUTPUT_FEATURES),
}
# merge config with defaults
config = merge_with_defaults(config_dict)
if HYPEROPT not in config:
raise ValueError("Hyperopt Section not present in config")
hyperopt_config = config[HYPEROPT]
update_hyperopt_params_with_defaults(hyperopt_config)
# print hyperopt config
logging.info("Hyperopt config")
logging.info(pformat(hyperopt_config, indent=4))
logging.info("\n")
logging.info(
"Features that may be updated in hyperopt trials if default parameters are specified in the search space"
)
logging.info(pformat(dict(features_eligible_for_shared_params), indent=4))
logging.info("\n")
search_alg = hyperopt_config["search_alg"]
executor = hyperopt_config[EXECUTOR]
parameters = hyperopt_config["parameters"]
split = hyperopt_config["split"]
output_feature = hyperopt_config["output_feature"]
metric = hyperopt_config["metric"]
goal = hyperopt_config["goal"]
######################
# check validity of output_feature / metric/ split combination
######################
splitter = get_splitter(**config[PREPROCESSING]["split"])
if split == TRAINING:
if training_set is None and not splitter.has_split(0):
raise ValueError(
'The data for the specified split for hyperopt "{}" '
"was not provided, "
"or the split amount specified in the preprocessing section "
"of the config is not greater than 0".format(split))
elif split == VALIDATION:
if validation_set is None and not splitter.has_split(1):
raise ValueError(
'The data for the specified split for hyperopt "{}" '
"was not provided, "
"or the split amount specified in the preprocessing section "
"of the config is not greater than 0".format(split))
elif split == TEST:
if test_set is None and not splitter.has_split(2):
raise ValueError(
'The data for the specified split for hyperopt "{}" '
"was not provided, "
"or the split amount specified in the preprocessing section "
"of the config is not greater than 0".format(split))
else:
raise ValueError('unrecognized hyperopt split "{}". '
"Please provide one of: {}".format(
split, {TRAINING, VALIDATION, TEST}))
if output_feature == COMBINED:
if metric != LOSS:
raise ValueError(
'The only valid metric for "combined" output feature is "loss"'
)
else:
output_feature_names = {of[NAME] for of in config[OUTPUT_FEATURES]}
if output_feature not in output_feature_names:
raise ValueError('The output feature specified for hyperopt "{}" '
"cannot be found in the config. "
'Available ones are: {} and "combined"'.format(
output_feature, output_feature_names))
output_feature_type = None
for of in config[OUTPUT_FEATURES]:
if of[NAME] == output_feature:
output_feature_type = of[TYPE]
feature_class = get_from_registry(output_feature_type,
output_type_registry)
if metric not in feature_class.metric_functions:
# todo v0.4: allow users to specify also metrics from the overall
# and per class metrics from the trainign stats and in general
# and potprocessed metric
raise ValueError(
'The specified metric for hyperopt "{}" is not a valid metric '
'for the specified output feature "{}" of type "{}". '
"Available metrics are: {}".format(
metric, output_feature, output_feature_type,
feature_class.metric_functions.keys()))
hyperopt_executor = get_build_hyperopt_executor(executor[TYPE])(
parameters,
output_feature,
metric,
goal,
split,
search_alg=search_alg,
**executor)
# Explicitly default to a local backend to avoid picking up Ray or Horovod
# backend from the environment.
backend = backend or config_dict.get("backend") or "local"
backend = initialize_backend(backend)
if not (isinstance(backend, LocalBackend) or
(isinstance(hyperopt_executor, RayTuneExecutor)
and isinstance(backend, RayBackend))):
raise ValueError(
"Hyperopt requires using a `local` backend at this time, or "
"`ray` backend with `ray` executor.")
for callback in callbacks or []:
callback.on_hyperopt_init(experiment_name)
if not should_tune_preprocessing(config):
# preprocessing is not being tuned, so generate it once before starting trials
for callback in callbacks or []:
callback.on_hyperopt_preprocessing_start(experiment_name)
model = LudwigModel(
config=config,
backend=backend,
gpus=gpus,
gpu_memory_limit=gpu_memory_limit,
allow_parallel_threads=allow_parallel_threads,
callbacks=callbacks,
)
training_set, validation_set, test_set, training_set_metadata = model.preprocess(
dataset=dataset,
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
training_set_metadata=training_set_metadata,
data_format=data_format,
skip_save_processed_input=skip_save_processed_input,
random_seed=random_seed,
)
dataset = None
for callback in callbacks or []:
callback.on_hyperopt_preprocessing_end(experiment_name)
for callback in callbacks or []:
callback.on_hyperopt_start(experiment_name)
hyperopt_results = hyperopt_executor.execute(
config,
dataset=dataset,
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
training_set_metadata=training_set_metadata,
data_format=data_format,
experiment_name=experiment_name,
model_name=model_name,
resume=resume,
skip_save_training_description=skip_save_training_description,
skip_save_training_statistics=skip_save_training_statistics,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
skip_save_log=skip_save_log,
skip_save_processed_input=skip_save_processed_input,
skip_save_unprocessed_output=skip_save_unprocessed_output,
skip_save_predictions=skip_save_predictions,
skip_save_eval_stats=skip_save_eval_stats,
output_directory=output_directory,
gpus=gpus,
gpu_memory_limit=gpu_memory_limit,
allow_parallel_threads=allow_parallel_threads,
callbacks=callbacks,
backend=backend,
random_seed=random_seed,
hyperopt_log_verbosity=hyperopt_log_verbosity,
features_eligible_for_shared_params=features_eligible_for_shared_params,
**kwargs,
)
if backend.is_coordinator():
print_hyperopt_results(hyperopt_results)
if not skip_save_hyperopt_statistics:
results_directory = os.path.join(output_directory, experiment_name)
makedirs(results_directory, exist_ok=True)
hyperopt_stats = {
"hyperopt_config":
hyperopt_config,
"hyperopt_results":
[t.to_dict() for t in hyperopt_results.ordered_trials],
}
save_hyperopt_stats(hyperopt_stats, results_directory)
logging.info(f"Hyperopt stats saved to: {results_directory}")
for callback in callbacks or []:
callback.on_hyperopt_end(experiment_name)
callback.on_hyperopt_finish(experiment_name)
logging.info("Finished hyperopt")
return hyperopt_results
