def test_server_integration_with_audio(single_record, csv_filename):
# Audio Inputs
audio_dest_folder = os.path.join(os.getcwd(), "generated_audio")
# Resnet encoder
input_features = [
audio_feature(
folder=audio_dest_folder,
),
text_feature(encoder="embed", min_len=1),
numerical_feature(normalization="zscore"),
]
output_features = [category_feature(vocab_size=4), numerical_feature()]
rel_path = generate_data(input_features, output_features, csv_filename)
model, output_dir = train_model(input_features, output_features, data_csv=rel_path)
app = server(model)
client = TestClient(app)
response = client.get("/")
assert response.status_code == 200
response = client.post("/predict")
# expect the HTTP 400 error code for this situation
assert response.status_code == 400
assert response.json() == ALL_FEATURES_PRESENT_ERROR
data_df = read_csv(rel_path)
if single_record:
# Single record prediction
first_entry = data_df.T.to_dict()[0]
data, files = convert_to_form(first_entry)
server_response = client.post("/predict", data=data, files=files)
assert server_response.status_code == 200
server_response = server_response.json()
server_response_keys = sorted(list(server_response.keys()))
assert server_response_keys == sorted(output_keys_for(output_features))
model_output, _ = model.predict(dataset=[first_entry], data_format=dict)
model_output = model_output.to_dict("records")[0]
assert model_output == server_response
else:
# Batch prediction
assert len(data_df) > 1
files = convert_to_batch_form(data_df)
server_response = client.post("/batch_predict", files=files)
assert server_response.status_code == 200
server_response = server_response.json()
server_response_keys = sorted(server_response["columns"])
assert server_response_keys == sorted(output_keys_for(output_features))
assert len(data_df) == len(server_response["data"])
model_output, _ = model.predict(dataset=data_df)
model_output = model_output.to_dict("split")
assert model_output == server_response
# Cleanup
shutil.rmtree(output_dir, ignore_errors=True)
shutil.rmtree(audio_dest_folder, ignore_errors=True)
# test for single output feature
@pytest.mark.parametrize(
'test_case',
[
TestCase(
[numerical_feature()],
['loss', 'mean_squared_error', 'mean_absolute_error', 'r2']
),
TestCase(
[binary_feature()],
['loss', 'accuracy']
),
TestCase(
[category_feature()],
['loss', 'accuracy', 'hits_at_k']
),
TestCase(
[text_feature()],
['loss', 'token_accuracy', 'last_accuracy', 'edit_distance',
'perplexity']
)
]
)
def test_validation_metrics(test_case: TestCase, csv_filename: str):
# setup test scenarios
test_scenarios = []
for output_feature in test_case.output_features:
# single output feature capture feature specific metrics
of_name = output_feature[NAME]
def test_cache_dataset(use_cache_dir, use_split, tmpdir):
dataset_manager = PandasDatasetManager(backend=LocalTestBackend())
cache_dir = os.path.join(tmpdir, "cache") if use_cache_dir else None
manager = CacheManager(dataset_manager, cache_dir=cache_dir)
config = {
"input_features": [sequence_feature(reduce_output="sum")],
"output_features":
[category_feature(vocab_size=2, reduce_input="sum")],
"combiner": {
"type": "concat",
"output_size": 14
},
"preprocessing": {},
}
def touch(basename):
path = os.path.join(tmpdir, f"{basename}.csv")
Path(path).touch()
return path
dataset = training_set = test_set = validation_set = None
if not use_split:
dataset = touch("dataset")
cache_key = manager.get_cache_key(dataset, config)
else:
training_set = touch("train")
test_set = touch("test")
validation_set = touch("validation")
cache_key = manager.get_cache_key(training_set, config)
training_set_metadata = {
CHECKSUM: cache_key,
}
cache = manager.get_dataset_cache(config, dataset, training_set, test_set,
validation_set)
cache_map = cache.cache_map
assert len(cache_map) == 4
train_path = os.path.join(
cache_dir, alphanum(cache_key)) if use_cache_dir else os.path.join(
tmpdir, "dataset")
test_path = val_path = train_path
if use_split and not use_cache_dir:
train_path = os.path.join(tmpdir, "train")
test_path = os.path.join(tmpdir, "test")
val_path = os.path.join(tmpdir, "validation")
assert cache_map[META] == f"{train_path}.meta.json"
assert cache_map[TRAINING] == f"{train_path}.{TRAINING_PREPROC_FILE_NAME}"
assert cache_map[TEST] == f"{test_path}.test.hdf5"
assert cache_map[VALIDATION] == f"{val_path}.validation.hdf5"
for cache_path in cache_map.values():
assert not os.path.exists(cache_path)
training_set = pd.DataFrame()
test_set = pd.DataFrame()
validation_set = pd.DataFrame()
if use_cache_dir:
os.makedirs(cache_dir)
cache.put(training_set, test_set, validation_set, training_set_metadata)
for cache_path in cache_map.values():
assert os.path.exists(cache_path)
cache.delete()
for cache_path in cache_map.values():
assert not os.path.exists(cache_path)
def hyperopt_results():
"""
This function generates hyperopt results
"""
input_features = [
text_feature(name="utterance", cell_type="lstm", reduce_output="sum"),
category_feature(vocab_size=2, reduce_input="sum")]
output_features = [category_feature(vocab_size=2, reduce_input="sum")]
csv_filename = uuid.uuid4().hex[:10].upper() + '.csv'
rel_path = generate_data(input_features, output_features, csv_filename)
config = {
"input_features": input_features,
"output_features": output_features,
"combiner": {"type": "concat", "num_fc_layers": 2},
"training": {"epochs": 2, "learning_rate": 0.001}
}
output_feature_name = output_features[0]['name']
hyperopt_configs = {
"parameters": {
"training.learning_rate": {
"type": "float",
"low": 0.0001,
"high": 0.01,
"space": "log",
"steps": 3,
},
output_feature_name + ".fc_size": {
"type": "int",
"low": 32,
"high": 256,
"steps": 5
},
output_feature_name + ".num_fc_layers": {
'type': 'int',
'low': 1,
'high': 5,
'space': 'linear',
'steps': 4
}
},
"goal": "minimize",
'output_feature': output_feature_name,
'validation_metrics': 'loss',
'executor': {'type': 'serial'},
'sampler': {'type': 'random', 'num_samples': 2}
}
# add hyperopt parameter space to the config
config['hyperopt'] = hyperopt_configs
hyperopt(
config,
dataset=rel_path,
output_directory='results'
)
return os.path.abspath('results')
# test for single output feature
@pytest.mark.parametrize(
"test_case",
[
TestCase(
[numerical_feature()],
[
"root_mean_squared_percentage_error",
"mean_squared_error",
"mean_absolute_error",
"r2",
"root_mean_squared_error",
],
),
TestCase([binary_feature()], ["loss", "accuracy"]),
TestCase([category_feature()], ["loss", "accuracy", "hits_at_k"]),
# TODO(#1333): Re-enable.
# TestCase(
# [text_feature()],
# ['loss', 'token_accuracy', 'last_accuracy', 'edit_distance',
# 'perplexity']
# )
],
)
def test_validation_metrics(test_case: TestCase, csv_filename: str):
# setup test scenarios
test_scenarios = []
for output_feature in test_case.output_features:
# single output feature capture feature specific metrics
of_name = output_feature[COLUMN]
for metric in test_case.validation_metrics:
input_features = [category_feature(vocab_size=10)]
output_features = [
set_feature(vocab_size=3, loss={"class_weights": [0, 1, 2, 3]})
]
# Generate test data
rel_path = generate_data(input_features, output_features, csv_filename)
run_experiment(input_features, output_features, dataset=rel_path)
@pytest.mark.parametrize(
"output_features",
[
# baseline test case
[
category_feature(vocab_size=2, reduce_input="sum"),
sequence_feature(vocab_size=10, max_len=5),
number_feature(),
],
# use generator as decoder
[
category_feature(vocab_size=2, reduce_input="sum"),
sequence_feature(vocab_size=10, max_len=5, decoder="generator"),
number_feature(),
],
# Generator decoder and reduce_input = None
[
category_feature(vocab_size=2, reduce_input="sum"),
sequence_feature(max_len=5, decoder="generator",
reduce_input=None),
number_feature(normalization="minmax"),
numerical_feature(),
{'loss': {'type': 'mean_absolute_error'}}
),
#
# # binary feature
# (binary_feature(), binary_feature(), None),
#
# # Categorical feature
=======
(numerical_feature(normalization="minmax"), numerical_feature(), {"loss": {"type": "mean_squared_error"}}),
(numerical_feature(normalization="zscore"), numerical_feature(), {"loss": {"type": "mean_absolute_error"}}),
# binary feature
(binary_feature(), binary_feature(), None),
# Categorical feature
>>>>>>> upstream/master
(category_feature(), category_feature(), None),
(category_feature(), category_feature(), {"loss": {"type": "softmax_cross_entropy"}}),
# (
# category_feature(),
# category_feature(),
# {'loss': {
# 'type': 'sampled_softmax_cross_entropy',
# 'sampler': 'fixed_unigram',
# 'negative_samples': 10
# }
# }
# ),
# (
# category_feature(),
# category_feature(),
# {'loss': {
def run_hyperopt_executor(sampler,
executor,
csv_filename,
validate_output_feature=False,
validation_metric=None):
input_features = [
text_feature(name="utterance", cell_type="lstm", reduce_output="sum"),
category_feature(vocab_size=2, reduce_input="sum")
]
output_features = [category_feature(vocab_size=2, reduce_input="sum")]
rel_path = generate_data(input_features, output_features, csv_filename)
config = {
"input_features": input_features,
"output_features": output_features,
"combiner": {
"type": "concat",
"num_fc_layers": 2
},
"training": {
"epochs": 2,
"learning_rate": 0.001
},
"hyperopt": {
**HYPEROPT_CONFIG,
"executor": executor,
"sampler": sampler,
},
}
config = merge_with_defaults(config)
hyperopt_config = config["hyperopt"]
if validate_output_feature:
hyperopt_config['output_feature'] = output_features[0]['name']
if validation_metric:
hyperopt_config['validation_metric'] = validation_metric
update_hyperopt_params_with_defaults(hyperopt_config)
parameters = hyperopt_config["parameters"]
if sampler.get("search_alg", {}).get("type", "") == 'bohb':
# bohb does not support grid_search search space
del parameters['utterance.cell_type']
split = hyperopt_config["split"]
output_feature = hyperopt_config["output_feature"]
metric = hyperopt_config["metric"]
goal = hyperopt_config["goal"]
hyperopt_sampler = get_build_hyperopt_sampler(sampler["type"])(goal,
parameters,
**sampler)
hyperopt_executor = get_build_hyperopt_executor(executor["type"])(
hyperopt_sampler, output_feature, metric, split, **executor)
hyperopt_executor.execute(config, dataset=rel_path)
def test_hyperopt_run_hyperopt(csv_filename, ray_start_4_cpus):
input_features = [
text_feature(name="utterance", cell_type="lstm", reduce_output="sum"),
category_feature(vocab_size=2, reduce_input="sum")
]
output_features = [category_feature(vocab_size=2, reduce_input="sum")]
rel_path = generate_data(input_features, output_features, csv_filename)
config = {
"input_features": input_features,
"output_features": output_features,
"combiner": {
"type": "concat",
"num_fc_layers": 2
},
"training": {
"epochs": 2,
"learning_rate": 0.001
}
}
output_feature_name = output_features[0]['name']
hyperopt_configs = {
"parameters": {
"training.learning_rate": {
"space": "loguniform",
"lower": 0.001,
"upper": 0.1,
},
output_feature_name + ".fc_size": {
"space": "randint",
"lower": 32,
"upper": 256
},
output_feature_name + ".num_fc_layers": {
"space": "randint",
"lower": 2,
"upper": 6
}
},
"goal": "minimize",
'output_feature': output_feature_name,
'validation_metrics': 'loss',
'executor': {
'type': 'ray'
},
'sampler': {
'type': 'ray',
'num_samples': 2
}
}
# add hyperopt parameter space to the config
config['hyperopt'] = hyperopt_configs
hyperopt_results = hyperopt(config,
dataset=rel_path,
output_directory='results_hyperopt')
# check for return results
assert isinstance(hyperopt_results, list)
# check for existence of the hyperopt statistics file
assert os.path.isfile(
os.path.join('results_hyperopt', 'hyperopt_statistics.json'))
def test_torchscript(csv_filename, should_load_model):
#######
# Setup
#######
with tempfile.TemporaryDirectory() as tmpdir:
dir_path = tmpdir
data_csv_path = os.path.join(tmpdir, csv_filename)
image_dest_folder = os.path.join(tmpdir, "generated_images")
audio_dest_folder = os.path.join(tmpdir, "generated_audio")
# Single sequence input, single category output
input_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
vector_feature(),
image_feature(image_dest_folder),
audio_feature(audio_dest_folder),
timeseries_feature(),
date_feature(),
date_feature(),
h3_feature(),
set_feature(vocab_size=3),
bag_feature(vocab_size=3),
]
output_features = [
category_feature(vocab_size=3),
binary_feature(),
numerical_feature(),
set_feature(vocab_size=3),
vector_feature()
# TODO(#1333): Re-enable.
# sequence_feature(vocab_size=3),
# text_feature(vocab_size=3),
]
predictions_column_name = "{}_predictions".format(output_features[0]["name"])
# Generate test data
data_csv_path = generate_data(input_features, output_features, data_csv_path)
#############
# Train model
#############
backend = LocalTestBackend()
config = {"input_features": input_features, "output_features": output_features, "training": {"epochs": 2}}
ludwig_model = LudwigModel(config, backend=backend)
ludwig_model.train(
dataset=data_csv_path,
skip_save_training_description=True,
skip_save_training_statistics=True,
skip_save_model=True,
skip_save_progress=True,
skip_save_log=True,
skip_save_processed_input=True,
)
###################
# save Ludwig model
###################
ludwigmodel_path = os.path.join(dir_path, "ludwigmodel")
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
ludwig_model.save(ludwigmodel_path)
###################
# load Ludwig model
###################
if should_load_model:
ludwig_model = LudwigModel.load(ludwigmodel_path, backend=backend)
##############################
# collect weight tensors names
##############################
original_predictions_df, _ = ludwig_model.predict(dataset=data_csv_path)
original_weights = deepcopy(list(ludwig_model.model.parameters()))
#################
# save torchscript
#################
torchscript_path = os.path.join(dir_path, "torchscript")
shutil.rmtree(torchscript_path, ignore_errors=True)
ludwig_model.model.save_torchscript(torchscript_path)
###################################################
# load Ludwig model, obtain predictions and weights
###################################################
ludwig_model = LudwigModel.load(ludwigmodel_path, backend=backend)
loaded_prediction_df, _ = ludwig_model.predict(dataset=data_csv_path)
loaded_weights = deepcopy(list(ludwig_model.model.parameters()))
#####################################################
# restore torchscript, obtain predictions and weights
#####################################################
training_set_metadata_json_fp = os.path.join(ludwigmodel_path, TRAIN_SET_METADATA_FILE_NAME)
dataset, training_set_metadata = preprocess_for_prediction(
ludwig_model.config,
dataset=data_csv_path,
training_set_metadata=training_set_metadata_json_fp,
backend=backend,
)
restored_model = torch.jit.load(torchscript_path)
# Check the outputs for one of the features for correctness
# Here we choose the first output feature (categorical)
of_name = list(ludwig_model.model.output_features.keys())[0]
data_to_predict = {
name: torch.from_numpy(dataset.dataset[feature.proc_column])
for name, feature in ludwig_model.model.input_features.items()
}
# Get predictions from restored torchscript.
logits = restored_model(data_to_predict)
restored_predictions = torch.argmax(
output_feature_utils.get_output_feature_tensor(logits, of_name, "logits"), -1
)
restored_predictions = [training_set_metadata[of_name]["idx2str"][idx] for idx in restored_predictions]
restored_weights = deepcopy(list(restored_model.parameters()))
#########
# Cleanup
#########
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
shutil.rmtree(torchscript_path, ignore_errors=True)
###############################################
# Check if weights and predictions are the same
###############################################
# Check to weight values match the original model.
assert utils.is_all_close(original_weights, loaded_weights)
assert utils.is_all_close(original_weights, restored_weights)
# Check that predictions are identical to the original model.
assert np.all(original_predictions_df[predictions_column_name] == loaded_prediction_df[predictions_column_name])
assert np.all(original_predictions_df[predictions_column_name] == restored_predictions)
def test_gbm_model_save_reload_api(tmpdir, csv_filename, tmp_path):
torch.manual_seed(1)
random.seed(1)
np.random.seed(1)
input_features = [
binary_feature(),
number_feature(),
category_feature(vocab_size=3)
]
output_features = [category_feature(vocab_size=3)]
# Generate test data
data_csv_path = generate_data(input_features, output_features,
csv_filename)
#############
# Train tree model
#############
config = {
"model_type": "gbm",
"input_features": input_features,
"output_features": output_features,
TRAINER: {
"num_boost_round": 2
},
}
data_df = read_csv(data_csv_path)
splitter = get_splitter("random")
training_set, validation_set, test_set = splitter.split(
data_df, LocalTestBackend())
# create sub-directory to store results
results_dir = tmp_path / "results"
results_dir.mkdir()
# perform initial model training
backend = LocalTestBackend()
ludwig_model1 = LudwigModel(config, backend=backend)
_, _, output_dir = ludwig_model1.train(
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
output_directory="results", # results_dir
)
preds_1, _ = ludwig_model1.predict(dataset=validation_set)
def check_model_equal(ludwig_model2):
# Compare model predictions
preds_2, _ = ludwig_model2.predict(dataset=validation_set)
assert set(preds_1.keys()) == set(preds_2.keys())
for key in preds_1:
assert preds_1[key].dtype == preds_2[key].dtype, key
assert np.all(a == b
for a, b in zip(preds_1[key], preds_2[key])), key
# Compare model weights
for if_name in ludwig_model1.model.input_features:
if1 = ludwig_model1.model.input_features[if_name]
if2 = ludwig_model2.model.input_features[if_name]
for if1_w, if2_w in zip(if1.encoder_obj.parameters(),
if2.encoder_obj.parameters()):
assert torch.allclose(if1_w, if2_w)
tree1 = ludwig_model1.model.compiled_model
tree2 = ludwig_model2.model.compiled_model
for t1_w, t2_w in zip(tree1.parameters(), tree2.parameters()):
assert torch.allclose(t1_w, t2_w)
for of_name in ludwig_model1.model.output_features:
of1 = ludwig_model1.model.output_features[of_name]
of2 = ludwig_model2.model.output_features[of_name]
for of1_w, of2_w in zip(of1.decoder_obj.parameters(),
of2.decoder_obj.parameters()):
assert torch.allclose(of1_w, of2_w)
# Test saving and loading the model explicitly
ludwig_model1.save(tmpdir)
ludwig_model_loaded = LudwigModel.load(tmpdir, backend=backend)
check_model_equal(ludwig_model_loaded)
# Test loading the model from the experiment directory
ludwig_model_exp = LudwigModel.load(os.path.join(output_dir, "model"),
backend=backend)
check_model_equal(ludwig_model_exp)
def test_hyperopt_run_hyperopt(csv_filename, samplers):
input_features = [
text_feature(name="utterance", cell_type="lstm", reduce_output="sum"),
category_feature(vocab_size=2, reduce_input="sum"),
]
output_features = [category_feature(vocab_size=2, reduce_input="sum")]
rel_path = generate_data(input_features, output_features, csv_filename)
config = {
"input_features": input_features,
"output_features": output_features,
"combiner": {
"type": "concat",
"num_fc_layers": 2
},
TRAINER: {
"epochs": 2,
"learning_rate": 0.001
},
}
output_feature_name = output_features[0]["name"]
hyperopt_configs = {
"parameters": {
"trainer.learning_rate": {
"type": "float",
"low": 0.0001,
"high": 0.01,
"space": "log",
"steps": 3,
},
output_feature_name + ".fc_layers": {
"type":
"category",
"values": [
[{
"output_size": 64
}, {
"output_size": 32
}],
[{
"output_size": 64
}],
[{
"output_size": 32
}],
],
},
output_feature_name + ".output_size": {
"type": "int",
"low": 16,
"high": 36,
"steps": 5
},
output_feature_name + ".num_fc_layers": {
"type": "int",
"low": 1,
"high": 5,
"space": "linear",
"steps": 4
},
},
"goal": "minimize",
"output_feature": output_feature_name,
"validation_metrics": "loss",
"executor": {
"type": "serial"
},
"sampler": {
"type": samplers["type"],
"num_samples": 2
},
}
# add hyperopt parameter space to the config
config["hyperopt"] = hyperopt_configs
hyperopt_results = hyperopt(config,
dataset=rel_path,
output_directory="results_hyperopt")
# check for return results
assert isinstance(hyperopt_results, HyperoptResults)
# check for existence of the hyperopt statistics file
assert os.path.isfile(
os.path.join("results_hyperopt", "hyperopt_statistics.json"))
if os.path.isfile(
os.path.join("results_hyperopt", "hyperopt_statistics.json")):
os.remove(os.path.join("results_hyperopt", "hyperopt_statistics.json"))
def test_confidence_thresholding_2thresholds_3d_vis_api(csv_filename):
"""Ensure pdf and png figures can be saved via visualization API call.
:param csv_filename: csv fixture from tests.fixtures.filenames.csv_filename
:return: None
"""
input_features = [
text_feature(vocab_size=10, min_len=1, encoder="stacked_cnn"),
numerical_feature(),
category_feature(vocab_size=10, embedding_size=5),
set_feature(),
sequence_feature(vocab_size=10, max_len=10, encoder="embed"),
]
output_features = [
category_feature(vocab_size=2, reduce_input="sum"),
category_feature(vocab_size=2, reduce_input="sum"),
]
encoder = "parallel_cnn"
with TemporaryDirectory() as tmpvizdir:
# Generate test data
data_csv = generate_data(input_features, output_features,
os.path.join(tmpvizdir, csv_filename))
input_features[0]["encoder"] = encoder
model = run_api_experiment(input_features, output_features)
test_df, train_df, val_df = obtain_df_splits(data_csv)
_, _, output_dir = model.train(training_set=train_df,
validation_set=val_df,
output_directory=os.path.join(
tmpvizdir, "results"))
test_stats, predictions, _ = model.evaluate(
dataset=test_df,
collect_predictions=True,
output_directory=output_dir)
output_feature_name1 = output_features[0]["name"]
output_feature_name2 = output_features[1]["name"]
ground_truth_metadata = model.training_set_metadata
feature1_cols = [
f"{output_feature_name1}_probabilities_{label}"
for label in ground_truth_metadata[output_feature_name1]["idx2str"]
]
feature2_cols = [
f"{output_feature_name2}_probabilities_{label}"
for label in ground_truth_metadata[output_feature_name2]["idx2str"]
]
# probabilities need to be list of lists containing each row data from the
# probability columns ref: https://ludwig-ai.github.io/ludwig-docs/api/#test - Return
probability1 = predictions.loc[:, feature1_cols].values
probability2 = predictions.loc[:, feature2_cols].values
target_predictions1 = test_df[output_feature_name1]
target_predictions2 = test_df[output_feature_name2]
ground_truth1 = np.asarray([
ground_truth_metadata[output_feature_name1]["str2idx"][prediction]
for prediction in target_predictions1
])
ground_truth2 = np.asarray([
ground_truth_metadata[output_feature_name2]["str2idx"][prediction]
for prediction in target_predictions2
])
viz_outputs = ("pdf", "png")
for viz_output in viz_outputs:
vis_output_pattern_pdf = os.path.join(output_dir,
f"*.{viz_output}")
visualize.confidence_thresholding_2thresholds_3d(
[probability1, probability2],
[ground_truth1, ground_truth2],
model.training_set_metadata,
[output_feature_name1, output_feature_name2],
labels_limit=0,
output_directory=output_dir,
file_format=viz_output,
)
figure_cnt = glob.glob(vis_output_pattern_pdf)
assert 1 == len(figure_cnt)
@pytest.mark.parametrize(
"input_features,output_features",
[
(
[
number_feature(num_layers=2,
encoder="dense",
preprocessing={"normalization": "zscore"})
],
[number_feature()],
),
([image_feature(IMAGE_DIR, encoder="stacked_cnn")], [number_feature()
]),
([image_feature(IMAGE_DIR, encoder="resnet")], [category_feature()]),
(
[category_feature(representation="dense")],
[
number_feature(decoder="regressor",
loss={"type": "mean_squared_error"},
num_fc_layers=5)
],
),
([date_feature()], [binary_feature()]),
([sequence_feature(encoder="parallel_cnn", cell_type="gru")
], [binary_feature()]),
([set_feature()], [set_feature()]),
],
)
def test_regularizers(
def test_savedmodel(csv_filename):
#######
# Setup
#######
dir_path = os.path.dirname(csv_filename)
# Single sequence input, single category output
sf = sequence_feature()
sf['encoder'] = 'parallel_cnn'
input_features = [sf]
input_feature_name = input_features[0]['name']
input_feature_tensor_name = '{}/{}_placeholder:0'.format(
input_feature_name, input_feature_name)
output_features = [category_feature(vocab_size=2)]
output_feature_name = output_features[0]['name']
output_feature_tensor_name = '{}/predictions_{}/predictions_{}:0'.format(
output_feature_name, output_feature_name, output_feature_name)
predictions_column_name = '{}_predictions'.format(output_feature_name)
weight_tensor_name = '{}/fc_0/weights:0'.format(input_feature_name)
# Generate test data
data_csv_path = generate_data(input_features, output_features,
csv_filename)
#############
# Train model
#############
model_definition = {
'input_features': input_features,
'output_features': output_features,
'training': {
'epochs': 2
}
}
ludwig_model = LudwigModel(model_definition)
ludwig_model.train(
data_csv=data_csv_path,
skip_save_training_description=True,
skip_save_training_statistics=True,
skip_save_model=True,
skip_save_progress=True,
skip_save_log=True,
skip_save_processed_input=True,
)
original_predictions_df = ludwig_model.predict(data_csv=data_csv_path)
###################
# save Ludwig model
###################
ludwigmodel_path = os.path.join(dir_path, 'ludwigmodel')
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
ludwig_model.save(ludwigmodel_path)
#################
# save savedmodel
#################
savedmodel_path = os.path.join(dir_path, 'savedmodel')
shutil.rmtree(savedmodel_path, ignore_errors=True)
ludwig_model.model.save_savedmodel(savedmodel_path)
##############################
# collect weight tensors names
##############################
with ludwig_model.model.session as sess:
all_variables = tf.compat.v1.trainable_variables()
all_variables_names = [v.name for v in all_variables]
ludwig_model.close()
###################################################
# load Ludwig model, obtain predictions and weights
###################################################
ludwig_model = LudwigModel.load(ludwigmodel_path)
ludwig_prediction_df = ludwig_model.predict(data_csv=data_csv_path)
ludwig_weights = ludwig_model.model.collect_weights(all_variables_names)
ludwig_model.close()
#################################################
# load savedmodel, obtain predictions and weights
#################################################
train_set_metadata_json_fp = os.path.join(ludwigmodel_path,
TRAIN_SET_METADATA_FILE_NAME)
dataset, train_set_metadata = preprocess_for_prediction(
ludwigmodel_path,
split=FULL,
data_csv=data_csv_path,
train_set_metadata=train_set_metadata_json_fp,
evaluate_performance=False)
with tf.compat.v1.Session() as sess:
tf.saved_model.loader.load(sess, [tf.saved_model.SERVING],
savedmodel_path)
predictions = sess.run(output_feature_tensor_name,
feed_dict={
input_feature_tensor_name:
dataset.get(input_feature_name),
})
savedmodel_prediction_df = pd.DataFrame(
data=[
train_set_metadata[output_feature_name]["idx2str"][p]
for p in predictions
],
columns=[predictions_column_name])
savedmodel_weights = sess.run({n: n for n in all_variables_names})
#########
# Cleanup
#########
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
shutil.rmtree(savedmodel_path, ignore_errors=True)
###############################################
# Check if weights and predictions are the same
###############################################
for var in all_variables_names:
print("Are the weights in {} identical?".format(var),
np.all(ludwig_weights[var] == savedmodel_weights[var]))
print(
"Are loaded model predictions identical to original ones?",
np.all(
original_predictions_df[predictions_column_name] == \
ludwig_prediction_df[predictions_column_name]
)
)
print(
"Are savedmodel predictions identical to loaded model?",
np.all(
ludwig_prediction_df[predictions_column_name] == \
savedmodel_prediction_df[predictions_column_name]
)
)
for var in all_variables_names:
assert np.all(ludwig_weights[var] == savedmodel_weights[var])
assert np.all(
original_predictions_df[predictions_column_name] == \
ludwig_prediction_df[predictions_column_name]
)
assert np.all(
ludwig_prediction_df[predictions_column_name] == \
savedmodel_prediction_df[predictions_column_name]
)
def test_config_bad_combiner_types_enums():
config = {
"input_features": [
category_feature(vocab_size=2, reduce_input="sum"),
number_feature(),
],
"output_features": [binary_feature(weight_regularization=None)],
"combiner": {
"type": "concat",
"weights_initializer": "zeros"
},
}
# config is valid at this point
validate_config(config)
# Test weights initializer:
config["combiner"]["weights_initializer"] = {"test": "fail"}
with pytest.raises(ValidationError, match=r"{'test': 'fail'} is not of*"):
validate_config(config)
config["combiner"]["weights_initializer"] = "fail"
with pytest.raises(ValidationError, match=r"'fail' is not of*"):
validate_config(config)
config["combiner"]["weights_initializer"] = {}
with pytest.raises(ValidationError, match=r"Failed validating 'type'"):
validate_config(config)
config["combiner"]["weights_initializer"] = {"type": "fail"}
with pytest.raises(ValidationError, match=r"'fail' is not one of*"):
validate_config(config)
config["combiner"]["weights_initializer"] = {"type": "normal", "stddev": 0}
validate_config(config)
# Test bias initializer:
del config["combiner"]["weights_initializer"]
config["combiner"]["bias_initializer"] = "kaiming_uniform"
validate_config(config)
config["combiner"]["bias_initializer"] = "fail"
with pytest.raises(ValidationError, match=r"'fail' is not of*"):
validate_config(config)
config["combiner"]["bias_initializer"] = {}
with pytest.raises(ValidationError, match=r"Failed validating 'type'"):
validate_config(config)
config["combiner"]["bias_initializer"] = {"type": "fail"}
with pytest.raises(ValidationError, match=r"'fail' is not one of*"):
validate_config(config)
config["combiner"]["bias_initializer"] = {"type": "zeros", "stddev": 0}
validate_config(config)
# Test norm:
del config["combiner"]["bias_initializer"]
config["combiner"]["norm"] = "batch"
validate_config(config)
config["combiner"]["norm"] = "fail"
with pytest.raises(ValidationError, match=r"'fail' is not one of*"):
validate_config(config)
# Test activation:
del config["combiner"]["norm"]
config["combiner"]["activation"] = "relu"
validate_config(config)
config["combiner"]["activation"] = 123
with pytest.raises(ValidationError, match=r"123 is not of type*"):
validate_config(config)
# Test reduce_output:
del config["combiner"]["activation"]
config2 = {**config}
config2["combiner"]["type"] = "tabtransformer"
config2["combiner"]["reduce_output"] = "sum"
validate_config(config)
config2["combiner"]["reduce_output"] = "fail"
with pytest.raises(ValidationError, match=r"'fail' is not one of*"):
validate_config(config2)
# Test reduce_output = None:
config2["combiner"]["reduce_output"] = None
validate_config(config2)
def t_neuropod(csv_filename):
#######
# Setup
#######
dir_path = os.path.dirname(csv_filename)
image_dest_folder = os.path.join(os.getcwd(), 'generated_images')
audio_dest_folder = os.path.join(os.getcwd(), 'generated_audio')
input_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
vector_feature(),
image_feature(image_dest_folder),
audio_feature(audio_dest_folder),
timeseries_feature(),
date_feature(),
h3_feature(),
set_feature(vocab_size=3),
bag_feature(vocab_size=3),
]
output_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
set_feature(vocab_size=3),
vector_feature()
]
# Generate test data
data_csv_path = generate_data(input_features, output_features,
csv_filename)
#############
# Train model
#############
model_definition = {
'input_features': input_features,
'output_features': output_features,
'training': {
'epochs': 2
}
}
ludwig_model = LudwigModel(model_definition)
ludwig_model.train(
data_csv=data_csv_path,
skip_save_training_description=True,
skip_save_training_statistics=True,
skip_save_model=True,
skip_save_progress=True,
skip_save_log=True,
skip_save_processed_input=True,
)
original_predictions_df = ludwig_model.predict(data_csv=data_csv_path)
###################
# save Ludwig model
###################
ludwigmodel_path = os.path.join(dir_path, 'ludwigmodel')
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
ludwig_model.save(ludwigmodel_path)
################
# build neuropod
################
neuropod_path = os.path.join(dir_path, 'neuropod')
export_neuropod(ludwigmodel_path, neuropod_path=neuropod_path)
########################
# predict using neuropod
########################
data_df = pd.read_csv(data_csv_path)
if_dict = {
input_feature['name']: np.expand_dims(
np.array([str(x) for x in data_df[input_feature['name']].tolist()],
dtype='str'), 1)
for input_feature in input_features
}
from neuropod.loader import load_neuropod
neuropod_model = load_neuropod(neuropod_path)
preds = neuropod_model.infer(if_dict)
for key in preds:
preds[key] = np.squeeze(preds[key])
#########
# cleanup
#########
# Delete the temporary data created
for path in [
ludwigmodel_path, neuropod_path, image_dest_folder,
audio_dest_folder
]:
if os.path.exists(path):
if os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path, ignore_errors=True)
########
# checks
########
for output_feature in output_features:
output_feature_name = output_feature['name']
output_feature_type = output_feature['type']
if (output_feature_name + "_predictions" in preds
and output_feature_name + "_predictions"
in original_predictions_df):
neuropod_pred = preds[output_feature_name +
"_predictions"].tolist()
if output_feature_type == BINARY:
neuropod_pred = list(map(lambda x: str2bool(x), neuropod_pred))
if output_feature_type in {SEQUENCE, TEXT, SET}:
neuropod_pred = list(map(lambda x: x.split(), neuropod_pred))
original_pred = original_predictions_df[output_feature_name +
"_predictions"].tolist()
assert neuropod_pred == original_pred
if (output_feature_name + "_probability" in preds
and output_feature_name + "_probability"
in original_predictions_df):
neuropod_prob = preds[output_feature_name +
"_probability"].tolist()
if output_feature_type in {SEQUENCE, TEXT, SET}:
neuropod_prob = list(
map(lambda x: [float(n) for n in x.split()],
neuropod_prob))
if any(isinstance(el, list) for el in neuropod_prob):
neuropod_prob = np.array(
list(itertools.zip_longest(*neuropod_prob, fillvalue=0))).T
original_prob = original_predictions_df[output_feature_name +
"_probability"].tolist()
if any(isinstance(el, list) for el in original_prob):
original_prob = np.array(
list(itertools.zip_longest(*original_prob, fillvalue=0))).T
assert np.isclose(neuropod_prob, original_prob).all()
if (output_feature_name + "_probabilities" in preds
and output_feature_name + "_probabilities"
in original_predictions_df):
neuropod_prob = preds[output_feature_name +
"_probabilities"].tolist()
original_prob = original_predictions_df[output_feature_name +
"_probabilities"].tolist()
assert np.isclose(neuropod_prob, original_prob).all()
def test_hyperopt_run_hyperopt(csv_filename, samplers):
input_features = [
text_feature(name="utterance", cell_type="lstm", reduce_output="sum"),
category_feature(vocab_size=2, reduce_input="sum")
]
output_features = [category_feature(vocab_size=2, reduce_input="sum")]
rel_path = generate_data(input_features, output_features, csv_filename)
config = {
"input_features": input_features,
"output_features": output_features,
"combiner": {
"type": "concat",
"num_fc_layers": 2
},
"training": {
"epochs": 2,
"learning_rate": 0.001
}
}
output_feature_name = output_features[0]['name']
hyperopt_configs = {
"parameters": {
"training.learning_rate": {
"type": "float",
"low": 0.0001,
"high": 0.01,
"space": "log",
"steps": 3,
},
output_feature_name + ".fc_layers": {
'type':
'category',
'values': [[{
'fc_size': 512
}, {
'fc_size': 256
}], [{
'fc_size': 512
}], [{
'fc_size': 256
}]]
},
output_feature_name + ".fc_size": {
"type": "int",
"low": 32,
"high": 256,
"steps": 5
},
output_feature_name + ".num_fc_layers": {
'type': 'int',
'low': 1,
'high': 5,
'space': 'linear',
'steps': 4
}
},
"goal": "minimize",
'output_feature': output_feature_name,
'validation_metrics': 'loss',
'executor': {
'type': 'serial'
},
'sampler': {
'type': samplers["type"],
'num_samples': 2
}
}
# add hyperopt parameter space to the config
config['hyperopt'] = hyperopt_configs
hyperopt_results = hyperopt(config,
dataset=rel_path,
output_directory='results_hyperopt')
# check for return results
assert isinstance(hyperopt_results, list)
# check for existence of the hyperopt statistics file
assert os.path.isfile(
os.path.join('results_hyperopt', 'hyperopt_statistics.json'))
if os.path.isfile(
os.path.join('results_hyperopt', 'hyperopt_statistics.json')):
os.remove(os.path.join('results_hyperopt', 'hyperopt_statistics.json'))
def test_experiment_image_dataset(train_format, train_in_memory, test_format,
test_in_memory, tmpdir):
# Image Inputs
image_dest_folder = os.path.join(tmpdir, "generated_images")
input_features = [
image_feature(
folder=image_dest_folder,
encoder="stacked_cnn",
preprocessing={
"in_memory": True,
"height": 12,
"width": 12,
"num_channels": 3,
"num_processes": 5
},
output_size=16,
num_filters=8,
),
]
output_features = [
category_feature(vocab_size=2, reduce_input="sum"),
]
config = {
"input_features": input_features,
"output_features": output_features,
"combiner": {
"type": "concat",
"output_size": 14
},
"preprocessing": {},
TRAINER: {
"epochs": 2
},
}
# create temporary name for train and test data sets
train_csv_filename = os.path.join(
tmpdir, "train_" + uuid.uuid4().hex[:10].upper() + ".csv")
test_csv_filename = os.path.join(
tmpdir, "test_" + uuid.uuid4().hex[:10].upper() + ".csv")
# setup training data format to test
train_data = generate_data(input_features, output_features,
train_csv_filename)
config["input_features"][0]["preprocessing"]["in_memory"] = train_in_memory
training_set_metadata = None
backend = LocalTestBackend()
if train_format == "hdf5":
# hdf5 format
train_set, _, _, training_set_metadata = preprocess_for_training(
config,
dataset=train_data,
backend=backend,
)
train_dataset_to_use = train_set.data_hdf5_fp
else:
train_dataset_to_use = create_data_set_to_use(train_format, train_data)
# define Ludwig model
model = LudwigModel(
config=config,
backend=backend,
)
model.train(dataset=train_dataset_to_use,
training_set_metadata=training_set_metadata)
model.config["input_features"][0]["preprocessing"][
"in_memory"] = test_in_memory
# setup test data format to test
test_data = generate_data(input_features, output_features,
test_csv_filename)
if test_format == "hdf5":
# hdf5 format
# create hdf5 data set
_, test_set, _, training_set_metadata_for_test = preprocess_for_training(
model.config,
dataset=test_data,
backend=backend,
)
test_dataset_to_use = test_set.data_hdf5_fp
else:
test_dataset_to_use = create_data_set_to_use(test_format, test_data)
# run functions with the specified data format
model.evaluate(dataset=test_dataset_to_use)
model.predict(dataset=test_dataset_to_use)
# input feature
[
number_feature(normalization="zscore"),
number_feature(normalization="zscore")
],
# output feature
[binary_feature()],
),
FeaturesToUse(
# input feature
[
number_feature(normalization="zscore"),
number_feature(normalization="zscore")
],
# output feature
[category_feature(vocab_size=4, reduce_input="sum")],
),
FeaturesToUse(
# input feature
# [sequence_feature(min_len=5, max_len=10, encoder="rnn", cell_type="lstm", reduce_output=None)],
[
number_feature(normalization="zscore"),
number_feature(normalization="zscore")
],
# output feature
[
sequence_feature(min_len=5,
max_len=10,
decoder="generator",
cell_type="lstm",
attention="bahdanau",
def test_mlflow_callback(tmpdir):
epochs = 2
batch_size = 8
num_examples = 32
input_features = [sequence_feature(reduce_output='sum')]
output_features = [category_feature(vocab_size=2, reduce_input='sum')]
config = {
'input_features': input_features,
'output_features': output_features,
'combiner': {
'type': 'concat',
'fc_size': 14
},
'training': {
'epochs': epochs,
'batch_size': batch_size
},
}
data_csv = generate_data(input_features,
output_features,
os.path.join(tmpdir, 'train.csv'),
num_examples=num_examples)
val_csv = shutil.copyfile(data_csv, os.path.join(tmpdir, 'validation.csv'))
test_csv = shutil.copyfile(data_csv, os.path.join(tmpdir, 'test.csv'))
mlflow_uri = f'file://{tmpdir}/mlruns'
mlflow.set_tracking_uri(mlflow_uri)
client = MlflowClient(tracking_uri=mlflow_uri)
exp_name = 'mlflow_test'
callback = MlflowCallback()
model = LudwigModel(config, callbacks=[callback])
model.train(training_set=data_csv,
validation_set=val_csv,
test_set=test_csv,
experiment_name=exp_name)
expected_df, _ = model.predict(test_csv)
# Check mlflow artifacts
assert callback.experiment_id is not None
assert callback.run is not None
experiment = mlflow.get_experiment_by_name(exp_name)
assert experiment.experiment_id == callback.experiment_id
df = mlflow.search_runs([experiment.experiment_id])
assert len(df) == 1
run_id = df.run_id[0]
assert run_id == callback.run.info.run_id
artifacts = [
f.path for f in client.list_artifacts(callback.run.info.run_id, "")
]
local_dir = f'{tmpdir}/local_artifacts'
os.makedirs(local_dir)
assert 'config.yaml' in artifacts
local_config_path = client.download_artifacts(callback.run.info.run_id,
"config.yaml", local_dir)
with open(local_config_path, 'r') as f:
config_artifact = yaml.safe_load(f)
assert config_artifact == config
model_path = f'runs:/{callback.run.info.run_id}/model'
loaded_model = mlflow.pyfunc.load_model(model_path)
assert 'ludwig' in loaded_model.metadata.flavors
flavor = loaded_model.metadata.flavors['ludwig']
def compare_features(key):
assert len(model.config[key]) == len(flavor['ludwig_schema'][key])
for feature, schema_feature in zip(model.config[key],
flavor['ludwig_schema'][key]):
assert feature['name'] == schema_feature['name']
assert feature['type'] == schema_feature['type']
compare_features('input_features')
compare_features('output_features')
test_df = pd.read_csv(test_csv)
pred_df = loaded_model.predict(test_df)
assert (pred_df.equals(expected_df))
def test_server_integration(csv_filename):
# Image Inputs
image_dest_folder = os.path.join(os.getcwd(), 'generated_images')
# Resnet encoder
input_features = [
image_feature(folder=image_dest_folder,
preprocessing={
'in_memory': True,
'height': 8,
'width': 8,
'num_channels': 3
},
fc_size=16,
num_filters=8),
text_feature(encoder='embed', min_len=1),
numerical_feature(normalization='zscore')
]
output_features = [category_feature(vocab_size=2), numerical_feature()]
rel_path = generate_data(input_features, output_features, csv_filename)
model, output_dir = train_model(input_features,
output_features,
data_csv=rel_path)
app = server(model)
client = TestClient(app)
response = client.get('/')
assert response.status_code == 200
response = client.post('/predict')
assert response.json() == ALL_FEATURES_PRESENT_ERROR
data_df = read_csv(rel_path)
# One-off prediction
first_entry = data_df.T.to_dict()[0]
data, files = convert_to_form(first_entry)
server_response = client.post('/predict', data=data, files=files)
server_response = server_response.json()
server_response_keys = sorted(list(server_response.keys()))
assert server_response_keys == sorted(output_keys_for(output_features))
model_output, _ = model.predict(dataset=[first_entry], data_format=dict)
model_output = model_output.to_dict('records')[0]
assert model_output == server_response
# Batch prediction
assert len(data_df) > 1
files = convert_to_batch_form(data_df)
server_response = client.post('/batch_predict', files=files)
server_response = server_response.json()
server_response_keys = sorted(server_response['columns'])
assert server_response_keys == sorted(output_keys_for(output_features))
assert len(data_df) == len(server_response['data'])
model_output, _ = model.predict(dataset=data_df)
model_output = model_output.to_dict('split')
assert model_output == server_response
# Cleanup
shutil.rmtree(output_dir, ignore_errors=True)
shutil.rmtree(image_dest_folder, ignore_errors=True)
def test_savedmodel(csv_filename):
#######
# Setup
#######
dir_path = os.path.dirname(csv_filename)
# Single sequence input, single category output
sf = sequence_feature()
sf['encoder'] = 'parallel_cnn'
input_features = [sf]
output_features = [category_feature(vocab_size=2)]
predictions_column_name = '{}_predictions'.format(
output_features[0]['name'])
# Generate test data
data_csv_path = generate_data(input_features, output_features,
csv_filename)
#############
# Train model
#############
model_definition = {
'input_features': input_features,
'output_features': output_features,
'training': {
'epochs': 2
}
}
ludwig_model = LudwigModel(model_definition)
ludwig_model.train(
data_csv=data_csv_path,
skip_save_training_description=True,
skip_save_training_statistics=True,
skip_save_model=True,
skip_save_progress=True,
skip_save_log=True,
skip_save_processed_input=True,
)
###################
# save Ludwig model
###################
ludwigmodel_path = os.path.join(dir_path, 'ludwigmodel')
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
ludwig_model.save(ludwigmodel_path)
#################
# save savedmodel
#################
savedmodel_path = os.path.join(dir_path, 'savedmodel')
shutil.rmtree(savedmodel_path, ignore_errors=True)
ludwig_model.model.save_savedmodel(savedmodel_path)
##############################
# collect weight tensors names
##############################
original_predictions_df = ludwig_model.predict(data_csv=data_csv_path)
original_weights = deepcopy(ludwig_model.model.model.trainable_variables)
ludwig_model.close()
###################################################
# load Ludwig model, obtain predictions and weights
###################################################
ludwig_model = LudwigModel.load(ludwigmodel_path)
loaded_prediction_df = ludwig_model.predict(data_csv=data_csv_path)
loaded_weights = deepcopy(ludwig_model.model.model.trainable_variables)
#################################################
# restore savedmodel, obtain predictions and weights
#################################################
train_set_metadata_json_fp = os.path.join(ludwigmodel_path,
TRAIN_SET_METADATA_FILE_NAME)
dataset, train_set_metadata = preprocess_for_prediction(
ludwigmodel_path,
split=FULL,
data_csv=data_csv_path,
train_set_metadata=train_set_metadata_json_fp,
evaluate_performance=False)
restored_model = tf.saved_model.load(savedmodel_path)
if_name = list(ludwig_model.model.model.input_features.keys())[0]
of_name = list(ludwig_model.model.model.output_features.keys())[0]
data_to_predict = {
if_name: tf.convert_to_tensor(dataset.dataset[if_name], dtype=tf.int32)
}
logits = restored_model(data_to_predict, False, None)
restored_predictions = tf.argmax(logits[of_name]['logits'],
-1,
name='predictions_{}'.format(of_name))
restored_predictions = tf.map_fn(
lambda idx: train_set_metadata[of_name]['idx2str'][idx],
restored_predictions,
dtype=tf.string)
restored_weights = deepcopy(restored_model.trainable_variables)
#########
# Cleanup
#########
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
shutil.rmtree(savedmodel_path, ignore_errors=True)
###############################################
# Check if weights and predictions are the same
###############################################
# check for same number of weights as original model
assert len(original_weights) == len(loaded_weights)
assert len(original_weights) == len(restored_weights)
# check to ensure weight valuess match the original model
loaded_weights_match = np.all([
np.all(
np.isclose(original_weights[i].numpy(), loaded_weights[i].numpy()))
for i in range(len(original_weights))
])
restored_weights_match = np.all([
np.all(
np.isclose(original_weights[i].numpy(),
restored_weights[i].numpy()))
for i in range(len(original_weights))
])
assert loaded_weights_match and restored_weights_match
# Are predictions identical to original ones?
loaded_predictions_match = np.all(
original_predictions_df[predictions_column_name] ==
loaded_prediction_df[predictions_column_name])
restored_predictions_match = np.all(
original_predictions_df[predictions_column_name] ==
restored_predictions.numpy().astype('str'))
assert loaded_predictions_match and restored_predictions_match
def test_model_save_reload_api(csv_filename, tmp_path):
torch.manual_seed(1)
random.seed(1)
np.random.seed(1)
image_dest_folder = os.path.join(os.getcwd(), "generated_images")
audio_dest_folder = os.path.join(os.getcwd(), "generated_audio")
input_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3,
encoder="rnn",
cell_type="lstm",
num_layers=2,
bidirections=True),
vector_feature(),
image_feature(image_dest_folder),
audio_feature(audio_dest_folder, encoder="stacked_cnn"),
timeseries_feature(encoder="parallel_cnn"),
sequence_feature(vocab_size=3, encoder="stacked_parallel_cnn"),
date_feature(),
h3_feature(),
set_feature(vocab_size=3),
bag_feature(vocab_size=3),
]
output_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
# TODO(#1333): Reintroduce sequence and text after sequence output feature.
# sequence_feature(vocab_size=3),
# text_feature(vocab_size=3),
set_feature(vocab_size=3),
vector_feature(),
]
# Generate test data
data_csv_path = generate_data(input_features, output_features,
csv_filename)
#############
# Train model
#############
config = {
"input_features": input_features,
"output_features": output_features,
"training": {
"epochs": 2
}
}
data_df = read_csv(data_csv_path)
data_df[SPLIT] = get_split(data_df)
training_set, test_set, validation_set = split_dataset_ttv(data_df, SPLIT)
training_set = pd.DataFrame(training_set)
validation_set = pd.DataFrame(validation_set)
test_set = pd.DataFrame(test_set)
# create sub-directory to store results
results_dir = tmp_path / "results"
results_dir.mkdir()
# perform initial model training
backend = LocalTestBackend()
ludwig_model1 = LudwigModel(config, backend=backend)
_, _, output_dir = ludwig_model1.train(
training_set=training_set,
validation_set=validation_set,
test_set=test_set,
output_directory="results", # results_dir
)
preds_1, _ = ludwig_model1.predict(dataset=validation_set)
def check_model_equal(ludwig_model2):
# Compare model predictions
preds_2, _ = ludwig_model2.predict(dataset=validation_set)
assert set(preds_1.keys()) == set(preds_2.keys())
for key in preds_1:
assert preds_1[key].dtype == preds_2[key].dtype, key
assert np.all(a == b
for a, b in zip(preds_1[key], preds_2[key])), key
# assert preds_2[key].dtype == preds_3[key].dtype, key
# assert list(preds_2[key]) == list(preds_3[key]), key
# Compare model weights
# this has to be done after predicts because of TF2 lazy restoration
for if_name in ludwig_model1.model.input_features:
if1 = ludwig_model1.model.input_features[if_name]
if2 = ludwig_model2.model.input_features[if_name]
for if1_w, if2_w in zip(if1.encoder_obj.parameters(),
if2.encoder_obj.parameters()):
assert torch.allclose(if1_w, if2_w)
c1 = ludwig_model1.model.combiner
c2 = ludwig_model2.model.combiner
for c1_w, c2_w in zip(c1.parameters(), c2.parameters()):
assert torch.allclose(c1_w, c2_w)
for of_name in ludwig_model1.model.output_features:
of1 = ludwig_model1.model.output_features[of_name]
of2 = ludwig_model2.model.output_features[of_name]
for of1_w, of2_w in zip(of1.decoder_obj.parameters(),
of2.decoder_obj.parameters()):
assert torch.allclose(of1_w, of2_w)
# Test saving and loading the model explicitly
with tempfile.TemporaryDirectory() as tmpdir:
ludwig_model1.save(tmpdir)
ludwig_model_loaded = LudwigModel.load(tmpdir, backend=backend)
check_model_equal(ludwig_model_loaded)
# Test loading the model from the experiment directory
ludwig_model_exp = LudwigModel.load(os.path.join(output_dir, "model"),
backend=backend)
check_model_equal(ludwig_model_exp)
def test_experiment_image_dataset(
train_format, train_in_memory,
test_format, test_in_memory
):
# primary focus of this test is to determine if exceptions are
# raised for different data set formats and in_memory setting
# Image Inputs
image_dest_folder = os.path.join(os.getcwd(), 'generated_images')
input_features = [
image_feature(
folder=image_dest_folder,
encoder='stacked_cnn',
preprocessing={
'in_memory': True,
'height': 12,
'width': 12,
'num_channels': 3,
'num_processes': 5
},
fc_size=16,
num_filters=8
),
]
output_features = [
category_feature(vocab_size=2, reduce_input='sum'),
]
model_definition = {
'input_features': input_features,
'output_features': output_features,
'combiner': {
'type': 'concat',
'fc_size': 14
},
'preprocessing': {},
'training': {'epochs': 2}
}
# create temporary name for train and test data sets
train_csv_filename = 'train_' + uuid.uuid4().hex[:10].upper() + '.csv'
test_csv_filename = 'test_' + uuid.uuid4().hex[:10].upper() + '.csv'
# setup training data format to test
train_data = generate_data(input_features, output_features,
train_csv_filename)
model_definition['input_features'][0]['preprocessing']['in_memory'] \
= train_in_memory
training_set_metadata = None
if train_format == 'csv':
train_dataset_to_use = train_data
elif train_format in {'df', 'dict'}:
train_dataset_to_use = pd.read_csv(train_data)
if train_format == 'dict':
train_dataset_to_use = train_dataset_to_use.to_dict(orient='list')
else:
# hdf5 format
train_set, _, _, training_set_metadata = preprocess_for_training(
model_definition,
dataset=train_data
)
train_dataset_to_use = train_set.data_hdf5_fp
# define Ludwig model
model = LudwigModel(
model_definition=model_definition,
random_seed=default_random_seed
)
model.train(
dataset=train_dataset_to_use,
training_set_metadata=training_set_metadata
)
model.model_definition['input_features'][0]['preprocessing']['in_memory'] \
= test_in_memory
# setup test data format to test
test_data = generate_data(input_features, output_features,
test_csv_filename)
if test_format == 'csv':
test_dataset_to_use = test_data
elif test_format in {'df', 'dict'}:
test_dataset_to_use = pd.read_csv(test_data)
if test_format == 'dict':
test_dataset_to_use = test_dataset_to_use.to_dict(orient='list')
else:
# hdf5 format
# create hdf5 data set
_, test_set, _, training_set_metadata_for_test = preprocess_for_training(
model.model_definition,
dataset=test_data
)
test_dataset_to_use = test_set.data_hdf5_fp
# run functions with the specified data format
model.evaluate(dataset=test_dataset_to_use)
model.predict(dataset=test_dataset_to_use)
# Delete the temporary data created
shutil.rmtree(image_dest_folder)
delete_temporary_data(train_csv_filename)
delete_temporary_data(test_csv_filename)
def test_cache_dataset(use_cache_dir, use_split, tmpdir):
dataset_manager = PandasDatasetManager(backend=LocalTestBackend())
cache_dir = os.path.join(tmpdir, 'cache') if use_cache_dir else None
manager = CacheManager(dataset_manager, cache_dir=cache_dir)
config = {
'input_features': [sequence_feature(reduce_output='sum')],
'output_features': [category_feature(vocab_size=2, reduce_input='sum')],
'combiner': {'type': 'concat', 'fc_size': 14},
'preprocessing': {},
}
def touch(basename):
path = os.path.join(tmpdir, f'{basename}.csv')
Path(path).touch()
return path
dataset = training_set = test_set = validation_set = None
if not use_split:
dataset = touch('dataset')
cache_key = manager.get_cache_key(dataset, config)
else:
training_set = touch('train')
test_set = touch('test')
validation_set = touch('validation')
cache_key = manager.get_cache_key(training_set, config)
training_set_metadata = {
CHECKSUM: cache_key,
}
cache = manager.get_dataset_cache(
config, dataset, training_set, test_set, validation_set
)
cache_map = cache.cache_map
assert len(cache_map) == 4
train_path = os.path.join(cache_dir, alphanum(cache_key)) if \
use_cache_dir else \
os.path.join(tmpdir, 'dataset')
test_path = val_path = train_path
if use_split and not use_cache_dir:
train_path = os.path.join(tmpdir, 'train')
test_path = os.path.join(tmpdir, 'test')
val_path = os.path.join(tmpdir, 'validation')
assert cache_map[META] == f'{train_path}.meta.json'
assert cache_map[TRAINING] == f'{train_path}.training.hdf5'
assert cache_map[TEST] == f'{test_path}.test.hdf5'
assert cache_map[VALIDATION] == f'{val_path}.validation.hdf5'
for cache_path in cache_map.values():
assert not os.path.exists(cache_path)
training_set = pd.DataFrame()
test_set = pd.DataFrame()
validation_set = pd.DataFrame()
if use_cache_dir:
os.makedirs(cache_dir)
cache.put(training_set, test_set, validation_set, training_set_metadata)
for cache_path in cache_map.values():
assert os.path.exists(cache_path)
cache.delete()
for cache_path in cache_map.values():
assert not os.path.exists(cache_path)
@contextlib.contextmanager
def graph_mode():
prev_mode = tf.config.experimental_functions_run_eagerly()
try:
tf.config.experimental_run_functions_eagerly(False)
yield
finally:
tf.config.experimental_run_functions_eagerly(prev_mode)
@pytest.mark.parametrize(
'output_features',
[
# baseline test case
[
category_feature(vocab_size=2, reduce_input='sum'),
sequence_feature(vocab_size=10, max_len=5),
numerical_feature()
],
# use generator as decoder
[
category_feature(vocab_size=2, reduce_input='sum'),
sequence_feature(vocab_size=10, max_len=5, decoder='generator'),
numerical_feature()
],
# Generator decoder and reduce_input = None
[
category_feature(vocab_size=2, reduce_input='sum'),
sequence_feature(max_len=5, decoder='generator',
def test_binary_predictions_with_number_dtype(tmpdir, backend,
distinct_values):
input_features = [
category_feature(vocab_size=3),
]
feature = binary_feature()
output_features = [
feature,
]
data_csv_path = generate_data(
input_features,
output_features,
os.path.join(tmpdir, "dataset.csv"),
num_examples=100,
)
data_df = pd.read_csv(data_csv_path)
# Optionally convert bool values to strings, e.g., {'Yes', 'No'}
false_value, true_value = distinct_values
data_df[feature[NAME]] = data_df[feature[NAME]].map(lambda x: true_value
if x else false_value)
data_df.to_csv(data_csv_path, index=False)
config = {
"input_features": input_features,
"output_features": output_features,
TRAINER: {
"epochs": 1
}
}
patch_args = (
"ludwig.features.binary_feature.BinaryOutputFeature.logits",
partial(random_binary_logits, num_predict_samples=len(data_df)),
)
preds_df, _ = predict_with_backend(tmpdir,
config,
data_csv_path,
backend,
patch_args=patch_args)
cols = set(preds_df.columns)
assert f"{feature[NAME]}_predictions" in cols
assert f"{feature[NAME]}_probabilities_False" in cols
assert f"{feature[NAME]}_probabilities_True" in cols
assert f"{feature[NAME]}_probability" in cols
for pred, prob_0, prob_1, prob in zip(
preds_df[f"{feature[NAME]}_predictions"],
preds_df[f"{feature[NAME]}_probabilities_False"],
preds_df[f"{feature[NAME]}_probabilities_True"],
preds_df[f"{feature[NAME]}_probability"],
):
assert isinstance(pred, bool)
if pred:
assert prob_1 == prob
else:
assert prob_0 == prob
assert np.allclose(prob_0, 1 - prob_1)
def test_kfold_cv():
num_folds = 3
# setup temporary directory to run test
with tempfile.TemporaryDirectory() as tmpdir:
training_data_fp = os.path.join(tmpdir, 'train.csv')
model_definition_fp = os.path.join(tmpdir, 'model_definition.yaml')
results_dir = os.path.join(tmpdir, 'results')
statistics_fp = os.path.join(results_dir,
'kfold_training_statistics.json')
indices_fp = os.path.join(results_dir, 'kfold_split_indices.json')
# generate synthetic data for the test
input_features = [
numerical_feature(normalization='zscore'),
numerical_feature(normalization='zscore')
]
output_features = [category_feature(vocab_size=2, reduce_input='sum')]
generate_data(input_features, output_features, training_data_fp)
# generate model definition file
model_definition = {
'input_features': input_features,
'output_features': output_features,
'combiner': {
'type': 'concat',
'fc_size': 14
},
'training': {
'epochs': 2
}
}
with open(model_definition_fp, 'w') as f:
yaml.dump(model_definition, f)
# run k-fold cv
kfold_cross_validate(k_fold=num_folds,
model_definition_file=model_definition_fp,
data_csv=training_data_fp,
output_directory=results_dir,
logging_level='warn')
# check for expected results
# check for existence and structure of statistics file
assert os.path.isfile(statistics_fp)
# check for required keys
cv_statistics = load_json(statistics_fp)
for key in ['fold_' + str(i + 1)
for i in range(num_folds)] + ['overall']:
assert key in cv_statistics
# check for existence and structure of split indices file
assert os.path.isfile(indices_fp)
# check for required keys
cv_indices = load_json(indices_fp)
for key in ['fold_' + str(i + 1) for i in range(num_folds)]:
assert key in cv_indices
def test_server_integration_with_images(csv_filename):
# Image Inputs
image_dest_folder = os.path.join(os.getcwd(), "generated_images")
# Resnet encoder
input_features = [
image_feature(
folder=image_dest_folder,
preprocessing={"in_memory": True, "height": 8, "width": 8, "num_channels": 3},
fc_size=16,
num_filters=8,
),
text_feature(encoder="embed", min_len=1),
numerical_feature(normalization="zscore"),
]
output_features = [category_feature(vocab_size=4), numerical_feature()]
np.random.seed(123) # reproducible synthetic data
rel_path = generate_data(input_features, output_features, csv_filename)
model, output_dir = train_model(input_features, output_features, data_csv=rel_path)
app = server(model)
client = TestClient(app)
response = client.get("/")
assert response.status_code == 200
response = client.post("/predict")
# expect the HTTP 400 error code for this situation
assert response.status_code == 400
assert response.json() == ALL_FEATURES_PRESENT_ERROR
data_df = read_csv(rel_path)
# One-off prediction
first_entry = data_df.T.to_dict()[0]
data, files = convert_to_form(first_entry)
server_response = client.post("/predict", data=data, files=files)
assert server_response.status_code == 200
server_response = server_response.json()
server_response_keys = sorted(list(server_response.keys()))
assert server_response_keys == sorted(output_keys_for(output_features))
model_output, _ = model.predict(dataset=[first_entry], data_format=dict)
model_output = model_output.to_dict("records")[0]
assert model_output == server_response
# Batch prediction
assert len(data_df) > 1
files = convert_to_batch_form(data_df)
server_response = client.post("/batch_predict", files=files)
assert server_response.status_code == 200
server_response = server_response.json()
server_response_keys = sorted(server_response["columns"])
assert server_response_keys == sorted(output_keys_for(output_features))
assert len(data_df) == len(server_response["data"])
model_output, _ = model.predict(dataset=data_df)
model_output = model_output.to_dict("split")
assert model_output == server_response
# Cleanup
shutil.rmtree(output_dir, ignore_errors=True)
shutil.rmtree(image_dest_folder, ignore_errors=True)
