def test_neuropod(csv_filename):
#######
# Setup
#######
with tempfile.TemporaryDirectory() as tmpdir:
dir_path = tmpdir
data_csv_path = os.path.join(tmpdir, csv_filename)
image_dest_folder = os.path.join(tmpdir, "generated_images")
audio_dest_folder = os.path.join(tmpdir, "generated_audio")
input_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
vector_feature(),
image_feature(image_dest_folder),
audio_feature(audio_dest_folder),
timeseries_feature(),
date_feature(),
h3_feature(),
set_feature(vocab_size=3),
bag_feature(vocab_size=3),
]
output_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
set_feature(vocab_size=3),
vector_feature(),
]
# Generate test data
data_csv_path = generate_data(input_features, output_features, data_csv_path)
#############
# Train model
#############
config = {"input_features": input_features, "output_features": output_features, "training": {"epochs": 2}}
ludwig_model = LudwigModel(config, backend=LocalTestBackend())
ludwig_model.train(
dataset=data_csv_path,
skip_save_training_description=True,
skip_save_training_statistics=True,
skip_save_progress=True,
skip_save_log=True,
skip_save_processed_input=True,
output_directory=dir_path,
)
data_df = pd.read_csv(data_csv_path)
original_predictions_df, _ = ludwig_model.predict(dataset=data_df)
###################
# save Ludwig model
###################
ludwigmodel_path = os.path.join(dir_path, "ludwigmodel")
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
ludwig_model.save(ludwigmodel_path)
################
# build neuropod
################
neuropod_path = os.path.join(dir_path, "neuropod")
shutil.rmtree(neuropod_path, ignore_errors=True)
export_neuropod(ludwigmodel_path, neuropod_path=neuropod_path, entrypoint="get_test_model")
########################
# predict using neuropod
########################
if_dict = {
input_feature["name"]: np.expand_dims(
np.array([str(x) for x in data_df[input_feature["name"]].tolist()], dtype="str"), 1
)
for input_feature in input_features
}
from neuropod.loader import load_neuropod
neuropod_model = load_neuropod(neuropod_path, _always_use_native=False)
preds = neuropod_model.infer(if_dict)
for key in preds:
preds[key] = np.squeeze(preds[key])
#########
# cleanup
#########
# Delete the temporary data created
for path in [ludwigmodel_path, neuropod_path, image_dest_folder, audio_dest_folder]:
if os.path.exists(path):
if os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path, ignore_errors=True)
########
# checks
########
for output_feature in output_features:
output_feature_name = output_feature["name"]
output_feature_type = output_feature["type"]
if (
output_feature_name + "_predictions" in preds
and output_feature_name + "_predictions" in original_predictions_df
):
neuropod_pred = preds[output_feature_name + "_predictions"].tolist()
if output_feature_type == BINARY:
neuropod_pred = [str2bool(x) for x in neuropod_pred]
if output_feature_type in {SEQUENCE, TEXT, SET}:
neuropod_pred = [x.split() for x in neuropod_pred]
original_pred = original_predictions_df[output_feature_name + "_predictions"].tolist()
assert neuropod_pred == original_pred
if (
output_feature_name + "_probability" in preds
and output_feature_name + "_probability" in original_predictions_df
):
neuropod_prob = preds[output_feature_name + "_probability"].tolist()
if output_feature_type in {SEQUENCE, TEXT, SET}:
neuropod_prob = [[float(n) for n in x.split()] for x in neuropod_prob]
if any(isinstance(el, list) for el in neuropod_prob):
neuropod_prob = np.array(list(itertools.zip_longest(*neuropod_prob, fillvalue=0))).T
original_prob = original_predictions_df[output_feature_name + "_probability"].tolist()
if any(isinstance(el, list) for el in original_prob):
original_prob = np.array(list(itertools.zip_longest(*original_prob, fillvalue=0))).T
assert np.allclose(neuropod_prob, original_prob)
if (
output_feature_name + "_probabilities" in preds
and output_feature_name + "_probabilities" in original_predictions_df
):
neuropod_prob = preds[output_feature_name + "_probabilities"].tolist()
original_prob = original_predictions_df[output_feature_name + "_probabilities"].tolist()
assert np.allclose(neuropod_prob, original_prob)
def _test_neuropod(csv_filename):
#######
# Setup
#######
dir_path = os.path.dirname(csv_filename)
image_dest_folder = os.path.join(os.getcwd(), 'generated_images')
audio_dest_folder = os.path.join(os.getcwd(), 'generated_audio')
input_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
vector_feature(),
image_feature(image_dest_folder),
audio_feature(audio_dest_folder),
timeseries_feature(),
date_feature(),
h3_feature(),
set_feature(vocab_size=3),
bag_feature(vocab_size=3),
]
output_features = [
binary_feature(),
numerical_feature(),
category_feature(vocab_size=3),
sequence_feature(vocab_size=3),
text_feature(vocab_size=3),
set_feature(vocab_size=3),
vector_feature()
]
# Generate test data
data_csv_path = generate_data(input_features, output_features, csv_filename)
#############
# Train model
#############
model_definition = {
'input_features': input_features,
'output_features': output_features,
'training': {'epochs': 2}
}
ludwig_model = LudwigModel(model_definition)
ludwig_model.train(
data_csv=data_csv_path,
skip_save_training_description=True,
skip_save_training_statistics=True,
skip_save_model=True,
skip_save_progress=True,
skip_save_log=True,
skip_save_processed_input=True,
)
original_predictions_df = ludwig_model.predict(data_csv=data_csv_path)
###################
# save Ludwig model
###################
ludwigmodel_path = os.path.join(dir_path, 'ludwigmodel')
shutil.rmtree(ludwigmodel_path, ignore_errors=True)
ludwig_model.save(ludwigmodel_path)
################
# build neuropod
################
neuropod_path = os.path.join(dir_path, 'neuropod')
export_neuropod(ludwigmodel_path, neuropod_path=neuropod_path)
########################
# predict using neuropod
########################
data_df = pd.read_csv(data_csv_path)
if_dict = {
input_feature['name']: np.expand_dims(np.array(
[str(x) for x in data_df[input_feature['name']].tolist()],
dtype='str'
), 1)
for input_feature in input_features
}
from neuropod.loader import load_neuropod
neuropod_model = load_neuropod(neuropod_path)
preds = neuropod_model.infer(if_dict)
for key in preds:
preds[key] = np.squeeze(preds[key])
#########
# cleanup
#########
# Delete the temporary data created
for path in [ludwigmodel_path, neuropod_path,
image_dest_folder, audio_dest_folder]:
if os.path.exists(path):
if os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path, ignore_errors=True)
########
# checks
########
for output_feature in output_features:
output_feature_name = output_feature['name']
output_feature_type = output_feature['type']
if (output_feature_name + "_predictions" in preds and
output_feature_name + "_predictions" in original_predictions_df):
neuropod_pred = preds[output_feature_name + "_predictions"].tolist()
if output_feature_type == BINARY:
neuropod_pred = list(map(lambda x: str2bool(x), neuropod_pred))
if output_feature_type in {SEQUENCE, TEXT, SET}:
neuropod_pred = list(map(lambda x: x.split(), neuropod_pred))
original_pred = original_predictions_df[
output_feature_name + "_predictions"].tolist()
assert neuropod_pred == original_pred
if (output_feature_name + "_probability" in preds and
output_feature_name + "_probability" in original_predictions_df):
neuropod_prob = preds[output_feature_name + "_probability"].tolist()
if output_feature_type in {SEQUENCE, TEXT, SET}:
neuropod_prob = list(
map(lambda x: [float(n) for n in x.split()], neuropod_prob))
if any(isinstance(el, list) for el in neuropod_prob):
neuropod_prob = np.array(list(
itertools.zip_longest(*neuropod_prob, fillvalue=0)
)).T
original_prob = original_predictions_df[
output_feature_name + "_probability"].tolist()
if any(isinstance(el, list) for el in original_prob):
original_prob = np.array(list(
itertools.zip_longest(*original_prob, fillvalue=0)
)).T
assert np.isclose(neuropod_prob, original_prob).all()
if (output_feature_name + "_probabilities" in preds and
output_feature_name + "_probabilities" in original_predictions_df):
neuropod_prob = preds[
output_feature_name + "_probabilities"].tolist()
original_prob = original_predictions_df[
output_feature_name + "_probabilities"].tolist()
assert np.isclose(neuropod_prob, original_prob).all()
def test_neuropod_torchscript(csv_filename, tmpdir):
data_csv_path = os.path.join(tmpdir, csv_filename)
# Configure features to be tested:
bin_str_feature = binary_feature()
input_features = [
bin_str_feature,
# binary_feature(),
number_feature(),
category_feature(vocab_size=3),
# TODO: future support
# sequence_feature(vocab_size=3),
# text_feature(vocab_size=3),
# vector_feature(),
# image_feature(image_dest_folder),
# audio_feature(audio_dest_folder),
# timeseries_feature(),
# date_feature(),
# h3_feature(),
# set_feature(vocab_size=3),
# bag_feature(vocab_size=3),
]
output_features = [
bin_str_feature,
# binary_feature(),
number_feature(),
category_feature(vocab_size=3),
# TODO: future support
# sequence_feature(vocab_size=3),
# text_feature(vocab_size=3),
# set_feature(vocab_size=3),
# vector_feature()
]
backend = LocalTestBackend()
config = {
"input_features": input_features,
"output_features": output_features,
TRAINER: {
"epochs": 2
}
}
# Generate training data
training_data_csv_path = generate_data(input_features, output_features,
data_csv_path)
# Convert bool values to strings, e.g., {'Yes', 'No'}
df = pd.read_csv(training_data_csv_path)
false_value, true_value = "No", "Yes"
df[bin_str_feature[NAME]] = df[bin_str_feature[NAME]].map(
lambda x: true_value if x else false_value)
df.to_csv(training_data_csv_path)
# Train Ludwig (Pythonic) model:
ludwig_model = LudwigModel(config, backend=backend)
ludwig_model.train(
dataset=training_data_csv_path,
skip_save_training_description=True,
skip_save_training_statistics=True,
skip_save_model=True,
skip_save_progress=True,
skip_save_log=True,
skip_save_processed_input=True,
)
# Obtain predictions from Python model
preds_dict, _ = ludwig_model.predict(dataset=training_data_csv_path,
return_type=dict)
# Create graph inference model (Torchscript) from trained Ludwig model.
neuropod_path = os.path.join(tmpdir, "neuropod")
export_neuropod(ludwig_model, neuropod_path)
from neuropod.loader import load_neuropod
neuropod_module = load_neuropod(neuropod_path)
def to_input(s: pd.Series) -> Union[List[str], torch.Tensor]:
if s.dtype == "object":
return np.array(s.to_list())
return s.to_numpy().astype(np.float32)
df = pd.read_csv(training_data_csv_path)
inputs = {
name: to_input(df[feature.column])
for name, feature in ludwig_model.model.input_features.items()
}
outputs = neuropod_module.infer(inputs)
# Compare results from Python trained model against Neuropod
assert len(preds_dict) == len(outputs)
for feature_name, feature_outputs_expected in preds_dict.items():
assert feature_name in outputs
output_values_expected = feature_outputs_expected[PREDICTIONS]
output_values = outputs[feature_name]
if output_values.dtype.type in {np.string_, np.str_}:
# Strings should match exactly
assert np.all(output_values == output_values_expected
), f"feature: {feature_name}, output: predictions"
else:
assert np.allclose(
output_values, output_values_expected
), f"feature: {feature_name}, output: predictions"