def test_initializers_with_pattern():
wide = Wide(100, 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
deeptext = DeepText(vocab_size=vocab_size, embed_dim=32, padding_idx=0)
model = WideDeep(wide=wide,
deeptabular=deeptabular,
deeptext=deeptext,
pred_dim=1)
cmodel = c(model)
org_word_embed = []
for n, p in cmodel.named_parameters():
if "word_embed" in n:
org_word_embed.append(p)
trainer = Trainer(model,
objective="binary",
verbose=0,
initializers=initializers_2)
init_word_embed = []
for n, p in trainer.model.named_parameters():
if "word_embed" in n:
init_word_embed.append(p)
assert torch.all(org_word_embed[0] == init_word_embed[0].cpu())
def test_fit_objectives(
X_wide,
X_tab,
target,
objective,
X_wide_test,
X_tab_test,
X_test,
pred_dim,
probs_dim,
):
wide = Wide(np.unique(X_wide).shape[0], pred_dim)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular, pred_dim=pred_dim)
trainer = Trainer(model, objective=objective, verbose=0)
trainer.fit(X_wide=X_wide, X_tab=X_tab, target=target, batch_size=16)
preds = trainer.predict(X_wide=X_wide, X_tab=X_tab, X_test=X_test)
if objective == "binary":
pass
else:
probs = trainer.predict_proba(X_wide=X_wide,
X_tab=X_tab,
X_test=X_test)
assert preds.shape[0] == 32, probs.shape[1] == probs_dim
def test_basic_run_with_metrics_multiclass():
wide = Wide(np.unique(X_wide).shape[0], 3)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx={k: v
for v, k in enumerate(colnames)},
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular, pred_dim=3)
trainer = Trainer(model,
objective="multiclass",
metrics=[Accuracy],
verbose=False)
trainer.fit(
X_wide=X_wide,
X_tab=X_tab,
target=target_multi,
n_epochs=1,
batch_size=16,
val_split=0.2,
)
assert ("train_loss" in trainer.history.keys()
and "train_acc" in trainer.history.keys())
def test_model_checkpoint(save_best_only, max_save, n_files):
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular)
trainer = Trainer(
model=model,
objective="binary",
callbacks=[
ModelCheckpoint(
"tests/test_model_functioning/weights/test_weights",
save_best_only=save_best_only,
max_save=max_save,
)
],
verbose=0,
)
trainer.fit(X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, val_split=0.2)
n_saved = len(os.listdir("tests/test_model_functioning/weights/"))
shutil.rmtree("tests/test_model_functioning/weights/")
assert n_saved <= n_files
def test_save_and_load_dict():
wide = Wide(np.unique(X_wide).shape[0], 1)
tabmlp = TabMlp(
mlp_hidden_dims=[32, 16],
column_idx={k: v
for v, k in enumerate(colnames)},
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model1 = WideDeep(wide=deepcopy(wide), deeptabular=deepcopy(tabmlp))
trainer1 = Trainer(model1, objective="binary", verbose=0)
trainer1.fit(
X_wide=X_wide,
X_tab=X_tab,
X_text=X_text,
X_img=X_img,
target=target,
batch_size=16,
)
wide_weights = model1.wide.wide_linear.weight.data
trainer1.save_model_state_dict(
"tests/test_model_functioning/model_dir/model_d.t")
model2 = WideDeep(wide=wide, deeptabular=tabmlp)
trainer2 = Trainer(model2, objective="binary", verbose=0)
trainer2.load_model_state_dict(
"tests/test_model_functioning/model_dir/model_d.t")
n_wide_weights = trainer2.model.wide.wide_linear.weight.data
shutil.rmtree("tests/test_model_functioning/model_dir/")
assert torch.allclose(wide_weights, n_wide_weights)
def test_filepath_error():
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[16, 4],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular)
with pytest.raises(ValueError):
trainer = Trainer( # noqa: F841
model=model,
objective="binary",
callbacks=[ModelCheckpoint(filepath="wrong_file_path")],
verbose=0,
)
def test_aliases():
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular, pred_dim=1)
trainer = Trainer(model, loss="regression", verbose=0)
trainer.fit(X_wide=X_wide,
X_tab=X_tab,
target=target_regres,
batch_size=16,
warmup=True)
assert ("train_loss" in trainer.history.keys()
and trainer.__wd_aliases_used["objective"] == "loss"
and trainer.__wd_aliases_used["finetune"] == "warmup")
def test_fit_with_regression_and_metric():
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular, pred_dim=1)
trainer = Trainer(model,
objective="regression",
metrics=[R2Score],
verbose=0)
trainer.fit(X_wide=X_wide,
X_tab=X_tab,
target=target_regres,
batch_size=16)
assert "train_r2" in trainer.history.keys()
def test_fit_with_deephead():
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
deephead = nn.Sequential(nn.Linear(16, 8), nn.Linear(8, 4))
model = WideDeep(wide=wide,
deeptabular=deeptabular,
pred_dim=1,
deephead=deephead)
trainer = Trainer(model, objective="binary", verbose=0)
trainer.fit(X_wide=X_wide,
X_tab=X_tab,
target=target_binary,
batch_size=16)
preds = trainer.predict(X_wide=X_wide, X_tab=X_tab, X_test=X_test)
probs = trainer.predict_proba(X_wide=X_wide, X_tab=X_tab, X_test=X_test)
assert preds.shape[0] == 32, probs.shape[1] == 2
def test_warning_when_missing_initializer():
wide = Wide(100, 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
deeptext = DeepText(vocab_size=vocab_size, embed_dim=32, padding_idx=0)
model = WideDeep(wide=wide,
deeptabular=deeptabular,
deeptext=deeptext,
pred_dim=1)
with pytest.warns(UserWarning):
trainer = Trainer( # noqa: F841
model,
objective="binary",
verbose=True,
initializers=initializers_3)
def test_initializers_1(initializers, test_layers):
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
deeptext = DeepText(vocab_size=vocab_size, embed_dim=32, padding_idx=0)
deepimage = DeepImage(pretrained=True)
model = WideDeep(
wide=wide,
deeptabular=deeptabular,
deeptext=deeptext,
deepimage=deepimage,
pred_dim=1,
)
cmodel = c(model)
org_weights = []
for n, p in cmodel.named_parameters():
if n in test_layers:
org_weights.append(p)
trainer = Trainer(model,
objective="binary",
verbose=0,
initializers=initializers)
init_weights = []
for n, p in trainer.model.named_parameters():
if n in test_layers:
init_weights.append(p)
res = all([
torch.all((1 - (a == b).int()).bool())
for a, b in zip(org_weights, init_weights)
])
assert res
def set_model(args, prepare_tab):
if args.mlp_hidden_dims == "auto":
n_inp_dim = sum([e[2] for e in prepare_tab.embeddings_input])
mlp_hidden_dims = [4 * n_inp_dim, 2 * n_inp_dim]
else:
mlp_hidden_dims = eval(args.mlp_hidden_dims)
deeptabular = TabMlp(
column_idx=prepare_tab.column_idx,
mlp_hidden_dims=mlp_hidden_dims,
mlp_activation=args.mlp_activation,
mlp_dropout=args.mlp_dropout,
mlp_batchnorm=args.mlp_batchnorm,
mlp_batchnorm_last=args.mlp_batchnorm_last,
mlp_linear_first=args.mlp_linear_first,
embed_input=prepare_tab.embeddings_input,
embed_dropout=args.embed_dropout,
)
model = WideDeep(deeptabular=deeptabular)
return model
def test_early_stop():
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular)
trainer = Trainer(
model=model,
objective="binary",
callbacks=[
EarlyStopping(
min_delta=5.0, patience=3, restore_best_weights=True, verbose=1
)
],
verbose=1,
)
trainer.fit(X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.2, n_epochs=5)
# length of history = patience+1
assert len(trainer.history["train_loss"]) == 3 + 1
X_tab_tr,
X_tab_val,
X_text_tr,
X_text_val,
X_img_tr,
X_img_val,
y_train,
y_val,
) = train_test_split(X_wide, X_tab, X_text, X_img, target)
# build model components
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx={k: v
for v, k in enumerate(colnames)},
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
deeptext = DeepText(vocab_size=vocab_size, embed_dim=32, padding_idx=0)
deepimage = DeepImage(pretrained=True)
# transforms
mean = [0.406, 0.456, 0.485] # BGR
std = [0.225, 0.224, 0.229] # BGR
transforms1 = [ToTensor, Normalize(mean=mean, std=std)]
transforms2 = [Normalize(mean=mean, std=std)]
deephead_ds = nn.Sequential(nn.Linear(16, 8), nn.Linear(8, 4))
deephead_dt = nn.Sequential(nn.Linear(64, 8), nn.Linear(8, 4))
deephead_di = nn.Sequential(nn.Linear(512, 8), nn.Linear(8, 4))
from copy import deepcopy
import pytest
from torch import nn
from pytorch_widedeep.models import Wide, TabMlp, DeepText, WideDeep, DeepImage
embed_input = [(u, i, j)
for u, i, j in zip(["a", "b", "c"][:4], [4] * 3, [8] * 3)]
column_idx = {k: v for v, k in enumerate(["a", "b", "c"])}
wide = Wide(10, 1)
deepdense = TabMlp(mlp_hidden_dims=[16, 8],
column_idx=column_idx,
embed_input=embed_input)
deeptext = DeepText(vocab_size=100, embed_dim=8)
deepimage = DeepImage(pretrained=False)
###############################################################################
# test raising 'output dim errors'
###############################################################################
@pytest.mark.parametrize(
"deepcomponent, component_name",
[
(None, "dense"),
(deeptext, "text"),
(deepimage, "image"),
],
)
def test_history_callback(deepcomponent, component_name):
X_valid = prepare_tab.transform(valid)
y_valid = valid.target.values
if args.mlp_hidden_dims == "auto":
n_inp_dim = sum([e[2] for e in prepare_tab.embeddings_input])
mlp_hidden_dims = [4 * n_inp_dim, 2 * n_inp_dim]
else:
mlp_hidden_dims = eval(args.mlp_hidden_dims)
deeptabular = TabMlp(
column_idx=prepare_tab.column_idx,
mlp_hidden_dims=mlp_hidden_dims,
mlp_activation=args.mlp_activation,
mlp_dropout=args.mlp_dropout,
mlp_batchnorm=args.mlp_batchnorm,
mlp_batchnorm_last=args.mlp_batchnorm_last,
mlp_linear_first=args.mlp_linear_first,
embed_input=prepare_tab.embeddings_input,
embed_dropout=args.embed_dropout,
continuous_cols=prepare_tab.continuous_cols,
batchnorm_cont=args.batchnorm_cont,
)
model = WideDeep(deeptabular=deeptabular)
optimizers = set_optimizer(model, args)
steps_per_epoch = (X_train.shape[0] // args.batch_size) + 1
lr_schedulers = set_lr_scheduler(optimizers, steps_per_epoch, args)
early_stopping = EarlyStopping(
monitor=args.monitor,
min_delta=args.early_stop_delta,
target = df[target].values
prepare_wide = WidePreprocessor(wide_cols=wide_cols,
crossed_cols=crossed_cols)
X_wide = prepare_wide.fit_transform(df)
prepare_deep = TabPreprocessor(
embed_cols=cat_embed_cols,
continuous_cols=continuous_cols # type: ignore[arg-type]
)
X_tab = prepare_deep.fit_transform(df)
wide = Wide(wide_dim=np.unique(X_wide).shape[0], pred_dim=1)
deeptabular = TabMlp(
mlp_hidden_dims=[200, 100],
mlp_dropout=[0.2, 0.2],
column_idx=prepare_deep.column_idx,
embed_input=prepare_deep.embeddings_input,
continuous_cols=continuous_cols,
)
# # To use TabResnet as the deeptabular component simply:
# deeptabular = TabResnet(
# blocks_dims=[200, 100],
# column_idx=prepare_deep.column_idx,
# embed_input=prepare_deep.embeddings_input,
# continuous_cols=continuous_cols,
# )
model = WideDeep(wide=wide, deeptabular=deeptabular)
wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.01)
cont_cols = [np.random.rand(32) for _ in range(5)]
column_idx = {k: v for v, k in enumerate(colnames)}
X_tab = np.vstack(embed_cols + cont_cols).transpose()
# target
target = np.random.choice(2, 32)
###############################################################################
# Test that history saves the information adequately
###############################################################################
wide = Wide(np.unique(X_wide).shape[0], 1)
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16],
mlp_dropout=[0.5, 0.5],
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=colnames[-5:],
)
model = WideDeep(wide=wide, deeptabular=deeptabular)
# 1. Single optimizers_1, single scheduler, not cyclic and both passed directly
optimizers_1 = RAdam(model.parameters())
lr_schedulers_1 = StepLR(optimizers_1, step_size=4)
# 2. Multiple optimizers, single scheduler, cyclic and pass via a 1 item
# dictionary
wide_opt_2 = torch.optim.Adam(model.wide.parameters())
deep_opt_2 = RAdam(model.deeptabular.parameters())
deep_sch_2 = CyclicLR(
deep_opt_2, base_lr=0.001, max_lr=0.01, step_size_up=5, cycle_momentum=False
# image
X_image = torch.rand(100, 3, 28, 28)
# Define the model components
# wide
wide = Wide(X_wide.unique().size(0), 1)
if use_cuda:
wide.cuda()
# deep
deeptabular = TabMlp(
mlp_hidden_dims=[32, 16, 8],
mlp_dropout=0.2,
column_idx=column_idx,
embed_input=embed_input,
continuous_cols=continuous_cols,
)
deeptabular = nn.Sequential(deeptabular, nn.Linear(8, 1)) # type: ignore[assignment]
if use_cuda:
deeptabular.cuda()
# text
deeptext = TestDeepText()
if use_cuda:
deeptext.cuda()
# image
deepimage = TestDeepImage()
if use_cuda:
crossed_cols=wide_cross_column_list)
x_wide = wide_preprocessor.fit_transform(train_df)
# Deep
tab_preprocessor = TabPreprocessor(
embed_cols=deep_embedding_columns_list,
continuous_cols=deep_continuous_column_list)
x_deep = tab_preprocessor.fit_transform(train_df)
"""
Model 구조 정의
"""
# Model
wide = Wide(wide_dim=np.unique(x_wide).shape[0], pred_dim=1)
deeptabular = TabMlp(mlp_hidden_dims=[64, 32],
column_idx=tab_preprocessor.column_idx,
embed_input=tab_preprocessor.embeddings_input,
continuous_cols=deep_continuous_column_list)
model = WideDeep(wide=wide, deeptabular=deeptabular)
"""
학습
"""
trainer = Trainer(model, objective="binary", metrics=[Accuracy])
trainer.fit(X_wide=x_wide,
X_tab=x_deep,
target=target,
n_epochs=5,
batch_size=256,
val_split=0.1)
trainer.save_model(MODEL_PATH)
save_pickle(WIDE_PROC_PATH, wide_preprocessor)
already_standard=already_standard,
)
X_tab = tab_preprocessor.fit_transform(df)
text_processor = TextPreprocessor(word_vectors_path=word_vectors_path,
text_col=text_col)
X_text = text_processor.fit_transform(df)
image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)
X_images = image_processor.fit_transform(df)
wide = Wide(wide_dim=np.unique(X_wide).shape[0], pred_dim=1)
deepdense = TabMlp(
mlp_hidden_dims=[64, 32],
mlp_dropout=[0.2, 0.2],
column_idx=tab_preprocessor.column_idx,
embed_input=tab_preprocessor.embeddings_input,
continuous_cols=continuous_cols,
)
# # To use TabResnet as the deepdense component simply:
# deepdense = TabResnet(
# blocks_dims=[64, 32],
# dropout=0.2,
# column_idx=tab_preprocessor.column_idx,
# embed_input=tab_preprocessor.embeddings_input,
# continuous_cols=continuous_cols,
# )
deeptext = DeepText(
vocab_size=len(text_processor.vocab.itos),
hidden_dim=64,
n_layers=3,
def objective(params):
deeptabular = TabMlp(
column_idx=prepare_tab.column_idx,
mlp_hidden_dims=mlp_hidden_dims,
mlp_activation=args.mlp_activation,
mlp_dropout=args.mlp_dropout,
mlp_batchnorm=args.mlp_batchnorm,
mlp_batchnorm_last=args.mlp_batchnorm_last,
mlp_linear_first=args.mlp_linear_first,
embed_input=prepare_tab.embeddings_input,
embed_dropout=args.embed_dropout,
)
model = WideDeep(deeptabular=deeptabular)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode=args.rop_mode,
factor=args.rop_factor,
patience=args.rop_patience,
threshold=args.rop_threshold,
threshold_mode=args.rop_threshold_mode,
)
early_stopping = EarlyStopping(
monitor=args.monitor,
min_delta=args.early_stop_delta,
patience=args.early_stop_patience,
)
trainer = Trainer(
model,
objective="binary_focal_loss",
optimizers=optimizer,
lr_schedulers=lr_scheduler,
reducelronplateau_criterion=args.monitor.split("_")[-1],
callbacks=[early_stopping,
LRHistory(n_epochs=args.n_epochs)],
metrics=[Accuracy, F1Score],
alpha=params["alpha"],
gamma=params["gamma"],
verbose=0,
)
trainer.fit(
X_train={
"X_tab": X_train,
"target": y_train
},
X_val={
"X_tab": X_valid,
"target": y_valid
},
n_epochs=args.n_epochs,
batch_size=args.batch_size,
validation_freq=args.eval_every,
)
score = early_stopping.best
return score