def get_model(name, dataset):
"""
Hyperparameters are empirically determined, not opitmized.
"""
field_dims = dataset.field_dims
print('field_dims={}, {}'.format(type(field_dims), field_dims.tolist()))
#1/0
if name == 'lr':
return LogisticRegressionModel(field_dims)
elif name == 'fm':
return FactorizationMachineModel(field_dims, embed_dim=16)
elif name == 'hofm':
return HighOrderFactorizationMachineModel(field_dims, order=3, embed_dim=16)
elif name == 'ffm':
return FieldAwareFactorizationMachineModel(field_dims, embed_dim=4)
elif name == 'fnn':
return FactorizationSupportedNeuralNetworkModel(field_dims, embed_dim=16, mlp_dims=(16, 16), dropout=0.2)
elif name == 'wd':
return WideAndDeepModel(field_dims, embed_dim=16, mlp_dims=(16, 16), dropout=0.2)
elif name == 'ipnn':
#return ProductNeuralNetworkModel(field_dims, embed_dim=16, mlp_dims=(16,), method='inner', dropout=0.2)
#return ProductNeuralNetworkModel(field_dims, embed_dim=64, mlp_dims=(16,16), method='inner', dropout=0.2)
return ProductNeuralNetworkModel(field_dims, embed_dim=64, mlp_dims=(128,64,128,128,128,), method='inner', dropout=0.2)
elif name == 'opnn':
return ProductNeuralNetworkModel(field_dims, embed_dim=16, mlp_dims=(16,), method='outer', dropout=0.2)
elif name == 'dcn':
return DeepCrossNetworkModel(field_dims, embed_dim=16, num_layers=3, mlp_dims=(16, 16), dropout=0.2)
elif name == 'nfm':
return NeuralFactorizationMachineModel(field_dims, embed_dim=64, mlp_dims=(64,), dropouts=(0.2, 0.2))
elif name == 'ncf':
# only supports MovieLens dataset because for other datasets user/item colums are indistinguishable
assert isinstance(dataset, MovieLens20MDataset) or isinstance(dataset, MovieLens1MDataset)
return NeuralCollaborativeFiltering(field_dims, embed_dim=16, mlp_dims=(16, 16), dropout=0.2,
user_field_idx=dataset.user_field_idx,
item_field_idx=dataset.item_field_idx)
elif name == 'fnfm':
return FieldAwareNeuralFactorizationMachineModel(field_dims, embed_dim=64, mlp_dims=(64,32,), dropouts=(0.2, 0.2))
elif name == 'dfm':
#return DeepFactorizationMachineModel(field_dims, embed_dim=16, mlp_dims=(16, 16), dropout=0.2)
#return DeepFactorizationMachineModel(field_dims, embed_dim=64, mlp_dims=(256, 256, 256, 256, 256,256), dropout=0.2)
#return DeepFactorizationMachineModel(field_dims, embed_dim=128, mlp_dims=(400, 400, 400, 400), dropout=0.2)
return DeepFactorizationMachineModel(field_dims, embed_dim=64, mlp_dims=(128, 128, 128), dropout=0.2)
elif name == 'xdfm':
return ExtremeDeepFactorizationMachineModel(
field_dims, embed_dim=16, cross_layer_sizes=(16, 16), split_half=False, mlp_dims=(16, 16), dropout=0.2)
elif name == 'afm':
return AttentionalFactorizationMachineModel(field_dims, embed_dim=16, attn_size=16, dropouts=(0.2, 0.2))
elif name == 'afi':
return AutomaticFeatureInteractionModel(
field_dims, embed_dim=16, atten_embed_dim=64, num_heads=2, num_layers=3, mlp_dims=(400, 400), dropouts=(0, 0, 0))
elif name == 'afn':
print("Model:AFN")
return AdaptiveFactorizationNetwork(
field_dims, embed_dim=64, LNN_dim=1500, mlp_dims=(400, 400, 400), dropouts=(0, 0, 0))
else:
raise ValueError('unknown model name: ' + name)
def __getitem__(self, index):
return self.train_1[index],self.train_2[index]
def __len__(self):
return len(self.train_1)
dataset = Kaggle_data(final_train_int,final_train_float,target)
train_length = int(len(dataset) * 0.9)
# valid_length = int(len(dataset) * 0.1)
test_length = len(dataset) - train_length
train_dataset, test_dataset = torch.utils.data.random_split(
dataset, (train_length, test_length))
train_data_loader = DataLoader(train_dataset, batch_size=512, num_workers=8)
test_data_loader = DataLoader(test_dataset, batch_size=1024, num_workers=8)
model = AutomaticFeatureInteractionModel(
field_dims, embed_dim=16, num_heads=4, num_layers=4, mlp_dims=(128,128,128), dropouts=(0.5,0.5)).cuda()
# model.load_state_dict(torch.load('./model/kaggle.pth'))
criterion = torch.nn.BCELoss()
optimizer = torch.optim.Adam(params=model.parameters(), lr=0.00001, weight_decay=0.001)
auc = 0
log_interval = 100
for epoch in range(50):
model.train()
total_loss = 0
for i, (train_1, train_2, target) in enumerate(tqdm.tqdm(train_data_loader, smoothing=0, mininterval=1.0)):
train_1,train_2,target = train_1.cuda(),train_2.cuda(),target.cuda()
y = model(train_1, train_2)
loss = criterion(y, target.float())
def get_model(name, field_dims):
"""
Hyperparameters are empirically determined, not opitmized.
"""
if name == 'lr':
return LogisticRegressionModel(field_dims)
elif name == 'fm':
return FactorizationMachineModel(field_dims, embed_dim=16)
elif name == 'ffm':
return FieldAwareFactorizationMachineModel(field_dims, embed_dim=4)
elif name == 'fnn':
return FactorizationSupportedNeuralNetworkModel(field_dims,
embed_dim=16,
mlp_dims=(16, 16),
dropout=0.2)
elif name == 'wd':
return WideAndDeepModel(field_dims,
embed_dim=16,
mlp_dims=(16, 16),
dropout=0.2)
elif name == 'ipnn':
return ProductNeuralNetworkModel(field_dims,
embed_dim=16,
mlp_dims=(16, ),
method='inner',
dropout=0.2)
elif name == 'opnn':
return ProductNeuralNetworkModel(field_dims,
embed_dim=16,
mlp_dims=(16, ),
method='outer',
dropout=0.2)
elif name == 'dcn':
return DeepCrossNetworkModel(field_dims,
embed_dim=16,
num_layers=3,
mlp_dims=(16, 16),
dropout=0.2)
elif name == 'nfm':
return NeuralFactorizationMachineModel(field_dims,
embed_dim=64,
mlp_dims=(64, ),
dropouts=(0.2, 0.2))
elif name == 'fnfm':
return FieldAwareNeuralFactorizationMachineModel(field_dims,
embed_dim=4,
mlp_dims=(64, ),
dropouts=(0.2, 0.2))
elif name == 'dfm':
return DeepFactorizationMachineModel(field_dims,
embed_dim=16,
mlp_dims=(16, 16),
dropout=0.2)
elif name == 'xdfm':
return ExtremeDeepFactorizationMachineModel(field_dims,
embed_dim=16,
cross_layer_sizes=(16, 16),
split_half=False,
mlp_dims=(16, 16),
dropout=0.2)
elif name == 'afm':
return AttentionalFactorizationMachineModel(field_dims,
embed_dim=16,
attn_size=16,
dropouts=(0.2, 0.2))
elif name == 'afi':
return AutomaticFeatureInteractionModel(field_dims,
embed_dim=32,
num_heads=4,
num_layers=2,
mlp_dims=(16, 16),
dropouts=(0.2, 0.2))
else:
raise ValueError('unknown model name: ' + name)
def get_model(model_name, training_method, features_dimension, dropout,
device):
training_method_signature = 'BP' if training_method == 'bp' else 'ALT'
if training_method_signature != 'BP' and training_method != 'dfa':
# use the same check used in the models to be sure of shallow
print("Will run in SHALLOW mode")
if model_name == 'fm':
if training_method_signature == 'ALT':
raise ValueError(f"Alternative training methods {training_method} "
f"not available on Factorization Machines !")
else:
model = FactorizationMachineModel(features_dimension, embed_dim=16)
elif model_name == 'deepfm':
if training_method_signature == 'ALT':
model = DFADeepFactorizationMachineModel(
features_dimension,
embed_dim=10,
mlp_dims=(400, 400, 400),
dropout=dropout,
training_method=training_method)
else:
model = DeepFactorizationMachineModel(features_dimension,
embed_dim=10,
mlp_dims=(400, 400, 400),
dropout=dropout)
elif model_name == 'deepcross':
if training_method_signature == 'ALT':
model = DFADeepCrossNetworkModel(features_dimension,
embed_dim=16,
num_layers=6,
mlp_dims=(1024, 1024),
dropout=dropout,
training_method=training_method)
else:
model = DeepCrossNetworkModel(features_dimension,
embed_dim=16,
num_layers=6,
mlp_dims=(1024, 1024),
dropout=dropout)
elif model_name == 'autoint':
if training_method_signature == 'ALT':
model = DFAAutomaticFeatureInteractionModel(
features_dimension,
embed_dim=16,
atten_embed_dim=32,
num_heads=2,
num_layers=3,
mlp_dims=(400, 400, 400),
dropouts=(dropout, dropout, dropout),
training_method=training_method)
else:
model = AutomaticFeatureInteractionModel(features_dimension,
embed_dim=16,
atten_embed_dim=32,
num_heads=2,
num_layers=3,
mlp_dims=(400, 400, 400),
dropouts=(dropout,
dropout,
dropout))
elif model_name == 'afn':
if training_method_signature == 'ALT':
model = DFAAdaptiveFactorizationNetwork(
features_dimension,
embed_dim=10,
LNN_dim=1500,
mlp_dims=(400, 400, 400),
dropouts=(dropout, dropout, dropout),
training_method=training_method)
else:
model = AdaptiveFactorizationNetwork(features_dimension,
embed_dim=10,
LNN_dim=1500,
mlp_dims=(400, 400, 400),
dropouts=(dropout, dropout,
dropout))
if isinstance(model, DeepCrossNetworkModel) or isinstance(
model, DFADeepCrossNetworkModel):
print(
"Using a DeepCross model, will use gradient clip norm=100. during training."
)
return model.to(device)