def model_test(json_file):
solver = hugectr.CreateSolver(max_eval_batches=100,
batchsize_eval=16384,
batchsize=16384,
vvgpu=[[0, 1, 2, 3], [4, 5, 6, 7]],
i64_input_key=False,
use_mixed_precision=False,
repeat_dataset=True,
use_cuda_graph=True)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Norm,
source=["./file_list.txt"],
eval_source="./file_list_test.txt",
check_type=hugectr.Check_t.Sum)
optimizer = hugectr.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.Adam)
model = hugectr.Model(solver, reader, optimizer)
model.construct_from_json(graph_config_file=json_file,
include_dense_network=True)
model.summary()
model.compile()
model.fit(max_iter=10000,
display=200,
eval_interval=1000,
snapshot=100000,
snapshot_prefix="criteo")
def single_node_test(args):
solver = hugectr.CreateSolver(max_eval_batches=args.max_eval_batches,
batchsize_eval=args.batchsize_eval,
batchsize=args.batchsize,
vvgpu=args.vvgpu,
lr=args.learning_rate,
warmup_steps=args.warmup_steps,
decay_start=args.decay_start,
decay_steps=args.decay_steps,
decay_power=args.decay_power,
end_lr=args.end_lr,
i64_input_key=args.i64_input_key,
use_mixed_precision=args.use_mixed_precision,
scaler=args.scaler)
reader = hugectr.DataReaderParams(data_reader_type=args.data_reader_type,
source=[args.source],
eval_source=args.eval_source,
check_type=args.check_type,
cache_eval_data=args.cache_eval_data,
num_samples=args.num_samples,
eval_num_samples=args.eval_num_samples,
float_label_dense=args.float_label_dense,
num_workers=args.num_workers,
slot_size_array=args.slot_size_array)
optimizer = hugectr.CreateOptimizer(optimizer_type=args.optimizer_type,
beta1=args.beta1,
beta2=args.beta2,
epsilon=args.epsilon,
update_type=args.update_type,
momentum_factor=args.momentum_factor,
atomic_update=args.atomic_update)
model = hugectr.Model(solver, reader, optimizer)
model.construct_from_json(graph_config_file=args.json_file,
include_dense_network=True)
model.compile()
model.summary()
if args.auc_check:
train(model, args.max_iter, args.display, args.max_eval_batches,
args.eval_interval, args.auc_threshold)
else:
model.fit(max_iter=args.max_iter,
display=args.display,
eval_interval=args.eval_interval,
snapshot=args.snapshot)
return
def wdl_test(json_file, export_path_prefix):
solver = hugectr.CreateSolver(max_eval_batches=2048,
batchsize_eval=16384,
batchsize=16384,
vvgpu=[[0, 1, 2, 3, 4, 5, 6, 7]],
lr=0.001,
i64_input_key=False,
use_mixed_precision=True,
scaler=1024,
repeat_dataset=True,
use_cuda_graph=True)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Norm,
source=["./file_list.txt"],
eval_source="./file_list_test.txt",
check_type=hugectr.Check_t.Sum)
optimizer = hugectr.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.Adam,
beta1=0.9,
beta2=0.999,
epsilon=0.0001)
model = hugectr.Model(solver, reader, optimizer)
model.construct_from_json(graph_config_file=json_file,
include_dense_network=True)
model.compile()
model.summary()
model.start_data_reading()
lr_sch = model.get_learning_rate_scheduler()
for i in range(10000):
lr = lr_sch.get_next()
model.set_learning_rate(lr)
model.train(False)
if (i % 100 == 0):
loss = model.get_current_loss()
print("[HUGECTR][INFO] iter: {}; loss: {}".format(i, loss))
if (i % 1000 == 0 and i != 0):
for _ in range(solver.max_eval_batches):
model.eval()
model.export_predictions(
export_path_prefix + "prediction" + str(i),
export_path_prefix + "label" + str(i))
metrics = model.get_eval_metrics()
print("[HUGECTR][INFO] iter: {}, {}".format(i, metrics))
return
def embedding_training_cache_test(json_file, output_dir):
dataset = [("file_list." + str(i) + ".txt",
"file_list." + str(i) + ".keyset") for i in range(5)]
solver = hugectr.CreateSolver(batchsize=16384,
batchsize_eval=16384,
vvgpu=[[0, 1, 2, 3], [4, 5, 6, 7]],
use_mixed_precision=False,
i64_input_key=False,
use_algorithm_search=True,
use_cuda_graph=True,
repeat_dataset=False)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Norm,
source=["file_list." + str(i) + ".txt" for i in range(5)],
keyset=["file_list." + str(i) + ".keyset" for i in range(5)],
eval_source="./file_list.5.txt",
check_type=hugectr.Check_t.Sum)
optimizer = hugectr.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.Adam)
hc_cnfg = hugectr.CreateHMemCache(4, 0.5, 2)
etc = hugectr.CreateETC(
ps_types=[hugectr.TrainPSType_t.Staged, hugectr.TrainPSType_t.Cached],
sparse_models=[
output_dir + "/wdl_0_sparse_model",
output_dir + "/wdl_1_sparse_model"
],
local_paths=[output_dir + "_1", output_dir + "_2"],
hmem_cache_configs=[hc_cnfg])
model = hugectr.Model(solver, reader, optimizer, etc)
model.construct_from_json(graph_config_file=json_file,
include_dense_network=True)
model.compile()
model.summary()
model.fit(num_epochs=1, eval_interval=200, display=200)
updated_model = model.get_incremental_model()
model.save_params_to_files("wdl")
model.set_source(
source=["file_list." + str(i) + ".txt" for i in range(6, 9)],
keyset=["file_list." + str(i) + ".keyset" for i in range(6, 9)],
eval_source="./file_list.5.txt")
model.fit(num_epochs=1, eval_interval=200, display=200)
updated_model = model.get_incremental_model()
model.save_params_to_files("wdl")
hugectr.MetricsType.AverageLoss: 0.0,
hugectr.MetricsType.AUC: 1.0
},
repeat_dataset=True)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Norm,
source=["./data/ml-20m/train_filelist.txt"],
eval_source="./data/ml-20m/test_filelist.txt",
check_type=hugectr.Check_t.Non,
num_workers=10)
optimizer = hugectr.CreateOptimizer(optimizer_type=hugectr.Optimizer_t.Adam,
update_type=hugectr.Update_t.Global,
beta1=0.25,
beta2=0.5,
epsilon=0.0000001)
model = hugectr.Model(solver, reader, optimizer)
model.add(
hugectr.Input(label_dim=1,
label_name="label",
dense_dim=1,
dense_name="dense",
data_reader_sparse_param_array=[
hugectr.DataReaderSparseParam("data", 1, True, 2)
]))
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
workspace_size_per_gpu_in_mb=49, # 3 for 1M dataset
embedding_vec_size=64,
combiner="sum",
sparse_embedding_name="mlp_embedding",
def _run_model(slot_sizes, total_cardinality):
solver = hugectr.CreateSolver(
vvgpu=[[0]],
batchsize=2048,
batchsize_eval=2048,
max_eval_batches=160,
i64_input_key=True,
use_mixed_precision=False,
repeat_dataset=True,
)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Parquet,
source=[DATA_DIR + "train/_file_list.txt"],
eval_source=DATA_DIR + "valid/_file_list.txt",
check_type=hugectr.Check_t.Non,
)
optimizer = hugectr.CreateOptimizer(optimizer_type=hugectr.Optimizer_t.Adam)
model = hugectr.Model(solver, reader, optimizer)
model.add(
hugectr.Input(
label_dim=1,
label_name="label",
dense_dim=0,
dense_name="dense",
data_reader_sparse_param_array=[
hugectr.DataReaderSparseParam("data1", len(slot_sizes) + 1, True, len(slot_sizes))
],
)
)
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
workspace_size_per_gpu_in_mb=107,
embedding_vec_size=16,
combiner="sum",
sparse_embedding_name="sparse_embedding1",
bottom_name="data1",
slot_size_array=slot_sizes,
optimizer=optimizer,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["sparse_embedding1"],
top_names=["reshape1"],
leading_dim=48,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["reshape1"],
top_names=["fc1"],
num_output=128,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc1"],
top_names=["relu1"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu1"],
top_names=["fc2"],
num_output=128,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc2"],
top_names=["relu2"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu2"],
top_names=["fc3"],
num_output=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,
bottom_names=["fc3", "label"],
top_names=["loss"],
)
)
model.compile()
model.summary()
model.fit(max_iter=2000, display=100, eval_interval=200, snapshot=1900)
model.graph_to_json(graph_config_file=NETWORK_FILE)
def embedding_training_cache_test(json_file, output_dir):
dataset = [("file_list." + str(i) + ".txt",
"file_list." + str(i) + ".keyset") for i in range(5)]
solver = hugectr.CreateSolver(batchsize=16384,
batchsize_eval=16384,
vvgpu=[[0]],
use_mixed_precision=False,
i64_input_key=False,
use_algorithm_search=True,
use_cuda_graph=True,
repeat_dataset=False)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Norm,
source=["./file_list.0.txt"],
keyset=["./file_list.0.keyset"],
eval_source="./file_list.5.txt",
check_type=hugectr.Check_t.Sum)
optimizer = hugectr.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.Adam)
etc = hugectr.CreateETC(
ps_types=[hugectr.TrainPSType_t.Staged, hugectr.TrainPSType_t.Staged],
sparse_models=[
output_dir + "/wdl_0_sparse_model",
output_dir + "/wdl_1_sparse_model"
])
model = hugectr.Model(solver, reader, optimizer, etc)
model.construct_from_json(graph_config_file=json_file,
include_dense_network=True)
model.compile()
model.summary()
lr_sch = model.get_learning_rate_scheduler()
data_reader_train = model.get_data_reader_train()
data_reader_eval = model.get_data_reader_eval()
embedding_training_cache = model.get_embedding_training_cache()
data_reader_eval.set_source("file_list.5.txt")
data_reader_eval_flag = True
iteration = 0
for file_list, keyset_file in dataset:
data_reader_train.set_source(file_list)
data_reader_train_flag = True
embedding_training_cache.update(keyset_file)
while True:
lr = lr_sch.get_next()
model.set_learning_rate(lr)
data_reader_train_flag = model.train(False)
if not data_reader_train_flag:
break
if iteration % 100 == 0:
batches = 0
while data_reader_eval_flag:
if batches >= solver.max_eval_batches:
break
data_reader_eval_flag = model.eval()
batches += 1
if not data_reader_eval_flag:
data_reader_eval.set_source()
data_reader_eval_flag = True
metrics = model.get_eval_metrics()
print("[HUGECTR][INFO] iter: {}, metrics: {}".format(
iteration, metrics))
iteration += 1
print("[HUGECTR][INFO] trained with data in {}".format(file_list))
updated_model = model.get_incremental_model()
model.save_params_to_files("wdl", iteration)
def set_source_raw_test(json_file):
train_data = "./train_data.bin"
test_data = "./test_data.bin"
solver = hugectr.CreateSolver(max_eval_batches=5441,
batchsize_eval=16384,
batchsize=16384,
vvgpu=[[0, 1, 2, 3, 4, 5, 6, 7]],
lr=24.0,
warmup_steps=8000,
decay_start=480000000,
decay_steps=240000000,
decay_power=2.0,
end_lr=0,
i64_input_key=False,
use_mixed_precision=True,
scaler=1024,
repeat_dataset=False,
use_cuda_graph=True)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Raw,
source=[train_data],
eval_source=test_data,
check_type=hugectr.Check_t.Non,
num_samples=4195197692,
eval_num_samples=89137319,
cache_eval_data=1361)
optimizer = hugectr.CreateOptimizer(optimizer_type=hugectr.Optimizer_t.SGD,
atomic_update=True)
model = hugectr.Model(solver, reader, optimizer)
model.construct_from_json(graph_config_file=json_file,
include_dense_network=True)
model.compile()
model.summary()
lr_sch = model.get_learning_rate_scheduler()
data_reader_train = model.get_data_reader_train()
data_reader_eval = model.get_data_reader_eval()
data_reader_eval.set_source(test_data)
data_reader_eval_flag = True
iteration = 1
for cnt in range(2):
data_reader_train.set_source(train_data)
data_reader_train_flag = True
print("[HUGECTR][INFO] round: {}".format(cnt), flush=True)
while True:
lr = lr_sch.get_next()
model.set_learning_rate(lr)
data_reader_train_flag = model.train(False)
if not data_reader_train_flag:
break
if iteration % 4000 == 0:
batches = 0
while data_reader_eval_flag:
if batches >= solver.max_eval_batches:
break
data_reader_eval_flag = model.eval()
batches += 1
if not data_reader_eval_flag:
data_reader_eval.set_source()
data_reader_eval_flag = True
metrics = model.get_eval_metrics()
print("[HUGECTR][INFO] iter: {}, metrics: {}".format(
iteration, metrics),
flush=True)
iteration += 1
print("[HUGECTR][INFO] trained with data in {}".format(train_data),
flush=True)
def train(input_train, input_val, max_iter,
batchsize, snapshot, num_gpus, eval_interval,
dense_model_file, sparse_model_files):
logging.info(f"GPU Devices: {num_gpus}")
# Configure and define the HugeCTR model
solver = hugectr.solver_parser_helper(num_epochs = 0,
max_iter = max_iter,
max_eval_batches = 100,
batchsize_eval = batchsize,
batchsize = batchsize,
model_file = dense_model_file,
embedding_files = sparse_model_files,
display = 200,
eval_interval = eval_interval,
i64_input_key = True,
use_mixed_precision = False,
repeat_dataset = True,
snapshot = snapshot,
vvgpu = [num_gpus],
use_cuda_graph = False
)
optimizer = hugectr.optimizer.CreateOptimizer(optimizer_type = hugectr.Optimizer_t.Adam,
use_mixed_precision = False)
model = hugectr.Model(solver, optimizer)
# The slot_size_array are the cardinalities of each categorical feature after NVTabular preprocessing
model.add(hugectr.Input(data_reader_type = hugectr.DataReaderType_t.Parquet,
source = input_train,
eval_source = input_val,
check_type = hugectr.Check_t.Non,
label_dim = 1, label_name = "label",
dense_dim = 13, dense_name = "dense",
slot_size_array = [18576837, 29428, 15128, 7296, 19902, 4, 6466, 1311, 62, 11700067, 622921, 219557, 11, 2209, 9780, 71, 4, 964, 15, 22022124, 4384510, 15960286, 290588, 10830, 96, 35],
data_reader_sparse_param_array =
[hugectr.DataReaderSparseParam(hugectr.DataReaderSparse_t.Distributed, 30, 1, 26)],
sparse_names = ["data1"]))
# Sparse Embedding Layer
model.add(hugectr.SparseEmbedding(embedding_type = hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
max_vocabulary_size_per_gpu = 88656602,
embedding_vec_size = 16,
combiner = 0,
sparse_embedding_name = "sparse_embedding1",
bottom_name = "data1"))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Reshape,
bottom_names = ["sparse_embedding1"],
top_names = ["reshape1"],
leading_dim=416))
# Concatenate sparse embedding and dense input
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Concat,
bottom_names = ["reshape1", "dense"], top_names = ["concat1"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Slice,
bottom_names = ["concat1"],
top_names = ["slice11", "slice12"],
ranges=[(0,429),(0,429)]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.MultiCross,
bottom_names = ["slice11"],
top_names = ["multicross1"],
num_layers=6))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
bottom_names = ["slice12"],
top_names = ["fc1"],
num_output=1024))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.ReLU,
bottom_names = ["fc1"],
top_names = ["relu1"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Dropout,
bottom_names = ["relu1"],
top_names = ["dropout1"],
dropout_rate=0.5))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
bottom_names = ["dropout1"],
top_names = ["fc2"],
num_output=1024))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.ReLU,
bottom_names = ["fc2"],
top_names = ["relu2"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Dropout,
bottom_names = ["relu2"],
top_names = ["dropout2"],
dropout_rate=0.5))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.Concat,
bottom_names = ["dropout2", "multicross1"],
top_names = ["concat2"]))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.InnerProduct,
bottom_names = ["concat2"],
top_names = ["fc3"],
num_output=1))
model.add(hugectr.DenseLayer(layer_type = hugectr.Layer_t.BinaryCrossEntropyLoss,
bottom_names = ["fc3", "label"],
top_names = ["loss"]))
model.compile()
model.summary()
model.fit()
def train_hugectr(workflow, devices, out_path):
# Gets embeddings and devices
embeddings = list(get_embedding_sizes(workflow).values())
embeddings = [emb[0] for emb in embeddings]
devices = [[int(d)] for d in list(devices)[0::2]]
# Set solver and model
solver = hugectr.solver_parser_helper(
vvgpu=[[0]],
max_iter=10000,
max_eval_batches=100,
batchsize_eval=2720,
batchsize=2720,
display=1000,
eval_interval=3200,
snapshot=3200,
i64_input_key=True,
use_mixed_precision=False,
repeat_dataset=True,
)
optimizer = hugectr.optimizer.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.SGD, use_mixed_precision=False)
model = hugectr.Model(solver, optimizer)
model.add(
hugectr.Input(
data_reader_type=hugectr.DataReaderType_t.Parquet,
source=out_path + "/output/train/_file_list.txt",
eval_source=out_path + "/output/valid/_file_list.txt",
check_type=hugectr.Check_t.Non,
label_dim=1,
label_name="label",
dense_dim=13,
dense_name="dense",
slot_size_array=embeddings,
data_reader_sparse_param_array=[
hugectr.DataReaderSparseParam(
hugectr.DataReaderSparse_t.Localized, 26, 1, 26)
],
sparse_names=["data1"],
))
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.
LocalizedSlotSparseEmbeddingHash,
max_vocabulary_size_per_gpu=15500000,
embedding_vec_size=128,
combiner=0,
sparse_embedding_name="sparse_embedding1",
bottom_name="data1",
))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dense"],
top_names=["fc1"],
num_output=512,
))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc1"],
top_names=["relu1"]))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu1"],
top_names=["fc2"],
num_output=256,
))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc2"],
top_names=["relu2"]))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu2"],
top_names=["fc3"],
num_output=128,
))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc3"],
top_names=["relu3"]))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Interaction,
bottom_names=["relu3", "sparse_embedding1"],
top_names=["interaction1"],
))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["interaction1"],
top_names=["fc4"],
num_output=1024,
))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc4"],
top_names=["relu4"]))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu4"],
top_names=["fc5"],
num_output=1024,
))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc5"],
top_names=["relu5"]))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu5"],
top_names=["fc6"],
num_output=512,
))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc6"],
top_names=["relu6"]))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu6"],
top_names=["fc7"],
num_output=256,
))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc7"],
top_names=["relu7"]))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu7"],
top_names=["fc8"],
num_output=1,
))
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,
bottom_names=["fc8", "label"],
top_names=["loss"],
))
# Run training
model.compile()
model.summary()
model.fit()
def DLRM(args):
vvgpu = [[g for g in range(args.gpu_num_per_node)]
for n in range(args.num_nodes)]
solver = hugectr.CreateSolver(max_eval_batches=args.eval_batchs,
batchsize_eval=args.batch_size,
batchsize=args.batch_size,
lr=args.learning_rate,
warmup_steps=args.warmup_steps,
decay_start=args.decay_start,
decay_steps=args.decay_steps,
decay_power=args.decay_power,
end_lr=args.end_lr,
vvgpu=vvgpu,
repeat_dataset=True)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Raw,
source=[f"{args.data_dir}/train_data.bin"],
eval_source=f"{args.data_dir}/test_data.bin",
num_samples=36672493,
eval_num_samples=4584062,
check_type=hugectr.Check_t.Non)
optimizer = hugectr.CreateOptimizer(optimizer_type=hugectr.Optimizer_t.SGD,
update_type=hugectr.Update_t.Local,
atomic_update=True)
model = hugectr.Model(solver, reader, optimizer)
model.add(
hugectr.Input(label_dim=1,
label_name="label",
dense_dim=13,
dense_name="dense",
data_reader_sparse_param_array=[
hugectr.DataReaderSparseParam("data1", 2, False, 26)
]))
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.
LocalizedSlotSparseEmbeddingOneHot,
slot_size_array=[
1460, 583, 10131227, 2202608, 305, 24, 12517, 633, 3, 93145,
5683, 8351593, 3194, 27, 14992, 5461306, 10, 5652, 2173, 4,
7046547, 18, 15, 286181, 105, 142572
],
workspace_size_per_gpu_in_mb=args.workspace_size_per_gpu_in_mb,
embedding_vec_size=args.embedding_vec_size,
combiner="sum",
sparse_embedding_name="sparse_embedding1",
bottom_name="data1",
optimizer=optimizer))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dense"],
top_names=["fc1"],
num_output=512))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc1"],
top_names=["relu1"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu1"],
top_names=["fc2"],
num_output=256))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc2"],
top_names=["relu2"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu2"],
top_names=["fc3"],
num_output=128))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc3"],
top_names=["relu3"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.Interaction,
bottom_names=["relu3", "sparse_embedding1"],
top_names=["interaction1"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["interaction1"],
top_names=["fc4"],
num_output=1024))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc4"],
top_names=["relu4"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu4"],
top_names=["fc5"],
num_output=1024))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc5"],
top_names=["relu5"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu5"],
top_names=["fc6"],
num_output=512))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc6"],
top_names=["relu6"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu6"],
top_names=["fc7"],
num_output=256))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.ReLU,
bottom_names=["fc7"],
top_names=["relu7"]))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["relu7"],
top_names=["fc8"],
num_output=1))
model.add(
hugectr.DenseLayer(layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,
bottom_names=["fc8", "label"],
top_names=["loss"]))
return model
def create_din(solver):
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Parquet,
source=["./din_data/train/_file_list.txt"],
eval_source="./din_data/valid/_file_list.txt",
check_type=hugectr.Check_t.Non,
num_workers=1,
slot_size_array=[
192403,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
63001,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
801,
],
)
optimizer = hugectr.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.Adam,
update_type=hugectr.Update_t.Global,
beta1=0.9,
beta2=0.999,
epsilon=0.000000001,
)
model = hugectr.Model(solver, reader, optimizer)
model.add(
hugectr.Input(
label_dim=1,
label_name="label",
dense_dim=0,
dense_name="dense",
data_reader_sparse_param_array=[
hugectr.DataReaderSparseParam("UserID", 1, True, 1),
hugectr.DataReaderSparseParam("GoodID", 1, True, 11),
hugectr.DataReaderSparseParam("CateID", 1, True, 11),
],
)
)
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
workspace_size_per_gpu_in_mb=28,
embedding_vec_size=18,
combiner="sum",
sparse_embedding_name="sparse_embedding_user",
bottom_name="UserID",
optimizer=optimizer,
)
)
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
workspace_size_per_gpu_in_mb=24,
embedding_vec_size=18,
combiner="sum",
sparse_embedding_name="sparse_embedding_good",
bottom_name="GoodID",
optimizer=optimizer,
)
)
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
workspace_size_per_gpu_in_mb=10,
embedding_vec_size=18,
combiner="sum",
sparse_embedding_name="sparse_embedding_cate",
bottom_name="CateID",
optimizer=optimizer,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.FusedReshapeConcat,
bottom_names=["sparse_embedding_good", "sparse_embedding_cate"],
top_names=["FusedReshapeConcat_item_his_em", "FusedReshapeConcat_item"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Slice,
bottom_names=["FusedReshapeConcat_item"],
top_names=["item1", "item2"],
ranges=[(0, 36), (0, 36)],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Slice,
bottom_names=["FusedReshapeConcat_item_his_em"],
top_names=["item_his1", "item_his2", "item_his3", "item_his4", "item_his5"],
ranges=[(0, 36), (0, 36), (0, 36), (0, 36), (0, 36)],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Scale,
bottom_names=["item1"],
top_names=["Scale_item"],
axis=1,
factor=10,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Slice,
bottom_names=["Scale_item"],
top_names=["Scale_item1", "Scale_item2", "Scale_item3"],
ranges=[(0, 36), (0, 36), (0, 36)],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Sub,
bottom_names=["Scale_item1", "item_his1"],
top_names=["sub_ih"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ElementwiseMultiply,
bottom_names=["Scale_item2", "item_his2"],
top_names=["ElementWiseMul_i"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Concat,
bottom_names=["Scale_item3", "item_his3", "sub_ih", "ElementWiseMul_i"],
top_names=["concat_i_h"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["concat_i_h"],
top_names=["fc_att_i2"],
num_output=40,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["fc_att_i2"],
top_names=["fc_att_i3"],
num_output=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["fc_att_i3"],
top_names=["reshape_score"],
leading_dim=10,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Softmax,
bottom_names=["reshape_score"],
top_names=["softmax_att_i"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Scale,
bottom_names=["softmax_att_i"],
top_names=["Scale_i"],
axis=0,
factor=36,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["item_his4"],
top_names=["reshape_item_his"],
leading_dim=360,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ElementwiseMultiply, # matmul
bottom_names=["Scale_i", "reshape_item_his"],
top_names=["ElementwiseMul_ih"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReduceSum,
bottom_names=["ElementwiseMul_ih"],
top_names=["reduce_ih"],
axis=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["item_his5"],
top_names=["reshape_his"],
leading_dim=36,
time_step=10,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReduceMean,
bottom_names=["reshape_his"],
top_names=["reduce_item_his"],
axis=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["reduce_item_his"],
top_names=["reshape_reduce_item_his"],
leading_dim=36,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["sparse_embedding_user"],
top_names=["reshape_user"],
leading_dim=18,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Concat,
bottom_names=[
"reshape_user",
"reshape_reduce_item_his",
"reduce_ih",
"item2",
],
top_names=["concat_din_i"],
)
)
# build_fcn_net
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["concat_din_i"],
top_names=["fc_din_i1"],
num_output=200,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.PReLU_Dice,
bottom_names=["fc_din_i1"],
top_names=["dice_1"],
elu_alpha=0.2,
eps=1e-8,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dice_1"],
top_names=["fc_din_i2"],
num_output=80,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.PReLU_Dice,
bottom_names=["fc_din_i2"],
top_names=["dice_2"],
elu_alpha=0.2,
eps=1e-8,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dice_2"],
top_names=["fc3"],
num_output=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,
bottom_names=["fc3", "label"],
top_names=["loss"],
)
)
return model
def create_deepfm(solver):
dataset_path = os.getenv("DCN_DATA_PATH")
os.symlink(dataset_path, "./dcn_data", target_is_directory=True)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Norm,
source=["./dcn_data/file_list.txt"],
eval_source="./dcn_data/file_list_test.txt",
check_type=hugectr.Check_t.Sum,
)
optimizer = hugectr.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.SGD,
update_type=hugectr.Update_t.Local,
atomic_update=True,
)
model = hugectr.Model(solver, reader, optimizer)
model.add(
hugectr.Input(
label_dim=1,
label_name="label",
dense_dim=13,
dense_name="dense",
data_reader_sparse_param_array=[
hugectr.DataReaderSparseParam("data1", 2, False, 26)
],
)
)
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
workspace_size_per_gpu_in_mb=61,
embedding_vec_size=11,
combiner="sum",
sparse_embedding_name="sparse_embedding1",
bottom_name="data1",
optimizer=optimizer,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["sparse_embedding1"],
top_names=["reshape1"],
leading_dim=11,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Slice,
bottom_names=["reshape1"],
top_names=["slice11", "slice12"],
ranges=[(0, 10), (10, 11)],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["slice11"],
top_names=["reshape2"],
leading_dim=260,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["slice12"],
top_names=["reshape3"],
leading_dim=26,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.WeightMultiply,
bottom_names=["dense"],
top_names=["weight_multiply1"],
weight_dims=[13, 10],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.WeightMultiply,
bottom_names=["dense"],
top_names=["weight_multiply2"],
weight_dims=[13, 1],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Concat,
bottom_names=["reshape2", "weight_multiply1"],
top_names=["concat1"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["concat1"],
top_names=["fc1"],
num_output=400,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReLU, bottom_names=["fc1"], top_names=["relu1"]
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Dropout,
bottom_names=["relu1"],
top_names=["dropout1"],
dropout_rate=0.5,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dropout1"],
top_names=["fc2"],
num_output=400,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReLU, bottom_names=["fc2"], top_names=["relu2"]
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Dropout,
bottom_names=["relu2"],
top_names=["dropout2"],
dropout_rate=0.5,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dropout2"],
top_names=["fc3"],
num_output=400,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReLU, bottom_names=["fc3"], top_names=["relu3"]
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Dropout,
bottom_names=["relu3"],
top_names=["dropout3"],
dropout_rate=0.5,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dropout3"],
top_names=["fc4"],
num_output=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.FmOrder2,
bottom_names=["concat1"],
top_names=["fmorder2"],
out_dim=10,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReduceSum,
bottom_names=["fmorder2"],
top_names=["reducesum1"],
axis=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Concat,
bottom_names=["reshape3", "weight_multiply2"],
top_names=["concat2"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReduceSum,
bottom_names=["concat2"],
top_names=["reducesum2"],
axis=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Add,
bottom_names=["fc4", "reducesum1", "reducesum2"],
top_names=["add"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,
bottom_names=["add", "label"],
top_names=["loss"],
)
)
return model
def create_dcn(solver):
dataset_path = os.getenv("DCN_DATA_PATH")
os.symlink(dataset_path, "./dcn_data", target_is_directory=True)
reader = hugectr.DataReaderParams(
data_reader_type=hugectr.DataReaderType_t.Norm,
source=["./dcn_data/file_list.txt"],
eval_source="./dcn_data/file_list_test.txt",
check_type=hugectr.Check_t.Sum,
)
optimizer = hugectr.CreateOptimizer(
optimizer_type=hugectr.Optimizer_t.SGD,
update_type=hugectr.Update_t.Local,
atomic_update=True,
)
model = hugectr.Model(solver, reader, optimizer)
model.add(
hugectr.Input(
label_dim=1,
label_name="label",
dense_dim=13,
dense_name="dense",
data_reader_sparse_param_array=[
hugectr.DataReaderSparseParam("data1", 2, False, 26)
],
)
)
model.add(
hugectr.SparseEmbedding(
embedding_type=hugectr.Embedding_t.DistributedSlotSparseEmbeddingHash,
workspace_size_per_gpu_in_mb=300,
embedding_vec_size=16,
combiner="sum",
sparse_embedding_name="sparse_embedding1",
bottom_name="data1",
optimizer=optimizer,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Reshape,
bottom_names=["sparse_embedding1"],
top_names=["reshape1"],
leading_dim=416,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Concat,
bottom_names=["reshape1", "dense"],
top_names=["concat1"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.MultiCross,
bottom_names=["concat1"],
top_names=["multicross1"],
num_layers=6,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["concat1"],
top_names=["fc1"],
num_output=1024,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReLU, bottom_names=["fc1"], top_names=["relu1"]
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Dropout,
bottom_names=["relu1"],
top_names=["dropout1"],
dropout_rate=0.5,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["dropout1"],
top_names=["fc2"],
num_output=1024,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.ReLU, bottom_names=["fc2"], top_names=["relu2"]
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Dropout,
bottom_names=["relu2"],
top_names=["dropout2"],
dropout_rate=0.5,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.Concat,
bottom_names=["dropout2", "multicross1"],
top_names=["concat2"],
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.InnerProduct,
bottom_names=["concat2"],
top_names=["fc3"],
num_output=1,
)
)
model.add(
hugectr.DenseLayer(
layer_type=hugectr.Layer_t.BinaryCrossEntropyLoss,
bottom_names=["fc3", "label"],
top_names=["loss"],
)
)
return model