def generate_din_feature_columns(data, sparse_features, dense_features):
feat_lbe_dict = get_glv('feat_lbe_dict')
sparse_feature_columns = [
SparseFeat(feat,
vocabulary_size=len(feat_lbe_dict[feat].classes_) + 1,
embedding_dim=EMBED_DIM)
for i, feat in enumerate(sparse_features) if feat not in time_feat
]
dense_feature_columns = [DenseFeat(
feat,
1,
) for feat in dense_features]
var_feature_columns = [
VarLenSparseFeat(SparseFeat(
'hist_item_id',
vocabulary_size=len(feat_lbe_dict['item_id'].classes_) + 1,
embedding_dim=EMBED_DIM,
embedding_name='item_id'),
maxlen=max_seq_len)
]
# DNN side
dnn_feature_columns = sparse_feature_columns + dense_feature_columns + var_feature_columns
# FM side
linear_feature_columns = sparse_feature_columns + dense_feature_columns + var_feature_columns
# all feature names
feature_names = get_feature_names(dnn_feature_columns +
linear_feature_columns)
return feature_names, linear_feature_columns, dnn_feature_columns
def _preprocess_criteo(df, **kw):
hash_feature = kw.get('hash_feature')
sparse_col = ['C' + str(i) for i in range(1, 27)]
dense_col = ['I' + str(i) for i in range(1, 14)]
df[sparse_col] = df[sparse_col].fillna('-1', )
df[dense_col] = df[dense_col].fillna(0, )
target = ["label"]
# set hashing space for each sparse field,and record dense feature field name
if hash_feature:
# Transformation for dense features
mms = MinMaxScaler(feature_range=(0, 1))
df[dense_col] = mms.fit_transform(df[dense_col])
sparse_col = ['C' + str(i) for i in range(1, 27)]
dense_col = ['I' + str(i) for i in range(1, 14)]
fixlen_cols = [SparseFeat(feat, vocabulary_size=1000, embedding_dim=4, use_hash=True, dtype='string')
# since the input is string
for feat in sparse_col] + [DenseFeat(feat, 1, ) for feat in dense_col]
else:
for feat in sparse_col:
lbe = LabelEncoder()
df[feat] = lbe.fit_transform(df[feat])
mms = MinMaxScaler(feature_range=(0, 1))
df[dense_col] = mms.fit_transform(df[dense_col])
fixlen_cols = [SparseFeat(feat, vocabulary_size=df[feat].nunique(), embedding_dim=4)
for i, feat in enumerate(sparse_col)] + [DenseFeat(feat, 1, ) for feat in dense_col]
linear_cols = fixlen_cols
dnn_cols = fixlen_cols
train, test = train_test_split(df, test_size=kw['test_size'])
return df, linear_cols, dnn_cols, train, test, target, test[target].values
def test_long_dense_vector():
feature_columns = [
SparseFeat(
'user_id',
4,
),
SparseFeat(
'item_id',
5,
),
DenseFeat("pic_vec", 5)
]
fixlen_feature_names = get_feature_names(feature_columns)
user_id = np.array([[1], [0], [1]])
item_id = np.array([[3], [2], [1]])
pic_vec = np.array([[0.1, 0.5, 0.4, 0.3, 0.2], [0.1, 0.5, 0.4, 0.3, 0.2],
[0.1, 0.5, 0.4, 0.3, 0.2]])
label = np.array([1, 0, 1])
input_dict = {'user_id': user_id, 'item_id': item_id, 'pic_vec': pic_vec}
model_input = [input_dict[name] for name in fixlen_feature_names]
model = DeepFM(feature_columns, feature_columns[:-1])
model.compile('adagrad', 'binary_crossentropy')
model.fit(model_input, label)
def get_xy_fd(hash_flag=False):
feature_columns = [SparseFeat('user', 3, hash_flag),
SparseFeat('gender', 2, hash_flag),
SparseFeat('item', 3 + 1, hash_flag),
SparseFeat('item_gender', 2 + 1, hash_flag),
DenseFeat('score', 1)]
feature_columns += [VarLenSparseFeat('sess_0_item',3+1,4,use_hash=hash_flag,embedding_name='item'),VarLenSparseFeat('sess_0_item_gender',2+1,4,use_hash=hash_flag,embedding_name='item_gender')]
feature_columns += [VarLenSparseFeat('sess_1_item', 3 + 1, 4, use_hash=hash_flag, embedding_name='item'),VarLenSparseFeat('sess_1_item_gender', 2 + 1, 4, use_hash=hash_flag,embedding_name='item_gender')]
behavior_feature_list = ["item", "item_gender"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3]) # 0 is mask value
igender = np.array([1, 2, 1]) # 0 is mask value
score = np.array([0.1, 0.2, 0.3])
sess1_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [0, 0, 0, 0]])
sess1_igender = np.array([[1, 1, 2, 0], [2, 1, 1, 0], [0, 0, 0, 0]])
sess2_iid = np.array([[1, 2, 3, 0], [0, 0, 0, 0], [0, 0, 0, 0]])
sess2_igender = np.array([[1, 1, 2, 0], [0, 0, 0, 0], [0, 0, 0, 0]])
sess_number = np.array([2, 1, 0])
feature_dict = {'user': uid, 'gender': ugender, 'item': iid, 'item_gender': igender,
'sess_0_item': sess1_iid, 'sess_0_item_gender': sess1_igender, 'score': score,
'sess_1_item': sess2_iid, 'sess_1_item_gender': sess2_igender, }
x = {name:feature_dict[name] for name in get_feature_names(feature_columns)}
x["sess_length"] = sess_number
y = [1, 0, 1]
return x, y, feature_columns, behavior_feature_list
def train_deepFM():
k = featureengineer.k
#缺失值填充+编码处理
data,appsnum, tags_nums = trainmodel.data,trainmodel.appsnum,trainmodel.tags_nums
data[trainmodel.sparse_features] = data[trainmodel.sparse_features].fillna('-1', )
for feat in trainmodel.dense_features:
data[feat].fillna(data[feat].dropna().mean(), inplace=True)
for feat in trainmodel.sparse_features:
data[feat] = data[feat].apply(lambda x:str(x))
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat])
mms = MinMaxScaler(feature_range=(0, 1))
data[trainmodel.dense_features] = mms.fit_transform(data[trainmodel.dense_features])
#数据格式转换
fixlen_feature_columns = [SparseFeat(feat, vocabulary_size=data[feat].max()+1, embedding_dim=8)
for i, feat in enumerate(trainmodel.sparse_features)] + \
[DenseFeat(feat, 1, ) for feat in trainmodel.dense_features]
lgbOut_feature_columns = [SparseFeat(feat, vocabulary_size=data[feat].max()+1, embedding_dim=1)
for i, feat in enumerate(trainmodel.lgbOut_Features)]
key2index_len = {'applist': appsnum+1, 'new_tag': tags_nums}
varlen_features = [VarLenSparseFeat('%s' % i, vocabulary_size=key2index_len[i], maxlen=k, embedding_dim=8, combiner='mean',weight_name=None) for i
in trainmodel.var_features]
dnn_feature_columns = fixlen_feature_columns + varlen_features
linear_feature_columns = fixlen_feature_columns + varlen_features + lgbOut_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)
sparse_dense_features = trainmodel.sparse_features + trainmodel.dense_features + trainmodel.lgbOut_Features
train, test = train_test_split(data, test_size=0.2)
train_model_input = {name: train[name] for name in sparse_dense_features}
test_model_input = {name: test[name] for name in sparse_dense_features}
for x in trainmodel.var_features:
if x == 'applist':
train_model_input[x] = np.array(train[x].tolist())
test_model_input[x] = np.array(test[x].tolist())
if x == 'new_tag':
train_model_input[x] = np.array(train[x].tolist())-appsnum
test_model_input[x] = np.array(test[x].tolist())-appsnum
# 模型
model = DeepFM(linear_feature_columns=linear_feature_columns, dnn_feature_columns=dnn_feature_columns,
dnn_hidden_units=(50, 30, 30), l2_reg_linear=0.001, l2_reg_embedding=0.001,
l2_reg_dnn=0, init_std=0.0001, seed=1024, dnn_dropout=0.1, dnn_activation='relu', dnn_use_bn=True,
task='binary')
model.compile("adam", "binary_crossentropy",metrics=['AUC'], )
history = model.fit(train_model_input, train['target'].values,
batch_size=256, epochs=1, verbose=2, validation_split=0.2, )
pred_ans = model.predict(test_model_input, batch_size=256)
print("test AUC", round(roc_auc_score(test['target'].values, pred_ans), 4))
def get_test_data(sample_size=1000,
sparse_feature_num=1,
dense_feature_num=1,
sequence_feature=('sum', 'mean', 'max'),
classification=True,
include_length=False,
hash_flag=False,
prefix=''):
feature_columns = []
for i in range(sparse_feature_num):
dim = np.random.randint(1, 10)
feature_columns.append(
SparseFeat(prefix + 'sparse_feature_' + str(i), dim, hash_flag,
tf.int32))
for i in range(dense_feature_num):
feature_columns.append(
DenseFeat(prefix + 'dense_feature_' + str(i), 1, tf.float32))
for i, mode in enumerate(sequence_feature):
dim = np.random.randint(1, 10)
maxlen = np.random.randint(1, 10)
feature_columns.append(
VarLenSparseFeat(prefix + 'sequence_' + str(i), dim, maxlen, mode))
model_input = []
sequence_input = []
sequence_len_input = []
for fc in feature_columns:
if isinstance(fc, SparseFeat):
model_input.append(np.random.randint(0, fc.dimension, sample_size))
elif isinstance(fc, DenseFeat):
model_input.append(np.random.random(sample_size))
else:
s_input, s_len_input = gen_sequence(fc.dimension, fc.maxlen,
sample_size)
sequence_input.append(s_input)
sequence_len_input.append(s_len_input)
if classification:
y = np.random.randint(0, 2, sample_size)
else:
y = np.random.random(sample_size)
x = model_input + sequence_input
if include_length:
for i, mode in enumerate(sequence_feature):
dim = np.random.randint(1, 10)
maxlen = np.random.randint(1, 10)
feature_columns.append(
SparseFeat(prefix + 'sequence_' + str(i) + '_seq_length',
1,
embedding=False))
x += sequence_len_input
return x, y, feature_columns
def get_test_data(sample_size=1000, embedding_size=4, sparse_feature_num=1, dense_feature_num=1,
sequence_feature=['sum', 'mean', 'max', 'weight'], classification=True, include_length=False,
hash_flag=False, prefix='', use_group=False):
feature_columns = []
model_input = {}
if 'weight' in sequence_feature:
feature_columns.append(
VarLenSparseFeat(SparseFeat(prefix + "weighted_seq", vocabulary_size=2, embedding_dim=embedding_size),
maxlen=3, length_name=prefix + "weighted_seq" + "_seq_length",
weight_name=prefix + "weight"))
s_input, s_len_input = gen_sequence(
2, 3, sample_size)
model_input[prefix + "weighted_seq"] = s_input
model_input[prefix + 'weight'] = np.random.randn(sample_size, 3, 1)
model_input[prefix + "weighted_seq" + "_seq_length"] = s_len_input
sequence_feature.pop(sequence_feature.index('weight'))
for i in range(sparse_feature_num):
if use_group:
group_name = str(i%3)
else:
group_name = DEFAULT_GROUP_NAME
dim = np.random.randint(1, 10)
feature_columns.append(
SparseFeat(prefix + 'sparse_feature_' + str(i), dim, embedding_size, use_hash=hash_flag, dtype=tf.int32,group_name=group_name))
for i in range(dense_feature_num):
feature_columns.append(DenseFeat(prefix + 'dense_feature_' + str(i), 1, dtype=tf.float32))
for i, mode in enumerate(sequence_feature):
dim = np.random.randint(1, 10)
maxlen = np.random.randint(1, 10)
feature_columns.append(
VarLenSparseFeat(SparseFeat(prefix + 'sequence_' + mode, vocabulary_size=dim, embedding_dim=embedding_size),
maxlen=maxlen, combiner=mode))
for fc in feature_columns:
if isinstance(fc, SparseFeat):
model_input[fc.name] = np.random.randint(0, fc.vocabulary_size, sample_size)
elif isinstance(fc, DenseFeat):
model_input[fc.name] = np.random.random(sample_size)
else:
s_input, s_len_input = gen_sequence(
fc.vocabulary_size, fc.maxlen, sample_size)
model_input[fc.name] = s_input
if include_length:
fc.length_name = prefix + "sequence_" + str(i) + '_seq_length'
model_input[prefix + "sequence_" + str(i) + '_seq_length'] = s_len_input
if classification:
y = np.random.randint(0, 2, sample_size)
else:
y = np.random.random(sample_size)
return model_input, y, feature_columns
def get_xy_fd(hash_flag=False):
# feature_dim_dict = {"sparse": [SingleFeat('user', 3, hash_flag), SingleFeat(
# 'gender', 2, hash_flag), SingleFeat('item', 3 + 1, hash_flag), SingleFeat('item_gender', 2 + 1, hash_flag)],
# "dense": [SingleFeat('score', 0)]}
feature_columns = [
SparseFeat('user', 3),
SparseFeat('gender', 2),
SparseFeat('item', 3 + 1),
SparseFeat('item_gender', 2 + 1),
DenseFeat('score', 0)
]
feature_columns += [
VarLenSparseFeat('hist_item', 3 + 1, maxlen=4, embedding_name='item'),
VarLenSparseFeat('hist_item_gender',
3 + 1,
maxlen=4,
embedding_name='item_gender')
]
behavior_feature_list = ["item", "item_gender"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3]) # 0 is mask value
igender = np.array([1, 2, 1]) # 0 is mask value
score = np.array([0.1, 0.2, 0.3])
hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
hist_igender = np.array([[1, 1, 2, 0], [2, 1, 1, 0], [2, 1, 0, 0]])
feature_dict = {
'user': uid,
'gender': ugender,
'item': iid,
'item_gender': igender,
'hist_item': hist_iid,
'hist_item_gender': hist_igender,
'score': score
}
feature_names = get_fixlen_feature_names(feature_columns)
varlen_feature_names = get_varlen_feature_names(feature_columns)
x = [feature_dict[name] for name in feature_names
] + [feature_dict[name] for name in varlen_feature_names]
# x = [feature_dict[feat.name] for feat in feature_dim_dict["sparse"]] + [feature_dict[feat.name] for feat in
# feature_dim_dict["dense"]] + [
# feature_dict['hist_' + feat] for feat in behavior_feature_list]
y = [1, 0, 1]
return x, y, feature_columns, behavior_feature_list
def get_xy_fd(use_neg=False, hash_flag=False):
feature_columns = [SparseFeat('user', 3,hash_flag),
SparseFeat('gender', 2,hash_flag),
SparseFeat('item', 3+1,hash_flag),
SparseFeat('item_gender', 2+1,hash_flag),
DenseFeat('score', 1)]
feature_columns += [VarLenSparseFeat('hist_item',3+1, maxlen=4, embedding_name='item'),
VarLenSparseFeat('hist_item_gender',3+1, maxlen=4, embedding_name='item_gender')]
behavior_feature_list = ["item","item_gender"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3])#0 is mask value
igender = np.array([1, 2, 1])# 0 is mask value
score = np.array([0.1, 0.2, 0.3])
hist_iid = np.array([[ 1, 2, 3,0], [ 1, 2, 3,0], [ 1, 2, 0,0]])
hist_igender = np.array([[1, 1, 2,0 ], [2, 1, 1, 0], [2, 1, 0, 0]])
behavior_length = np.array([3,3,2])
feature_dict = {'user': uid, 'gender': ugender, 'item': iid, 'item_gender': igender,
'hist_item': hist_iid, 'hist_item_gender': hist_igender,
'score': score}
#x = [feature_dict[feat.name] for feat in feature_dim_dict["sparse"]] + [feature_dict[feat.name] for feat in
# feature_dim_dict["dense"]] + [
# feature_dict['hist_' + feat] for feat in behavior_feature_list]
if use_neg:
feature_dict['neg_hist_item'] = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
feature_dict['neg_hist_item_gender'] = np.array([[1, 1, 2, 0], [2, 1, 1, 0], [2, 1, 0, 0]])
feature_columns += [VarLenSparseFeat('neg_hist_item',3+1, maxlen=4, embedding_name='item'),
VarLenSparseFeat('neg_hist_item_gender',3+1, maxlen=4, embedding_name='item_gender')]
#x += [feature_dict['neg_hist_'+feat] for feat in behavior_feature_list]
feature_names = get_fixlen_feature_names(feature_columns)
varlen_feature_names = get_varlen_feature_names(feature_columns)
print(varlen_feature_names)
x = [feature_dict[name] for name in feature_names] + [feature_dict[name] for name in varlen_feature_names]
x += [behavior_length]
y = [1, 0, 1]
print(len(x))
return x, y, feature_columns, behavior_feature_list
def simple_pre(df):
# Label Encoding for sparse features,and normalization for dense numerical features
for feat in config.sparse_features:
lbe = LabelEncoder()
df[feat] = lbe.fit_transform(df[feat])
mms = MinMaxScaler(feature_range=(0, 1))
df[config.dense_features] = mms.fit_transform(df[config.dense_features])
#Generate feature columns
#For sparse features, we transform them into dense vectors by embedding techniques.
#For dense numerical features, we concatenate them to the input tensors of fully connected layer.
# count #unique features for each sparse field
fixlen_feature_columns = [
SparseFeat(feat, df[feat].nunique(), embedding_dim=4)
for feat in config.sparse_features
]
linear_feature_columns = fixlen_feature_columns
dnn_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns +
dnn_feature_columns)
return linear_feature_columns, dnn_feature_columns, feature_names
def get_xy_fd():
feature_columns = [
SparseFeat('user', 3, embedding_dim=10),
SparseFeat('gender', 2, embedding_dim=4),
SparseFeat('item', 3 + 1, embedding_dim=8),
SparseFeat('item_gender', 2 + 1, embedding_dim=4),
DenseFeat('score', 1)
]
feature_columns += [
VarLenSparseFeat('hist_item',
maxlen=4,
vocabulary_size=3 + 1,
embedding_dim=8,
embedding_name='item'),
VarLenSparseFeat('hist_item_gender',
maxlen=4,
vocabulary_size=3 + 1,
embedding_dim=4,
embedding_name='item_gender')
]
behavior_feature_list = ["item", "item_gender"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3]) # 0 is mask value
igender = np.array([1, 2, 1]) # 0 is mask value
score = np.array([0.1, 0.2, 0.3])
hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
hist_igender = np.array([[1, 1, 2, 0], [2, 1, 1, 0], [2, 1, 0, 0]])
feature_dict = {
'user': uid,
'gender': ugender,
'item': iid,
'item_gender': igender,
'hist_item': hist_iid,
'hist_item_gender': hist_igender,
'score': score
}
x = {
name: feature_dict[name]
for name in get_feature_names(feature_columns)
}
y = [1, 0, 1]
return x, y, feature_columns, behavior_feature_list
def load_stats(self):
fixlen_feature_columns = [SparseFeat(feat, self.cat_meta[feat])
for feat in self.sparse_features] + [DenseFeat(feat, 1,)
for feat in self.dense_features]
self.dnn_feature_columns = fixlen_feature_columns
self.linear_feature_columns = fixlen_feature_columns
self.fixlen_feature_names = get_fixlen_feature_names(self.linear_feature_columns + self.dnn_feature_columns)
def get_xy_fd():
feature_columns = [
SparseFeat('user', 3),
SparseFeat('gender', 2),
SparseFeat('item', 3 + 1),
SparseFeat('item_gender', 2 + 1),
DenseFeat('score', 1)
]
feature_columns += [
VarLenSparseFeat('hist_item', 3 + 1, maxlen=4, embedding_name='item'),
VarLenSparseFeat('hist_item_gender',
3 + 1,
maxlen=4,
embedding_name='item_gender')
]
behavior_feature_list = ["item", "item_gender"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3]) # 0 is mask value
igender = np.array([1, 2, 1]) # 0 is mask value
score = np.array([0.1, 0.2, 0.3])
hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
hist_igender = np.array([[1, 1, 2, 0], [2, 1, 1, 0], [2, 1, 0, 0]])
feature_dict = {
'user': uid,
'gender': ugender,
'item': iid,
'item_gender': igender,
'hist_item': hist_iid,
'hist_item_gender': hist_igender,
'score': score
}
fixlen_feature_names = get_fixlen_feature_names(feature_columns)
varlen_feature_names = get_varlen_feature_names(feature_columns)
x = [feature_dict[name] for name in fixlen_feature_names
] + [feature_dict[name] for name in varlen_feature_names]
y = [1, 0, 1]
return x, y, feature_columns, behavior_feature_list
def __init__(self, uNum, iNum, dim, maxlen):
self.uNum = uNum
self.iNum = iNum
self.dim = dim
self.maxlen = maxlen
hash_flag = True
self.feature_columns = [SparseFeat('user', self.uNum, hash_flag),
SparseFeat('item', self.iNum, hash_flag),
VarLenSparseFeat('sess_0_item', self.iNum, self.dim, use_hash=hash_flag,
embedding_name='item')]
self.behavior_feature_list = ["item"]
self.model = DSIN(self.feature_columns, self.behavior_feature_list, sess_max_count=1,
embedding_size=self.dim,
att_head_num=self.dim,
dnn_hidden_units=[self.dim, self.dim, self.dim], dnn_dropout=0.5)
self.model.compile('adam', 'binary_crossentropy',
metrics=['acc'])
def get_xy_fd(hash_flag=False):
user_feature_columns = [SparseFeat('user',3),SparseFeat(
'gender', 2),VarLenSparseFeat(SparseFeat('hist_item', vocabulary_size=3 + 1,embedding_dim=4,embedding_name='item'), maxlen=4,length_name="hist_len") ]
item_feature_columns = [SparseFeat('item', 3 + 1,embedding_dim=4,)]
uid = np.array([0, 1, 2,1])
ugender = np.array([0, 1, 0,1])
iid = np.array([1, 2, 3,1]) # 0 is mask value
hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0],[3, 0, 0, 0]])
hist_len = np.array([3,3,2,1])
feature_dict = {'user': uid, 'gender': ugender, 'item': iid,
'hist_item': hist_iid, "hist_len":hist_len}
#feature_names = get_feature_names(feature_columns)
x = feature_dict
y = np.array([1, 1, 1,1])
return x, y, user_feature_columns,item_feature_columns
def get_xy_fd(use_neg=False, hash_flag=False):
feature_columns = [SparseFeat('user', 3,hash_flag),
SparseFeat('gender', 2,hash_flag),
SparseFeat('item', 3+1,hash_flag),
SparseFeat('item_gender', 2+1,hash_flag),
DenseFeat('score', 1)]
feature_columns += [VarLenSparseFeat('hist_item',3+1, maxlen=4, embedding_name='item'),
VarLenSparseFeat('hist_item_gender',3+1, maxlen=4, embedding_name='item_gender')]
behavior_feature_list = ["item","item_gender"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3])#0 is mask value
igender = np.array([1, 2, 1])# 0 is mask value
score = np.array([0.1, 0.2, 0.3])
hist_iid = np.array([[ 1, 2, 3,0], [ 1, 2, 3,0], [ 1, 2, 0,0]])
hist_igender = np.array([[1, 1, 2,0 ], [2, 1, 1, 0], [2, 1, 0, 0]])
behavior_length = np.array([3,3,2])
feature_dict = {'user': uid, 'gender': ugender, 'item': iid, 'item_gender': igender,
'hist_item': hist_iid, 'hist_item_gender': hist_igender,
'score': score}
if use_neg:
feature_dict['neg_hist_item'] = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
feature_dict['neg_hist_item_gender'] = np.array([[1, 1, 2, 0], [2, 1, 1, 0], [2, 1, 0, 0]])
feature_columns += [VarLenSparseFeat('neg_hist_item',3+1, maxlen=4, embedding_name='item'),
VarLenSparseFeat('neg_hist_item_gender',3+1, maxlen=4, embedding_name='item_gender')]
x = {name:feature_dict[name] for name in get_feature_names(feature_columns)}
x["seq_length"] = behavior_length
y = [1, 0, 1]
return x, y, feature_columns, behavior_feature_list
def getFeatureColumns(self):
feature_columns = []
for feat, value in self.fixed_sparse_dict.items():
feature_columns.append(
SparseFeat(feat,
vocabulary_size=value[0],
embedding_dim=value[1]))
for feat, value in self.var_sparse_dict.items():
feature_columns.append(
VarLenSparseFeat(feat,
maxlen=value[0],
vocabulary_size=value[1],
embedding_dim=value[2],
embedding_name=value[3]))
return feature_columns
def main():
Use_SF = False
if len(sys.argv) > 0 and sys.argv[0] == 'SF':
Use_SF = True
train, vali, test = GetFeatures(Use_SF)
feature_count = []
for feat in sparse_features:
print("Fitting {}".format(feat))
labels = {}
for x in train[feat]:
if x not in labels:
labels[x] = len(labels) + 1
print("Transforming {}".format(feat))
for df in [train, vali, test]:
df[feat] = df[feat].map(lambda x: labels.get(x, 0))
feature_count.append(len(labels) + 1)
sparse_feature_columns = [
SparseFeat(f, f_c) for f, f_c in zip(sparse_features, feature_count)
]
dense_feature_columns = [DenseFeat(f, 1) for f in dense_features]
fixlen_feature_columns = sparse_feature_columns + dense_feature_columns
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns +
dnn_feature_columns)
train_model_input = [train[name] for name in fixlen_feature_names]
vali_model_input = [vali[name] for name in fixlen_feature_names]
test_model_input = [test[name] for name in fixlen_feature_names]
def eval(target):
model, history = model_generate(train_model_input, train[[target]],
vali_model_input, vali[[target]],
linear_feature_columns,
dnn_feature_columns)
pred_ans = model.predict(test_model_input, batch_size=256)
print(target + " test LogLoss",
round(log_loss(test[target].values, pred_ans), 4))
print(target + " test AUC",
round(roc_auc_score(test[target].values, pred_ans), 4))
for target in targets:
eval(target)
def read(data, lbe_store):
# data['time'] = data['time'].apply(lambda x: timestamp(x), convert_dtype='int32')
sparse_features = ["user_id", "item_id", "item_category", "time"]
# 1.Label Encoding for sparse features,and do simple Transformation for dense features
for feat in sparse_features:
data[feat] = lbe_store[feat].transform(data[feat])
# 2.count #unique features for each sparse field
fixlen_feature_columns = [
SparseFeat(feat, data[feat].nunique()) for feat in sparse_features
]
linear_feature_columns = fixlen_feature_columns
dnn_feature_columns = fixlen_feature_columns
fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns +
dnn_feature_columns)
data_model_input = [data[name].values for name in fixlen_feature_names]
return data, data_model_input
def get_xy_fd():
# 固定长度的离散特征
feature_columns = [
SparseFeat('user', 3, embedding_dim=10),
SparseFeat('gender', 2, embedding_dim=4),
SparseFeat('item_id', 3 + 1, embedding_dim=8),
SparseFeat('cate_id', 2 + 1, embedding_dim=4),
DenseFeat('pay_score', 1)
]
# 不固定长度的离散特征
feature_columns += [
VarLenSparseFeat(SparseFeat('hist_item_id',
vocabulary_size=3 + 1,
embedding_dim=8,
embedding_name='item_id'),
maxlen=4),
VarLenSparseFeat(SparseFeat('hist_cate_id',
2 + 1,
embedding_dim=4,
embedding_name='cate_id'),
maxlen=4)
]
behavior_feature_list = ["item", "cate_id"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3]) # 0 is mask value
cate_id = np.array([1, 2, 1]) # 0 is mask value
pay_score = np.array([0.1, 0.2, 0.3])
hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
hist_cate_id = np.array([[1, 1, 2, 0], [2, 1, 1, 0], [2, 1, 0, 0]])
feature_dict = {
'user': uid,
'gender': ugender,
'item_id': iid,
'cate_id': cate_id,
'hist_item_id': hist_iid,
'hist_cate_id': hist_cate_id,
'pay_score': pay_score
}
x = {
name: feature_dict[name]
for name in get_feature_names(feature_columns)
}
print('x=', x)
y = [1, 0, 1]
return x, y, feature_columns, behavior_feature_list
def prepare_data(cls, path, sparse_features, task='binary'):
data_path = path
dataframe = pd.read_csv(
data_path, names='user_id,movie_id,rating,timestamp'.split(','))
sparse_features = sparse_features # ["movie_id", "user_id"]
y = ['rating']
for feat in sparse_features:
lbe = LabelEncoder()
dataframe[feat] = lbe.fit_transform(dataframe[feat])
#feature_columns = [DenseFeat(feat, dataframe[feat].nunique()) for feat in sparse_features]
feature_columns = [
SparseFeat(feat, dataframe[feat].nunique())
for feat in sparse_features
]
trainset, testset = train_test_split(dataframe, test_size=0.2)
#train_model_input = [to_categorical(trainset[fc.name].values, num_classes= fc.dimension) for fc in feature_columns]#includes values from only data[user_id], data[movie_id]
#test_model_input = [to_categorical(testset[fc.name].values, num_classes= fc.dimension) for fc in feature_columns]#includes values from only data[user_id], data[movie_id]
train_model_input = [
trainset[name].values for name in sparse_features
] #includes values from only data[user_id], data[movie_id]
test_model_input = [
testset[name].values for name in sparse_features
] #includes values from only data[user_id], data[movie_id]
if task == 'binary':
train_lbl = trainset[y].values
test_lbl = testset[y].values
#elif task == 'multiclass':
# train_lbl = to_categorical(trainset[y])[:,1:]#stripping 0th column as rating is (1,5)
# test_lbl= to_categorical(testset[y])[:,1:]#stripping 0th column as rating is (1,5)
#else:
# raise ValueError("Enter task either 'binary' or 'multiclass'")
return cls(feature_columns), (train_model_input, train_lbl), (
test_model_input, test_lbl
) #try returning train_model_input from inside __init__()
def prepare(self):
if self.data_format == "deepctr":
# 1.Label Encoding for sparse features, and
# simple Transformation for dense features
for feat in self.sparse_features:
lbe = LabelEncoder()
self.input[feat] = lbe.fit_transform(self.input[feat])
self.encoders[feat] = lbe
# 2.count #unique features for each sparse field
fixlen_feature_columns = [
SparseFeat(feat, self.input[feat].nunique(), embedding_dim=4)
for feat in self.sparse_features
]
self.linear_feature_columns = fixlen_feature_columns
self.dnn_feature_columns = fixlen_feature_columns
self.feature_names = get_feature_names(
self.linear_feature_columns + self.dnn_feature_columns)
# 3.generate input data for model
train, test = train_test_split(self.input,
test_size=self.test_size)
self.X_train = {
name: train[name].values
for name in self.feature_names
}
self.y_train = train[self.target].values
self.X_test = {
name: test[name].values
for name in self.feature_names
}
self.y_test = test[self.target].values
else:
raise ("Not supported dataset:" + self.data_format)
def client_restful_criteo():
data = pd.read_csv('./data/criteo_sample.txt')
sparse_features = ['C' + str(i) for i in range(1, 27)]
dense_features = ['I' + str(i) for i in range(1, 14)]
data[sparse_features] = data[sparse_features].fillna('-1', )
data[dense_features] = data[dense_features].fillna(0, )
target = ['label']
# 1.Label Encoding for sparse features,and do simple Transformation for dense features
for feat in sparse_features:
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat])
mms = MinMaxScaler(feature_range=(0, 1))
data[dense_features] = mms.fit_transform(data[dense_features])
# 2.count #unique features for each sparse field,and record dense feature field name
fixlen_feature_columns = [SparseFeat(feat, vocabulary_size=data[feat].nunique(),embedding_dim=4)
for i,feat in enumerate(sparse_features)] + [DenseFeat(feat, 1,)
for feat in dense_features]
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)
# model_input = [data[name].iloc[0] for name in feature_names]
# model_input = [{name:data[name].iloc[0]} for name in feature_names]
model_input = [{name:data[name].iloc[0] for name in feature_names}]
print(model_input)
data = json.dumps({"signature_name": "serving_default", "instances": model_input}, cls=NpEncoder)
headers = {"content-type": "application/json"}
json_response = requests.post('http://localhost:8501/v1/models/criteo:predict', data=data, headers=headers)
json_response = json.loads(json_response.text)
print(json_response)
def get_xy_fd_sdm(hash_flag=False):
user_feature_columns = [SparseFeat('user',3),
SparseFeat('gender', 2),
VarLenSparseFeat(SparseFeat('prefer_item', vocabulary_size=100,embedding_dim=8,
embedding_name='item'), maxlen=6, length_name="prefer_sess_length"),
VarLenSparseFeat(SparseFeat('prefer_cate', vocabulary_size=100, embedding_dim=8,
embedding_name='cate'), maxlen=6, length_name="prefer_sess_length"),
VarLenSparseFeat(SparseFeat('short_item', vocabulary_size=100,embedding_dim=8,
embedding_name='item'), maxlen=4, length_name="short_sess_length"),
VarLenSparseFeat(SparseFeat('short_cate', vocabulary_size=100, embedding_dim=8,
embedding_name='cate'), maxlen=4, length_name="short_sess_length"),
]
item_feature_columns = [SparseFeat('item', 100, embedding_dim=8,)]
uid = np.array([0, 1, 2, 1])
ugender = np.array([0, 1, 0, 1])
iid = np.array([1, 2, 3, 1]) # 0 is mask value
prefer_iid = np.array([[1, 2, 3, 4, 5, 6], [1, 2, 3, 4, 5, 0], [1, 2, 3, 3, 0, 0], [1, 2, 4, 0, 0, 0]])
prefer_cate = np.array([[1, 2, 3, 4, 5, 6], [1, 2, 3, 4, 5, 0], [1, 2, 3, 3, 0, 0], [1, 2, 4, 0, 0, 0]])
short_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0], [3, 0, 0, 0]])
short_cate = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0], [3, 0, 0, 0]])
prefer_len = np.array([6, 5, 4, 3])
short_len = np.array([3, 3, 2, 1])
feature_dict = {'user': uid, 'gender': ugender, 'item': iid, 'prefer_item': prefer_iid, "prefer_cate":prefer_cate,
'short_item': short_iid, 'short_cate': short_cate, 'prefer_sess_length': prefer_len, 'short_sess_length':short_len}
#feature_names = get_feature_names(feature_columns)
x = feature_dict
y = np.array([1, 1, 1, 0])
history_feature_list = ['item', 'cate']
return x, y, user_feature_columns, item_feature_columns, history_feature_list
data[sparse_features] = data[sparse_features].fillna('-1', )
data[dense_features] = data[dense_features].fillna(0, )
target = ['label']
# 1.Label Encoding for sparse features,and do simple Transformation for dense features
for feat in sparse_features:
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat])
mms = MinMaxScaler(feature_range=(0, 1))
data[dense_features] = mms.fit_transform(data[dense_features])
# 2.count #unique features for each sparse field,and record dense feature field name
fixlen_feature_columns = [
SparseFeat(feat, data[feat].nunique()) for feat in sparse_features
] + [DenseFeat(
feat,
1,
) for feat in dense_features]
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns +
dnn_feature_columns)
# 3.generate input data for model
train, test = train_test_split(data, test_size=0.2)
from sklearn.preprocessing import LabelEncoder
from deepctr.models import DeepFM
from deepctr.inputs import SparseFeat, get_feature_names
data = pd.read_csv("ratings.csv")
sparse_features = ["userId", "movieId", 'timestamp']
target = ['rating']
# 对特征标签进行编码
for feature in sparse_features:
lbe = LabelEncoder()
data[feature] = lbe.fit_transform(data[feature])
# 计算每个特征中的 不同特征值的个数
fixlen_feature_columns = [
SparseFeat(feature, data[feature].nunique()) for feature in sparse_features
]
linear_feature_columns = fixlen_feature_columns
dnn_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)
# 将数据集切分成训练集和测试集
train, test = train_test_split(data, test_size=0.2)
train_model_input = {name: train[name].values for name in feature_names}
test_model_input = {name: test[name].values for name in feature_names}
# 使用DeepFM进行训练
model = DeepFM(linear_feature_columns, dnn_feature_columns, task='regression')
model.compile(
"adam",
"mse",
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from deepctr.models import WDL
from deepctr.inputs import SparseFeat,get_feature_names
import matplotlib.pyplot as plt
data = pd.read_csv("movielens_sample.txt")
sparse_features = ["movie_id", "user_id", "gender", "age", "occupation", "zip"]
target = ['rating']
# 对特征标签进行编码
for feature in sparse_features:
lbe = LabelEncoder()
data[feature] = lbe.fit_transform(data[feature])
# 计算每个特征中的 不同特征值的个数
fixlen_feature_columns = [SparseFeat(feature, data[feature].nunique()) for feature in sparse_features]
linear_feature_columns = fixlen_feature_columns
dnn_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)
# 将数据集切分成训练集和测试集
train, test = train_test_split(data, test_size=0.2)
train_model_input = {name:train[name].values for name in feature_names}
test_model_input = {name:test[name].values for name in feature_names}
# 使用WDL进行训练
model = WDL(linear_feature_columns, dnn_feature_columns, task='regression')
model.compile("adam", "mse", metrics=['mse'], )
history = model.fit(train_model_input, train[target].values, batch_size=256, epochs=100, verbose=True, validation_split=0.2, )
plt.figure()
x = range(len(history.history['loss']))
def get_xy_fd(use_neg=False, hash_flag=False):
feature_columns = [
SparseFeat('user', 3, embedding_dim=10, use_hash=hash_flag),
SparseFeat('gender', 2, embedding_dim=4, use_hash=hash_flag),
SparseFeat('item_id', 3 + 1, embedding_dim=8, use_hash=hash_flag),
SparseFeat('cate_id', 2 + 1, embedding_dim=4, use_hash=hash_flag),
DenseFeat('pay_score', 1)
]
feature_columns += [
VarLenSparseFeat(SparseFeat('hist_item_id',
vocabulary_size=3 + 1,
embedding_dim=8,
embedding_name='item_id'),
maxlen=4,
length_name="seq_length"),
VarLenSparseFeat(SparseFeat('hist_cate_id',
2 + 1,
embedding_dim=4,
embedding_name='cate_id'),
maxlen=4,
length_name="seq_length")
]
behavior_feature_list = ["item_id", "cate_id"]
uid = np.array([0, 1, 2])
ugender = np.array([0, 1, 0])
iid = np.array([1, 2, 3]) # 0 is mask value
cate_id = np.array([1, 2, 2]) # 0 is mask value
score = np.array([0.1, 0.2, 0.3])
hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]])
hist_cate_id = np.array([[1, 2, 2, 0], [1, 2, 2, 0], [1, 2, 0, 0]])
behavior_length = np.array([3, 3, 2])
feature_dict = {
'user': uid,
'gender': ugender,
'item_id': iid,
'cate_id': cate_id,
'hist_item_id': hist_iid,
'hist_cate_id': hist_cate_id,
'pay_score': score,
"seq_length": behavior_length
}
if use_neg:
feature_dict['neg_hist_item_id'] = np.array([[1, 2, 3,
0], [1, 2, 3, 0],
[1, 2, 0, 0]])
feature_dict['neg_hist_cate_id'] = np.array([[1, 2, 2,
0], [1, 2, 2, 0],
[1, 2, 0, 0]])
feature_columns += [
VarLenSparseFeat(SparseFeat('neg_hist_item_id',
vocabulary_size=3 + 1,
embedding_dim=8,
embedding_name='item_id'),
maxlen=4,
length_name="seq_length"),
VarLenSparseFeat(SparseFeat('neg_hist_cate_id',
2 + 1,
embedding_dim=4,
embedding_name='cate_id'),
maxlen=4,
length_name="seq_length")
]
x = {
name: feature_dict[name]
for name in get_feature_names(feature_columns)
}
y = [1, 0, 1]
return x, y, feature_columns, behavior_feature_list
def get_item(root):
print('load')
csv_file = os.path.join(root, 'test', 'test_data', 'test_data')
item = pd.read_csv(csv_file,
dtype={
'article_id': str,
'hh': int, 'gender': str,
'age_range': str,
'read_article_ids': str
}, sep='\t')
print('loaded!!')
sparse_features = ['article_id', 'hh','gender','age_range','len_bin']
dense_features = ['image_feature']
target = ['label']
len_lis = []
read_article_ids_all = item['read_article_ids'].tolist()
for i in range(len(item)):
li = read_article_ids_all[i]
if type(li) == float:
len_lis.append(0)
continue
len_li = len(li.split(','))
len_lis.append(len_li)
item['len'] = len_lis
item['len_bin'] = pd.qcut(item['len'],6,duplicates='drop')
id_to_artic = dict()
artics = item['article_id'].tolist()
with open(os.path.join(DATASET_PATH, 'test', 'test_data', 'test_image_features.pkl'), 'rb') as handle:
image_feature_dict = pickle.load(handle)
print('image_feaeture_dict loaded..')
for feat in sparse_features:
lbe = LabelEncoder()
item[feat] = lbe.fit_transform(item[feat])
# test set으로 구성해도 되고 item 을..
fixlen_feature_columns = []
for feat in sparse_features:
if feat == 'article_id':
fixlen_feature_columns.append(SparseFeat(feat,1896))
else:
fixlen_feature_columns.append(SparseFeat(feat,item[feat].nunique()))
#fixlen_feature_columns = [SparseFeat(feat, item[feat].nunique()) for feat in sparse_features]
fixlen_feature_columns += [DenseFeat(feat,len(image_feature_dict[artics[0]])) for feat in dense_features]
print(fixlen_feature_columns)
idx_artics_all = item['article_id'].tolist()
for i in range(len(artics)):
idx_artic = idx_artics_all[i]
if idx_artic not in id_to_artic.keys():
id_to_artic[idx_artic] = artics[i]
linear_feature_columns = fixlen_feature_columns
dnn_feature_columns = fixlen_feature_columns
fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns + dnn_feature_columns)
fixlen_feature_names_global = fixlen_feature_names
model = xDeepFM(linear_feature_columns, dnn_feature_columns, task= 'binary')
#bind_nsml(model, list(), args.task)
return model, fixlen_feature_names_global, item,image_feature_dict, id_to_artic
def main(args, local):
if args.arch == 'xDeepFM' and args.mode == 'train':
s = time.time()
csv_file = os.path.join(DATASET_PATH, 'train', 'train_data', 'train_data')
item = pd.read_csv(csv_file,
dtype={
'article_id': str,
'hh': int, 'gender': str,
'age_range': str,
'read_article_ids': str
}, sep='\t')
label_data_path = os.path.join(DATASET_PATH, 'train',
os.path.basename(os.path.normpath(csv_file)).split('_')[0] + '_label')
label = pd.read_csv(label_data_path,
dtype={'label': int},
sep='\t')
item['label'] = label
sparse_features = ['article_id', 'hh','gender','age_range','len_bin']
dense_features = ['image_feature']
target = ['label']
len_lis = []
read_article_ids_all = item['read_article_ids'].tolist()
for i in range(len(item)):
li = read_article_ids_all[i]
if type(li) == float:
len_lis.append(0)
continue
len_li = len(li.split(','))
len_lis.append(len_li)
item['len'] = len_lis
item['len_bin'] = pd.qcut(item['len'],6,duplicates='drop')
id_to_artic = dict()
artics = item['article_id'].tolist()
with open(os.path.join(DATASET_PATH, 'train', 'train_data', 'train_image_features.pkl'), 'rb') as handle:
image_feature_dict = pickle.load(handle)
for feat in sparse_features:
lbe = LabelEncoder()
item[feat] = lbe.fit_transform(item[feat])
fixlen_feature_columns = [SparseFeat(feat, item[feat].nunique()) for feat in sparse_features]
fixlen_feature_columns += [DenseFeat(feat,len(image_feature_dict[artics[0]])) for feat in dense_features]
idx_artics_all = item['article_id'].tolist()
for i in range(len(artics)):
idx_artic = idx_artics_all[i]
if idx_artic not in id_to_artic.keys():
id_to_artic[idx_artic] = artics[i]
#image_feature_dict[article_id] 로 가져오면 되니까 일단 패스
linear_feature_columns = fixlen_feature_columns
dnn_feature_columns = fixlen_feature_columns
fixlen_feature_names = get_fixlen_feature_names(linear_feature_columns + dnn_feature_columns)
print(fixlen_feature_names)
global fixlen_feature_names_global
fixlen_feature_names_global = fixlen_feature_names
model = xDeepFM(linear_feature_columns, dnn_feature_columns, task= 'binary')
print('---model defined---')
# 만들었던 파일들 저장하는 것도 하나 짜기, 매번 돌릴 수 없으니까
print(time.time() - s ,'seconds')
if use_nsml and args.mode == 'train':
bind_nsml(model,[], args.task)
if args.mode == 'test':
print('_infer root - : ', DATASET_PATH)
print('test')
model, fixlen_feature_names_global, item, image_feature_dict, id_to_artic = get_item(DATASET_PATH)
bind_nsml(model, [], args.task)
checkpoint_session = ['401','team_62/airush2/176']
nsml.load(checkpoint = str(checkpoint_session[0]), session = str(checkpoint_session[1]))
print('successfully loaded')
if (args.mode == 'train'):
if args.dry_run:
print('start dry-running...!')
args.num_epochs = 1
else:
print('start training...!')
# 미리 전체를 다 만들어놓자 굳이 generator 안써도 되겠네
nsml.save('infer')
print('end')
print('end_main')
if args.pause:
nsml.paused(scope=local)
