def __init__(self, random_seed,
embedding_sizelist, embedding_dimlist,
c_input_dim,
learning_rate, out_ratio):
nn.Module.__init__(self)
self.random_seed = random_seed
set_random_seed(random_seed)
self.out_ratio = out_ratio
self.id_model = idModel(embedding_sizelist, embedding_dimlist)
id_out_size = sum(embedding_dimlist)
self.dnn_feature_extract = nn.Sequential(
nn.Linear(c_input_dim + id_out_size, 128),
nn.ReLU(),
nn.Linear(128, 256),
nn.ReLU(),
nn.Linear(256, 512),
nn.ReLU(),
nn.Linear(512, 600),
nn.Dropout(0.05),
nn.ReLU(),
)
self.dnn_predict_val = nn.Sequential(
nn.Linear(600+c_input_dim + id_out_size, 1, bias=False),
)
self.out_bias = torch.tensor(1, dtype=torch.float32)
self.out_bias = nn.Parameter(self.out_bias)
def loss_fn(x, y):
x = torch.nn.functional.leaky_relu(x -1) + 1
y = torch.nn.functional.relu(y -1) + 1
re = torch.abs(x - y)/(torch.abs(x + y))
return torch.sum(re)
self.loss_fn = loss_fn
# self.loss_fn = lambda x, y: torch.sum(torch.abs(torch.abs(x -1) - torch.abs(y - 1))/(torch.abs(x - 1) + torch.abs(y - 1) + 2))
# self.loss_fn = torch.nn.MSELoss(reduction='sum')
# self.loss_fn = lambda x, y: torch.sum((x - y)**2)
self.optimizer = torch.optim.Adam(self.parameters(), lr=learning_rate)
self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lambda epoch_num: 1/math.sqrt(epoch_num+1))
self.vis = None
def unsupervisedTraining(self, x, y, epoch_nums, batch_size, train_ratio, custom_metric=None, plot_fold=None):
set_random_seed(self.random_seed)
samples_num = x.shape[0]
index_list = np.random.permutation(np.arange(samples_num))
train_index = index_list[0: int(train_ratio*samples_num)]
valid_index = index_list[int(train_ratio*samples_num): ]
train_x = x[train_index]
train_y = y[train_index]
valid_x = x[valid_index]
valid_y = y[valid_index]
self.fit(train_x, train_y, epoch_nums, batch_size, valid_x, valid_y, custom_metric, plot_fold, unsupervised_flag=True)
from torch_geometric.nn import GCNConv
from torch_geometric.nn import ChebConv
from torch_geometric.nn import SAGEConv
from torch_geometric.nn import GraphConv
from torch_geometric.nn import GATConv
from torch_geometric.nn import ARMAConv
from torch_geometric.nn import APPNP
from specialTools.GatedGraphConv import GatedGraphConv
from specialTools.STChebConv import STChebConv
from torch_geometric.data import Batch
from lxyTools.pytorchTools import set_random_seed
set_random_seed(2018)
raw_train = pd.read_csv('./processedData/train.csv')
test = pd.read_csv('./processedData/test.csv')
roadmap = pd.read_csv('./rawdata/Metro_roadMap.csv')
#
# roadmap.loc[roadmap.index==55, '53'] = 1
# roadmap.loc[np.array(roadmap['53']==1)&np.array(roadmap.index==54), '53'] = 0
#
# roadmap.loc[roadmap.index==53, '55'] = 1
# roadmap.loc[np.array(roadmap['55']==1)&np.array(roadmap.index==54), '55'] = 0
#
# roadmap.loc[roadmap['54']==1, '54'] = 0
from lxyTools.pytorchTools import BiInteraction
from lxyTools.pytorchTools import CrossNet
from lxyTools.pytorchTools import FM
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
import lightgbm as lgb
from tqdm import tqdm
tqdm.pandas(desc="my bar!")
set_random_seed(1546)
# 读取数据
train = pd.read_csv('./processedData/train.csv')
train.create_time = pd.to_datetime(train.create_time)
train.days = pd.to_datetime(train.days)
valid = pd.read_csv('./processedData/valid.csv')
valid.create_time = pd.to_datetime(valid.create_time)
valid.days = pd.to_datetime(valid.days)
# 合并训练数据，以及验证数据，给模型足够的数据去训练
train = pd.concat([train, valid])
train = train.reset_index(drop=True)
# 对于为零的样本进行降采样
def fit(self, x, y, epoch_nums, batch_size, valid_x, valid_y, custom_metric=None, plot_fold=None, unsupervised_flag=False):
if self.vis is not None:
vis = visdom.Visdom(env=self.vis)
set_random_seed(self.random_seed)
self.batch_size = batch_size
x_train = torch.tensor(x, dtype=torch.float32).cuda()
y_train = torch.tensor(y, dtype=torch.float32).cuda()
x_val = torch.tensor(valid_x, dtype=torch.float32).cuda()
y_val = torch.tensor(valid_y, dtype=torch.float32).cuda()
loss_fn = self.supervised_loss_fn
optimizer = self.optimizer
if unsupervised_flag:
loss_fn = self.unsupervised_loss_fn
optimizer = self.unsupervised_optimizer
scheduler = self.scheduler
train = torch.utils.data.TensorDataset(x_train, y_train)
valid = torch.utils.data.TensorDataset(x_val, y_val)
train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
for epoch in range(epoch_nums):
scheduler.step()
# print('lr:\t', scheduler.get_lr()[0])
start_time = time.time()
self.train()
avg_loss = 0.
avg_l2_reg = 0.
for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = self(x_batch)
if unsupervised_flag:
y_pred = y_pred[1]
else:
y_pred = y_pred[0]
bceloss = loss_fn(y_pred, y_batch)
optimizer.zero_grad()
bceloss.backward()
optimizer.step()
avg_loss += bceloss.item() / len(train_loader)
self.eval()
valid_preds = np.zeros((x_val.size(0)))
avg_val_loss = 0.
for i, (x_batch, y_batch) in enumerate(valid_loader):
y_pred = self(x_batch)
if unsupervised_flag:
y_pred = y_pred[1]
else:
y_pred = y_pred[0]
y_pred = y_pred.detach()
avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)
valid_preds[i * batch_size:(i+1) * batch_size] = y_pred.cpu().numpy()[:, 0]
if custom_metric is not None:
score = custom_metric(valid_y, valid_preds)
elapsed_time = time.time() - start_time
if self.vis is not None:
vis.line(X=torch.Tensor([[epoch, epoch]]),
Y=torch.Tensor([[avg_loss/batch_size, avg_val_loss/batch_size]]),
win='loss'+plot_fold,
opts={'legend':['local_loss', 'valid_loss'],
'xlabel': 'epoch',
'title': 'train'+plot_fold},
update='append' if epoch > 0 else None)
if custom_metric is not None:
if self.vis is not None:
vis.line(X=torch.Tensor([epoch]),
Y=torch.Tensor([score]),
win='score'+plot_fold,
opts={'legend':['score'],
'xlabel': 'epoch',
'title': 'valid'+plot_fold},
update='append' if epoch > 0 else None)
if custom_metric is not None:
print('Epoch {}/{} \t loss={:.4f} \t l2={:.4f} \t val_loss={:.4f} \t score={:.4f} \t time={:.2f}s'.format(
epoch + 1, epoch_nums, avg_loss/batch_size, avg_l2_reg, avg_val_loss/batch_size, score, elapsed_time))
else:
print('Epoch {}/{} \t loss={:.4f} \t l2={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
epoch + 1, epoch_nums, avg_loss/batch_size, avg_l2_reg, avg_val_loss/batch_size, elapsed_time))
def fit(self, train_x, train_y, valid_x, valid_y, kfold, train_epochs,
batch_size):
seed_start = 10086
seed_step = 500
self.fittedModelslist = []
custom_metric = lambda x, y: Online_Metric(x, y)[1]
train_preds = np.zeros((len(train_x)))
for fold_num, (train_idx,
valid_idx) in enumerate(kfold.split(train_x, train_y)):
x_train_fold = train_x[train_idx]
y_train_fold = train_y[train_idx]
x_val_fold = train_x[valid_idx]
y_val_fold = train_y[valid_idx]
model = self.modelFun(random_seed=seed_start +
fold_num * seed_step,
**self.param)
model = model.cuda()
model.batch_size = batch_size
vis = visdom.Visdom(env=model.vis)
set_random_seed(model.random_seed)
x_train_fold_tensor = torch.tensor(x_train_fold,
dtype=torch.float32).cuda()
y_train_fold_tensor = torch.tensor(y_train_fold,
dtype=torch.float32).cuda()
x_val_fold_tensor = torch.tensor(x_val_fold,
dtype=torch.float32).cuda()
y_val_fold_tensor = torch.tensor(y_val_fold,
dtype=torch.float32).cuda()
loss_fn = model.loss_fn
optimizer = model.optimizer
scheduler = model.scheduler
train_fold = torch.utils.data.TensorDataset(
x_train_fold_tensor, y_train_fold_tensor)
valid_fold = torch.utils.data.TensorDataset(
x_val_fold_tensor, y_val_fold_tensor)
train_fold_loader = torch.utils.data.DataLoader(
train_fold, batch_size=batch_size, shuffle=True)
valid_fold_loader = torch.utils.data.DataLoader(
valid_fold, batch_size=batch_size, shuffle=False)
for epoch in range(train_epochs):
scheduler.step()
start_time = time.time()
model.train()
avg_loss = 0.
for x_batch, y_batch in tqdm(train_fold_loader, disable=True):
y_pred = model(x_batch)
loss = loss_fn(y_pred, y_batch)
optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_fold_loader)
avg_loss /= batch_size
model.eval()
valid_fold_preds = np.zeros((y_val_fold.shape[0]))
avg_val_loss = 0.
for i, (x_batch, y_batch) in enumerate(valid_fold_loader):
y_pred = model(x_batch)
y_pred = y_pred.detach()
avg_val_loss += loss_fn(
y_pred, y_batch).item() / len(valid_fold_loader)
valid_fold_preds[i * batch_size:(i + 1) *
batch_size] = y_pred.cpu().numpy()
avg_val_loss /= batch_size
kfold_score = custom_metric(y_val_fold, valid_fold_preds)
valid_score = custom_metric(valid_y,
model.predict_proba(valid_x))
elapsed_time = time.time() - start_time
vis.line(X=torch.Tensor([[epoch, epoch]]),
Y=torch.Tensor([[avg_loss, avg_val_loss]]),
win='loss' + '_' + str(fold_num),
opts={
'legend': ['local_loss', 'valid_loss'],
'xlabel': 'epoch',
'title': 'train' + '_' + str(fold_num)
},
update='append' if epoch > 0 else None)
vis.line(X=torch.Tensor([[epoch, epoch]]),
Y=torch.Tensor([[kfold_score, valid_score]]),
win='score' + '_' + str(fold_num),
opts={
'legend': ['kfold score', 'valid score'],
'xlabel': 'epoch',
'title': 'valid' + '_' + str(fold_num)
},
update='append' if epoch > 0 else None)
print(
'Epoch {}/{} \t loss={:.4f} \t val_fold_loss={:.4f} \t fold_score={:.4f} \t valid_score={:.4f} \t time={:.2f}s'
.format(epoch + 1, train_epochs, avg_loss, avg_val_loss,
kfold_score, valid_score, elapsed_time))
train_preds[valid_idx] = model.predict_proba(x_val_fold)
self.fittedModelslist.append(model)
print('-' * 50)
print('-' * 20, 'finished', '-' * 20)
print('kfold score:\t', custom_metric(train_y, train_preds))
print('valid score:\t', custom_metric(valid_y, self.predict(valid_x)))
print('-' * 50)
def fit(self, train_x, train_y, cat_features, valid_x, valid_y, kfold,
train_epochs, batch_size):
seed_start = 10086
seed_step = 500
self.fittedLgbModellist = []
self.fittedNNModellist = []
self.train_pred = np.zeros(train_x.shape[0])
for fold_num, (kTrainIndex,
kTestIndex) in enumerate(kfold.split(train_x, train_y)):
kTrain_x = train_x.iloc[kTrainIndex]
kTrain_y = train_y.iloc[kTrainIndex]
kTest_x = train_x.iloc[kTestIndex]
kTest_y = train_y.iloc[kTestIndex]
self.lgb_param['random_state'] = seed_start + fold_num * seed_step
lgb_model = self.lgb_model_fun(**self.lgb_param)
lgb_model.fit(kTrain_x,
kTrain_y,
categorical_feature=cat_features,
eval_metric=Online_Metric,
eval_set=[(kTest_x, kTest_y), (valid_x, valid_y)],
verbose=50)
kTrain_x_leaves = lgb_model.predict(kTrain_x, pred_leaf=True)
kTest_x_leaves = lgb_model.predict(kTest_x, pred_leaf=True)
valid_x_leaves = lgb_model.predict(valid_x, pred_leaf=True)
for i in range(self.lgb_param['n_estimators']):
kTrain_x_leaves[:, i] += i * self.lgb_param['num_leaves']
kTest_x_leaves[:, i] += i * self.lgb_param['num_leaves']
valid_x_leaves[:, i] += i * self.lgb_param['num_leaves']
self.nn_param['random_seed'] = seed_start + fold_num * seed_step
set_random_seed(self.nn_param['random_seed'])
nn_model = self.nn_model_fun(**self.nn_param).cuda()
nn_model.fit(kTrain_x_leaves,
kTrain_y.values,
train_epochs,
batch_size,
kTest_x_leaves,
kTest_y.values,
custom_metric=lambda x, y: Online_Metric(x, y)[1],
plot_fold=str(fold_num))
valid_pre = nn_model.predict(valid_x_leaves)
self.train_pred[kTestIndex] = nn_model.predict(kTest_x_leaves)
print('valid socre:\t', Online_Metric(valid_y, valid_pre)[1])
print('--' * 50)
self.fittedLgbModellist.append(lgb_model)
self.fittedNNModellist.append(nn_model)
print('train kfold score:\t', Online_Metric(train_y, self.train_pred))
print('valid score:\t', Online_Metric(valid_y, self.predict(valid_x)))
def fit(self, train_x, train_y, valid_x, valid_y, kfold, train_epochs,
batch_size):
seed_start = 10086
seed_step = 500
self.fittedModelslist = []
custom_metric = lambda x, y: Online_Metric(x, y)[1]
train_preds = np.zeros((len(train_x[0])))
for fold_num, (train_idx, valid_idx) in enumerate(
kfold.split(train_x[0], train_y)):
x_train_fold = [data[train_idx] for data in train_x]
y_train_fold = train_y[train_idx]
x_val_fold = [data[valid_idx] for data in train_x]
y_val_fold = train_y[valid_idx]
model = self.modelFun(random_seed=seed_start +
fold_num * seed_step,
**self.param)
model = model.cuda()
model.batch_size = batch_size
set_random_seed(model.random_seed)
x_train_fold_tensor = [
torch.tensor(x_train_fold[0], dtype=torch.long).cuda(),
torch.tensor(x_train_fold[1], dtype=torch.float32).cuda(),
]
x_val_fold_tensor = [
torch.tensor(x_val_fold[0], dtype=torch.long).cuda(),
torch.tensor(x_val_fold[1], dtype=torch.float32).cuda(),
]
y_train_fold_tensor = torch.tensor(y_train_fold,
dtype=torch.float32).cuda()
y_val_fold_tensor = torch.tensor(y_val_fold,
dtype=torch.float32).cuda()
loss_fn = model.loss_fn
optimizer = model.optimizer
scheduler = model.scheduler
train_fold = torch.utils.data.TensorDataset(
*x_train_fold_tensor, y_train_fold_tensor)
valid_fold = torch.utils.data.TensorDataset(
*x_val_fold_tensor, y_val_fold_tensor)
train_fold_loader = torch.utils.data.DataLoader(
train_fold, batch_size=batch_size, shuffle=True)
valid_fold_loader = torch.utils.data.DataLoader(
valid_fold, batch_size=batch_size, shuffle=False)
for epoch in range(train_epochs):
scheduler.step()
start_time = time.time()
model.train()
avg_loss = 0.
for x_batch_1, x_batch_2, y_batch in tqdm(train_fold_loader,
disable=True):
x_batch = [x_batch_1, x_batch_2]
y_pred = model(*x_batch)
loss = loss_fn(y_pred, y_batch)
optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_fold_loader)
avg_loss /= batch_size
model.eval()
valid_fold_preds = np.zeros((y_val_fold.shape[0]))
avg_val_loss = 0.
for i, (x_batch_1, x_batch_2,
y_batch) in enumerate(valid_fold_loader):
x_batch = [x_batch_1, x_batch_2]
y_pred = model(*x_batch)
y_pred = y_pred.detach()
avg_val_loss += loss_fn(
y_pred, y_batch).item() / len(valid_fold_loader)
valid_fold_preds[i * batch_size:(i + 1) *
batch_size] = y_pred.cpu().numpy()
avg_val_loss /= batch_size
kfold_score = custom_metric(y_val_fold, valid_fold_preds)
valid_score = custom_metric(valid_y, model.predict(valid_x))
elapsed_time = time.time() - start_time
print(
'Epoch {}/{} \t loss={:.4f} \t val_fold_loss={:.4f} \t fold_score={:.4f} \t valid_score={:.4f} \t time={:.2f}s'
.format(epoch + 1, train_epochs, avg_loss, avg_val_loss,
kfold_score, valid_score, elapsed_time))
train_preds[valid_idx] = model.predict(x_val_fold)
self.fittedModelslist.append(model)
print('-' * 50)
print('-' * 20, 'finished', '-' * 20)
print('kfold score:\t', custom_metric(train_y, train_preds))
print('valid score:\t', custom_metric(valid_y, self.predict(valid_x)))
print('-' * 50)