Beispiel #1
0
    def model_test(self, test_file):
        test_data = load_data(test_file)

        xgb_clf = XGB(xgb_model_name)
        xgb_clf.test_model(test_data)

        lr_clf = LR(lr_model_name)
        x, y = load_lr_data(test_file, '\t')
        lr_clf.test_model(x, y)
Beispiel #2
0
    def model_train(self, train_file):
        train_data = load_data(train_file)
        # xgboost
        print('train a single xgb model...')
        xgb_clf = XGB(xgb_model_name)
        xgb_clf.train_model(train_data)
        print('train a single xgb model done.\n')

        # lr
        print('train a single lr model...')
        lr_clf = LR(lr_model_name)
        x, y = load_lr_data(train_file, '\t')
        lr_clf.train_model(x, y)
        print('train a single LR model done.\n')
Beispiel #3
0
    def model_test(self, test_file):
        test_x, test_y = load_sample_data(test_file)
        features = Feature(tfidf_model_name, best_feature_model_name)
        features.load_model()
        model_test_x_feature = features.transform(test_x)

        xgb_clf = XGB(xgb_model_name)
        xgb_clf.test_model(model_test_x_feature, test_y)

        lr_clf = LR(lr_model_name)
        lr_clf.test_model(model_test_x_feature, test_y)

        xgb_lr_clf = XGBLR(xgblr_xgb_model_name, xgblr_lr_model_name,
                           one_hot_encoder_model_name)
        xgb_lr_clf.test_model(model_test_x_feature, test_y)
Beispiel #4
0
def model_test(test_file):
    test_x, test_y = load_sample_data(test_file, sep=sep, has_pos=True)
    features = Feature(tfidf_model_name, best_feature_model_name)
    features.load_model()
    model_test_x_feature = features.transform(test_x)

    xgb_clf = XGB(xgb_model_name)
    xgb_preds = xgb_clf.test_model(model_test_x_feature, test_y)

    lr_clf = LR(lr_model_name)
    lr_preds = lr_clf.test_model(model_test_x_feature, test_y)

    xgb_lr_clf = XGBLR(xgblr_xgb_model_name, xgblr_lr_model_name,
                       one_hot_encoder_model_name)
    xgb_lr_preds = xgb_lr_clf.test_model(model_test_x_feature, test_y)

    save(xgb_preds, pred_save_path=xgb_pred_name)
    save(lr_preds, pred_save_path=lr_pred_name)
    save(xgb_lr_preds, pred_save_path=xgblr_pred_name)
Beispiel #5
0
def trainLR(model_id, train_features, train_label, val_features, val_label):
    '''
    @Description    :
    @Time    :2020/07/24 16:23:41
    @Author    :sam.qi
    @Param    :
    @Return    :
    '''
    print(str.format("LR-{} start training", model_id))

    use_cuda = True
    if use_cuda:
        device = torch.device(str.format('cuda:{}', model_id)
                              ) if torch.cuda.is_available() else 'cpu'
    else:
        device = torch.device('cpu')

    clf = LR(120, 20, device=device)
    clf.to(device)
    train_features = train_features.to(device)
    train_label = train_label.to(device)
    val_features = val_features.to(device)
    val_label = val_label.to(device)
    clf.fit(train_features, train_label,
            val_features, val_label, train_iters=2000)
Beispiel #6
0
    def model_train(self, train_file):
        train_x, train_y = load_sample_data(train_file)
        features = Feature(tfidf_model_name, best_feature_model_name)
        features.fit(max_feature_cnt, feature_max_df, feature_min_df,
                     ngram_range, train_x, train_y)
        model_train_x_feature = features.transform(train_x)
        # xgboost
        print('train a single xgb model...')
        xgb_clf = LR(xgb_model_name)
        xgb_clf.train_model(model_train_x_feature, train_y)
        print('train a single xgb model done.\n')

        # lr
        print('train a single lr model...')
        lr_clf = LR(lr_model_name)
        lr_clf.train_model(model_train_x_feature, train_y)
        print('train a single LR model done.\n')

        # xgboost+lr
        print('train a xgboost+lr model...')
        xgb_lr_clf = XGBLR(xgblr_xgb_model_name, xgblr_lr_model_name,
                           one_hot_encoder_model_name)
        xgb_lr_clf.train_model(model_train_x_feature, train_y)
        print('train a xgboost+lr model done.\n')
Beispiel #7
0
with open('../dataset.pkl', 'rb') as f:
    train_set = pickle.load(f)
    test_set = pickle.load(f)
    cate_list = pickle.load(f)
    user_count, item_count, cate_count = pickle.load(f)

if sys.argv[1] == 'deepfm':
    from deepfm import DeepFM
    model = DeepFM(user_count=user_count,
                   item_count=item_count,
                   cate_count=cate_count,
                   cate_list=cate_list)
elif sys.argv[1] == 'lr':
    from lr import LR
    model = LR(user_count=user_count,
               item_count=item_count,
               cate_count=cate_count,
               cate_list=cate_list)
elif sys.argv[1] == 'lrcross':
    from lrcross import LR
    model = LR(user_count=user_count,
               item_count=item_count,
               cate_count=cate_count,
               cate_list=cate_list)
elif sys.argv[1] == 'dnn':
    from dnn import DNN
    model = DNN(user_count=user_count,
                item_count=item_count,
                cate_count=cate_count,
                cate_list=cate_list)
elif sys.argv[1] == 'widedeep':
    from widedeep import WideDeep
Beispiel #8
0
cv = CountVectorizer()
X = cv.fit_transform(corpus).toarray()

##############################################################
# Training the model
print("Training the model with train_set=80% & test_set=20%")

# Splitting the dataset into the Training set and Test set
from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,
                                                    y,
                                                    test_size=0.20,
                                                    random_state=0)

# Fitting Logistic Regression to the Training set
model = LR(X_train, y_train)
y_pred = model.predictor(X_test)

# Calculating Accuracy
accuracy = model.accuracy(y_test, y_pred)
print("Model Accuracy : ", accuracy)

# Retraining the model with complete dataset
model = LR(X, y)
print("Model Retrained With the Complete DataSet")

##############################################################
# For Testing New Sentences :
##############################################################

while True: