def test_experiment_fit_params(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=2, class_sep=0.98, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'objective': 'binary', 'max_depth': 8, 'n_estimators': 500} with get_temp_directory() as temp_path: result1 = experiment_gbdt(params, X_train, y_train, X_test, temp_path, fit_params={'early_stopping_rounds': None}) with get_temp_directory() as temp_path: result2 = experiment_gbdt(params, X_train, y_train, X_test, temp_path, fit_params={'early_stopping_rounds': 5}) assert result1.models[-1].booster_.num_trees() == params['n_estimators'] assert result2.models[-1].booster_.num_trees() < params['n_estimators']
def test_with_feature_attachment(): X, y = make_classification_df(n_num_features=5, class_sep=0.7) params = { 'objective': 'binary', 'max_depth': 8 } with get_temp_directory() as temp_feature_path: cols = list(X.columns) for i, c in enumerate(cols): if X.shape[1] == 1: break save_feature(X[[c]], i, directory=temp_feature_path) X.drop(c, axis=1, inplace=True) X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=False) with get_temp_directory() as temp_path: result_wo_feature = run_experiment(params, X_train, y_train, X_test, logging_directory=temp_path) with get_temp_directory() as temp_path: result_w_feature = run_experiment(params, X_train, y_train, X_test, logging_directory=temp_path, feature_list=[0, 1, 2, 3], feature_directory=temp_feature_path) assert result_w_feature.metrics[-1] > result_wo_feature.metrics[-1]
def test_experiment_sklearn_multiclass(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=0, n_classes=5, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'n_neighbors': 10} with get_temp_directory() as temp_path: result = run_experiment(params, X_train, y_train, X_test, temp_path, algorithm_type=KNeighborsClassifier, with_auto_prep=False) assert len(np.unique(result.oof_prediction[:, 0]) ) > 5 # making sure prediction is not binarized assert len(np.unique(result.test_prediction[:, 0])) > 5 assert result.oof_prediction.shape == (len(y_train), 5) assert result.test_prediction.shape == (len(y_test), 5) _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_experiment_sklearn_regressor(): X, y = make_regression_df(n_samples=1024, n_num_features=10, n_cat_features=0, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'fit_intercept': True} with get_temp_directory() as temp_path: result = run_experiment(params, X_train, y_train, X_test, temp_path, with_auto_prep=False, algorithm_type=LinearRegression) assert len(np.unique(result.oof_prediction) ) > 5 # making sure prediction is not binarized assert len(np.unique(result.test_prediction)) > 5 assert mean_squared_error(y_train, result.oof_prediction) == result.metrics[-1] _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_experiment_lgb_classifier(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=2, class_sep=0.98, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'objective': 'binary', 'max_depth': 8} with get_temp_directory() as temp_path: result = experiment_gbdt(params, X_train, y_train, X_test, temp_path, eval_func=roc_auc_score) assert len(np.unique(result.oof_prediction) ) > 5 # making sure prediction is not binarized assert len(np.unique(result.test_prediction)) > 5 assert roc_auc_score(y_train, result.oof_prediction) >= 0.9 assert roc_auc_score(y_test, result.test_prediction) >= 0.9 _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_experiment_xgb_regressor(): X, y = make_regression_df(n_samples=1024, n_num_features=10, n_cat_features=2, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'max_depth': 8, 'num_boost_round': 100} with get_temp_directory() as temp_path: result = run_experiment(params, X_train, y_train, X_test, temp_path, algorithm_type='xgb', with_auto_prep=True) assert mean_squared_error(y_train, result.oof_prediction) == result.metrics[-1] _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_experiment_sklearn_classifier(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=0, class_sep=0.98, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'C': 0.1} with get_temp_directory() as temp_path: result = run_experiment(params, X_train, y_train, X_test, temp_path, eval_func=roc_auc_score, algorithm_type=LogisticRegression, with_auto_prep=False) assert len(np.unique(result.oof_prediction) ) > 5 # making sure prediction is not binarized assert len(np.unique(result.test_prediction)) > 5 assert roc_auc_score(y_train, result.oof_prediction) >= 0.8 assert roc_auc_score(y_test, result.test_prediction) >= 0.8 _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_submission_filename(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=2, class_sep=0.98, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'objective': 'binary', 'max_depth': 8} with get_temp_directory() as temp_path: experiment_gbdt(params, X_train, y_train, X_test, temp_path, submission_filename='sub.csv') df = pd.read_csv(os.path.join(temp_path, 'sub.csv')) assert list(df.columns) == ['id', 'target']
def test_experiment_mlflow(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=2, class_sep=0.98, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'objective': 'binary', 'max_depth': 8} with get_temp_directory() as temp_path: experiment_gbdt(params, X_train, y_train, None, temp_path, with_mlflow=True) _check_file_exists( temp_path, ('oof_prediction.npy', 'metrics.txt', 'mlflow.json')) # test if output files are also stored in the mlflow artifact uri with open(os.path.join(temp_path, 'mlflow.json'), 'r') as f: mlflow_meta = json.load(f) p = unquote(urlparse(mlflow_meta['artifact_uri']).path) if os.name == 'nt' and p.startswith("/"): p = p[1:] _check_file_exists(p, ('oof_prediction.npy', 'metrics.txt'))
def test_experiment_lgb_multiclass(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=2, n_classes=5, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'objective': 'multiclass', 'max_depth': 8} with get_temp_directory() as temp_path: result = experiment_gbdt(params, X_train, y_train, X_test, temp_path) assert len(np.unique(result.oof_prediction[:, 0]) ) > 5 # making sure prediction is not binarized assert len(np.unique(result.test_prediction[:, 0])) > 5 assert result.oof_prediction.shape == (len(y_train), 5) assert result.test_prediction.shape == (len(y_test), 5) _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_experiment_lgb_regressor(): X, y = make_regression_df(n_samples=1024, n_num_features=10, n_cat_features=2, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'objective': 'regression', 'max_depth': 8} with get_temp_directory() as temp_path: result = experiment_gbdt(params, X_train, y_train, X_test, temp_path) assert len(np.unique(result.oof_prediction) ) > 5 # making sure prediction is not binarized assert len(np.unique(result.test_prediction)) > 5 assert mean_squared_error(y_train, result.oof_prediction) == result.metrics[-1] _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_experiment_manual_cv_group(): df1 = pd.DataFrame() df1['x'] = np.random.randint(0, 10, size=1000) df1['y'] = df1['x'] > 5 df1['grp'] = 0 df2 = pd.DataFrame() df2['x'] = np.random.randint(0, 10, size=100) df2['y'] = df2['x'] <= 5 df2['grp'] = 1 X = pd.concat([df1, df2]).reset_index(drop=True) y = X['y'] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) grp = X_train['grp'] X_train = X_train.drop(['y', 'grp'], axis=1) X_test = X_test.drop(['y', 'grp'], axis=1) params = { 'objective': 'binary', 'max_depth': 8 } with get_temp_directory() as temp_path: result = run_experiment(params, X_train, y_train, X_test, temp_path, cv=GroupKFold(2), groups=grp) assert result.metrics[-1] < 0.7
def test_experiment_cat_custom_eval(): X, y = make_regression_df(n_samples=1024, n_num_features=10, n_cat_features=2, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = {'max_depth': 8, 'num_boost_round': 100, 'eval_metric': 'MAE'} with get_temp_directory() as temp_path: result = experiment_gbdt(params, X_train, y_train, X_test, temp_path, gbdt_type='cat', eval_func=mean_absolute_error) assert mean_absolute_error(y_train, result.oof_prediction) == result.metrics[-1] _check_file_exists( temp_path, ('oof_prediction.npy', 'test_prediction.npy', 'metrics.txt'))
def test_experiment_sample_submission_multiclass(): X, y = make_classification_df(n_classes=5) X_train, X_test, y_train, y_test = train_test_split(X, y) sample_df = pd.DataFrame() sample_df['target_id_abc'] = np.arange(len(y_test)) + 10000 for i in range(5): sample_df['target_class_{}'.format(i)] = 0 params = { 'objective': 'multiclass', 'max_depth': 8 } with get_temp_directory() as temp_path: result = run_experiment(params, X_train, y_train, X_test, temp_path, sample_submission=sample_df) assert list(result.submission_df.columns) == ['target_id_abc', 'target_class_0', 'target_class_1', 'target_class_2', 'target_class_3', 'target_class_4' ] log_loss_trianed = log_loss(y_test, result.submission_df.drop('target_id_abc', axis=1), labels=[0, 1, 2, 3, 4]) log_loss_default = log_loss(y_test, np.full((len(y_test), 5), 0.2), labels=[0, 1, 2, 3, 4]) assert log_loss_trianed < log_loss_default
def test_log_metrics_empty(): with get_temp_directory() as logging_dir: with Experiment(logging_dir): pass with open(os.path.join(logging_dir, 'metrics.json'), 'r') as f: params = json.load(f) assert params == {}
def test_feature_exists(): df = pd.DataFrame({ 'a': [1, 2, 3, 4, 5] + [None] * 5 }) with get_temp_directory() as tmp: fs.save_feature(df[['a']], 0, directory=tmp) with pytest.raises(RuntimeError): fs.save_feature(df, 0, overwrite=False, directory=tmp)
def test_save_feature(): df = pd.DataFrame() df['a'] = np.arange(100) with get_temp_directory() as tmp: fs.save_feature(df, 0, tmp) assert os.path.exists(os.path.join(tmp, '0.f'))
def test_load_feature(): df = pd.DataFrame() df['a'] = np.arange(100) with get_temp_directory() as tmp: fs.save_feature(df, 0, tmp) df_loaded = fs.load_feature(0, tmp) assert_frame_equal(df, df_loaded)
def test_with_rare_categories(): X = pd.DataFrame({ 'x0': [None] * 100, 'x1': np.random.choice([np.inf, -np.inf], size=100), 'x2': ['nan'] + [None] * 99, 'x3': np.concatenate([ np.random.choice(['A', 'B'], size=50), np.random.choice(['C', 'D', 'na'], size=50) ]) }) y = pd.Series(np.random.choice([0, 1], size=100), name='y') params = { 'lgbm': { 'objective': 'binary', 'max_depth': 8 }, 'xgb': { 'objective': 'binary:logistic', 'max_depth': 8 }, 'cat': { 'loss_function': 'Logloss', 'max_depth': 8 } } for cat_cast in (True, False): X_ = X.copy() y_ = y.copy() if cat_cast: for c in X.columns: X_[c] = X_[c].astype('category') X_ = X_.iloc[:50, :] y_ = y_.iloc[:50] X_train, X_test, y_train, y_test = train_test_split(X_, y_, shuffle=False, test_size=0.5) for algorithm in ('cat', 'xgb', 'lgbm'): with get_temp_directory() as temp_path: run_experiment(params[algorithm], X_train, y_train, X_test, algorithm_type=algorithm, logging_directory=temp_path, with_mlflow=True, with_auto_prep=True, categorical_feature=['x0', 'x1', 'x2', 'x3'])
def test_log_metrics(): with get_temp_directory() as logging_dir: with Experiment(logging_dir) as e: e.log_metric('x', 1) e.log_metric('x', 2) with open(os.path.join(logging_dir, 'metrics.json'), 'r') as f: params = json.load(f) expected = {'x': 2} assert params == expected
def test_experiment_duplicated_rename_mlflow(): with get_temp_directory() as logging_dir: with Experiment(logging_dir, with_mlflow=True) as e: e.log_metric('CV', 0.97) run_id_old = e.mlflow_run_id with Experiment(logging_dir, with_mlflow=True, if_exists='rename') as e: e.log_metric('LB', 0.95) run_id_new = e.mlflow_run_id assert run_id_old != run_id_new
def test_experiment_duplicated_error(): with get_temp_directory() as logging_dir: with Experiment(logging_dir) as e: e.log_metric('CV', 0.97) with pytest.raises(ValueError): with Experiment(logging_dir): pass with pytest.raises(ValueError): with Experiment(logging_dir, if_exists='error'): pass
def test_custom_experiment(): params = { 'objective': 'binary', 'max_depth': 8 } X, y = make_classification_df() with get_temp_directory() as temp_path: with Experiment(temp_path, with_mlflow=True) as e: run_experiment(params, X, y, logging_directory='foobar', inherit_experiment=e) # all files are logged into e.logging_directory, instead of 'foobar' _check_file_exists(temp_path, with_mlflow=True)
def test_load_feature_ignore_all_columns(): df = pd.DataFrame() df['a'] = np.arange(100).astype(float) df['b'] = np.arange(100).astype(int) df['c'] = np.arange(100).astype(int) with get_temp_directory() as tmp: fs.save_feature(df, 0, tmp) df_loaded = fs.load_feature(0, tmp, ignore_columns=['a', 'b', 'c', 'X']) assert_frame_equal(df_loaded, df.drop(['a', 'b', 'c'], axis=1))
def test_load_features(): df = pd.DataFrame() df['a'] = np.arange(100).astype(float) df['b'] = np.arange(100).astype(int) df['c'] = np.arange(100).astype(int) with get_temp_directory() as tmp: fs.save_feature(df[['b']], 0, tmp) fs.save_feature(df[['c']], 1, tmp) df_loaded = fs.load_features(df[['a']], [0, 1], tmp) assert_frame_equal(df, df_loaded)
def test_with_long_params(): X, y = make_classification_df(1024, n_num_features=5, n_cat_features=400) params = { 'objective': 'binary', 'max_depth': 8 } X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=False) with get_temp_directory() as temp_path: # just to make sure experiment finish run_experiment(params, X_train, y_train, X_test, logging_directory=temp_path, with_mlflow=True)
def test_load_features_no_base(): df = pd.DataFrame() df['a'] = np.arange(100).astype(float) df['b'] = np.arange(100).astype(int) df['c'] = np.arange(100).astype(int) with get_temp_directory() as tmp: fs.save_feature(df[['b']], 0, tmp) fs.save_feature(df[['c']], 1, tmp) fs.save_feature(df[['a']], '2', tmp) df_loaded = fs.load_features(None, [0, 1, '2'], tmp) assert list(df_loaded.columns) == ['b', 'c', 'a']
def test_experiment_duplicated_replace(): with get_temp_directory() as logging_dir: with Experiment(logging_dir) as e: e.log_metric('CV', 0.97) with Experiment(logging_dir, if_exists='replace') as e: e.log_metric('LB', 0.95) with open(os.path.join(logging_dir, 'metrics.json')) as f: metrics = json.load(f) # replaced by the new result assert 'LB' in metrics assert 'CV' not in metrics
def test_experiment_duplicated_append(): with get_temp_directory() as logging_dir: with Experiment(logging_dir) as e: e.log_metric('CV', 0.97) with Experiment(logging_dir, if_exists='append') as e: e.log_metric('LB', 0.95) with open(os.path.join(logging_dir, 'metrics.json')) as f: metrics = json.load(f) # appended to the existing result assert 'LB' in metrics assert 'CV' in metrics
def test_experiment_manual_cv_int(): X, y = make_classification_df(n_samples=1024, n_num_features=10, n_cat_features=2, class_sep=0.98, random_state=0, id_column='user_id') X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) params = { 'objective': 'binary', 'max_depth': 8 } with get_temp_directory() as temp_path: result = run_experiment(params, X_train, y_train, None, temp_path, cv=KFold(2)) assert len(result.models) == 2 assert len(result.metrics) == 2 + 1