def test_train_model(dataset_path: str, target_name: str, conf_path: str):
training_pipeline_params = read_training_pipeline_params(conf_path)
data = read_data(dataset_path)
X, y = extract_target(data, target_name)
X_transformed = full_transform(X)
X_train, X_test, y_train, y_test = split_train_val_data(
X_transformed, y, training_pipeline_params.splitting_params)
model = train_model(X_train, y_train,
training_pipeline_params.train_params)
assert isinstance(model, LogisticRegression)
def test_split_train_val_data(dataset_path: str, target_name: str,
conf_path: str):
training_pipeline_params = read_training_pipeline_params(conf_path)
data = read_data(dataset_path)
X, y = extract_target(data, target_name)
X_train, X_test, y_train, y_test = split_train_val_data(
X, y, training_pipeline_params.splitting_params)
assert len(X_train) > 0
assert len(X_test) > 0
assert len(y_train) > 0
assert len(y_test) > 0
def test_predict_model(dataset_path: str, target_name: str, conf_path: str):
training_pipeline_params = read_training_pipeline_params(conf_path)
data = read_data(dataset_path)
X, y = extract_target(data, target_name)
X_transformed = full_transform(X)
X_train, X_test, y_train, y_test = split_train_val_data(
X_transformed, y, training_pipeline_params.splitting_params)
model = train_model(X_train, y_train,
training_pipeline_params.train_params)
pred_labels, pred_proba = predict_model(model, X_test)
assert len(set(pred_labels)) == 2
assert max(pred_proba) < 1
def test_train_pipeline(dataset_path: str, target_name: str, conf_path: str):
training_pipeline_params = read_training_pipeline_params(conf_path)
data = read_data(dataset_path)
X, y = extract_target(data, target_name)
X_transformed = full_transform(X)
X_train, X_test, y_train, y_test = split_train_val_data(
X_transformed, y, training_pipeline_params.splitting_params)
model = train_model(X_train, y_train,
training_pipeline_params.train_params)
pred_labels, pred_proba = predict_model(model, X_test)
res = evaluate_model(y_test, pred_labels, pred_proba)
assert res['accuracy'] > 0
assert res['roc_auc_score'] > 0.5
def train_pipeline_run(training_pipeline_params):
logger.info(f"Start training pipeline")
data = read_data(training_pipeline_params.input_data_path)
X, y = extract_target(data, training_pipeline_params.target_name)
logger.info(f"X and y shape is {X.shape, y.shape}")
X_transformed = full_transform(X)
X_train, X_test, y_train, y_test = split_train_val_data(
X_transformed, y, training_pipeline_params.splitting_params)
model = train_model(X_train, y_train,
training_pipeline_params.train_params)
dump_model(training_pipeline_params.dump_model, model)
logger.info(f"model fitted and dumped")
pred_labels, pred_proba = predict_model(model, X_test)
res = evaluate_model(y_test, pred_labels, pred_proba)
logger.info(f"metrics is {res}")