def test_hpo_many_columns(test_dir, data_frame):
feature_col, label_col = "feature", "label"
n_samples = 300
num_labels = 3
ncols = 10
seq_len = 4
# generate some random data
df = data_frame(feature_col=feature_col,
label_col=label_col,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
for col in range(ncols):
df[feature_col + '_' + str(col)] = df[feature_col]
imputer = SimpleImputer(
input_columns=[col for col in df.columns if not col in ['label']],
output_column=label_col,
output_path=test_dir)
imputer.fit_hpo(df, num_evals=2, num_epochs=10)
assert imputer.hpo.results.precision_weighted.max() > .75
def test_label_shift_weight_computation():
"""
Tests that label shift detection can determine the label marginals of validation data.
"""
train_proportion = [.7, .3]
target_proportion = [.3, .7]
data = synthetic_label_shift_simple(N=2000, label_proportions=train_proportion,
error_proba=.1, covariates=['foo', 'bar'])
# original train test splits
tr, te = random_split(data, [.5, .5])
# train domain classifier
imputer = SimpleImputer(
input_columns=['covariate'],
output_column='label',
output_path='/tmp/imputer_model')
# Fit an imputer model on the train data (coo_imputed_proba, coo_imputed)
imputer.fit(tr, te, num_epochs=15, learning_rate=3e-4, weight_decay=0)
target_data = synthetic_label_shift_simple(1000, target_proportion,
error_proba=.1, covariates=['foo', 'bar'])
weights = imputer.check_for_label_shift(target_data)
# compare the product of weights and training marginals
# (i.e. estimated target marginals) with the true target marginals.
for x in list(zip(list(weights.values()), train_proportion, target_proportion)):
assert x[0]*x[1] - x[2] < .1
def test_imputer_hpo_numeric():
"""
Tests SimpleImputer HPO for numeric data/imputation
"""
N = 200
numeric_data = np.random.uniform(-np.pi, np.pi, (N, ))
df = pd.DataFrame({
'x': numeric_data,
'**2': numeric_data**2 + np.random.normal(0, .1, (N, )),
})
df_train, df_test = random_split(df, [.8, .2])
output_path = os.path.join(dir_path, "resources", "tmp",
"real_data_experiment_numeric_hpo")
imputer_numeric = SimpleImputer(input_columns=['x'],
output_column="**2",
output_path=output_path).fit_hpo(
train_df=df_train,
learning_rate=1e-3,
num_epochs=100,
patience=10,
num_hash_bucket_candidates=[2**10],
tokens_candidates=['words'],
latent_dim_candidates=[10, 50, 100],
hidden_layers_candidates=[1, 2])
imputer_numeric.predict(df_test)
assert mean_squared_error(df_test['**2'], df_test['**2_imputed']) < 1.0
shutil.rmtree(output_path)
def test_hpo_runs(test_dir, data_frame):
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
imputer = SimpleImputer(
input_columns=[col for col in df.columns if col != label_col],
output_column=label_col,
output_path=test_dir)
hps = dict()
max_tokens = [1024, 2048]
hps[feature_col] = {'max_tokens': max_tokens}
hps['global'] = {}
hps['global']['concat_columns'] = [False]
hps['global']['num_epochs'] = [10]
hps['global']['num_epochs'] = [10]
hps['global']['num_epochs'] = [10]
imputer.fit_hpo(df,
hps=hps,
num_hash_bucket_candidates=[2**15],
tokens_candidates=['words'])
# only search over specified parameter ranges
assert set(
imputer.hpo.results[feature_col + ':' +
'max_tokens'].unique().tolist()) == set(max_tokens)
assert imputer.hpo.results.shape[0] == 2
def test_hpo_many_columns(test_dir, data_frame):
"""
"""
label_col = "label"
n_samples = 300
num_labels = 3
ncols = 10
seq_len = 4
# generate some random data
df = data_frame(feature_col="string_feature",
label_col=label_col,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
for col in range(ncols):
df['string_featur_' + str(col)] = df['string_feature']
df_train, df_test = random_split(df, [.8, .2])
output_path = os.path.join(test_dir, "tmp",
"real_data_experiment_text_hpo")
imputer = SimpleImputer(
input_columns=[col for col in df.columns if not col in ['label']],
output_column='label',
output_path=output_path)
imputer.fit_hpo(df_train, num_evals=2)
assert imputer.hpo.results.precision_weighted.max() > .8
def test_hpo_defaults(test_dir, data_frame):
label_col = "label"
n_samples = 500
num_labels = 3
seq_len = 10
# generate some random data
df = data_frame(feature_col="string_feature",
label_col=label_col,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
# add categorical feature
df['categorical_feature'] = [
'foo' if r > .5 else 'bar' for r in np.random.rand(n_samples)
]
# add numerical feature
df['numeric_feature'] = np.random.rand(n_samples)
df_train, df_test = random_split(df, [.8, .2])
output_path = os.path.join(test_dir, "tmp",
"real_data_experiment_text_hpo")
imputer = SimpleImputer(input_columns=[
'string_feature', 'categorical_feature', 'numeric_feature'
],
output_column='label',
output_path=output_path)
imputer.fit_hpo(df_train, num_evals=10, num_epochs=5)
assert imputer.hpo.results.precision_weighted.max() > .9
def test_hpo_numeric_best_pick(test_dir, data_frame):
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
df.loc[:, label_col] = np.random.randn(df.shape[0])
imputer = SimpleImputer(input_columns=[feature_col],
output_column=label_col,
output_path=test_dir,
is_explainable=True)
hps = {feature_col: {'max_tokens': [1, 2, 3]}}
hps[feature_col]['tokens'] = [['chars']]
imputer.fit_hpo(df, hps=hps)
results = imputer.hpo.results
max_tokens_of_encoder = imputer.imputer.data_encoders[
0].vectorizer.max_features
# model with minimal MSE
best_hpo_run = imputer.hpo.results['mse'].astype('float').idxmin()
loaded_hpo_run = results.loc[results[feature_col + ':max_tokens'] ==
max_tokens_of_encoder].index[0]
assert best_hpo_run == loaded_hpo_run
def test_imputer_complete():
"""
Tests the complete functionality of SimpleImputer
:param data_frame:
"""
feature_col = "string_feature"
label_col = "label"
feature_col_numeric = "numeric_feature"
label_col_numeric = "numeric_label"
num_samples = 1000
num_labels = 3
seq_len = 20
missing_ratio = .1
df_string = generate_df_string(num_labels=num_labels,
num_words=seq_len,
num_samples=num_samples,
label_column_name=label_col,
data_column_name=feature_col)
df_numeric = generate_df_numeric(num_samples=num_samples,
label_column_name=label_col_numeric,
data_column_name=feature_col_numeric)
df = pd.concat([
df_string[[feature_col, label_col]],
df_numeric[[feature_col_numeric, label_col_numeric]]
],
ignore_index=True,
axis=1)
df.columns = [
feature_col, label_col, feature_col_numeric, label_col_numeric
]
# delete some entries
for col in df.columns:
missing = np.random.random(len(df)) < missing_ratio
df[col].iloc[missing] = np.nan
feature_col_missing = df[feature_col].isnull()
label_col_missing = df[label_col].isnull()
df = SimpleImputer.complete(data_frame=df)
assert all(df[feature_col].isnull() == feature_col_missing)
assert df[label_col].isnull().sum() < label_col_missing.sum()
assert df[feature_col_numeric].isnull().sum() == 0
assert df[label_col_numeric].isnull().sum() == 0
df = SimpleImputer.complete(data_frame=df, output_path='some_path')
def test_hpo_mixed_hps_and_kwargs(test_dir, data_frame):
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
imputer = SimpleImputer(input_columns=[feature_col],
output_column=label_col,
output_path=test_dir)
hps = {feature_col: {'max_tokens': [1024]}}
imputer.fit_hpo(df, hps=hps, learning_rate_candidates=[0.1])
assert imputer.hpo.results['global:learning_rate'].values[0] == 0.1
def test_hpo_num_evals_empty_hps(test_dir, data_frame):
feature_col, label_col = "feature", "label"
# generate some random data
df = data_frame(feature_col=feature_col, label_col=label_col)
imputer = SimpleImputer(
input_columns=[col for col in df.columns if col != label_col],
output_column=label_col,
output_path=test_dir)
num_evals = 2
imputer.fit_hpo(df, num_evals=num_evals, num_epochs=10)
assert imputer.hpo.results.shape[0] == 2
def test_hpo_num_evals_given_hps(test_dir, data_frame):
feature_col, label_col = "feature", "label"
# generate some random data
df = data_frame(feature_col=feature_col, label_col=label_col)
# assert that num_evals is an upper bound on the number of hpo runs
for num_evals in range(1, 3):
imputer = SimpleImputer(
input_columns=[col for col in df.columns if col != label_col],
output_column=label_col,
output_path=test_dir)
imputer.fit_hpo(df, num_evals=num_evals, num_epochs=5)
assert imputer.hpo.results.shape[0] == num_evals
def test_hpo_similar_input_col_mixed_types(test_dir, data_frame):
feature_col, label_col = "feature", "label"
numeric_col = "numeric_feature"
categorical_col = "categorical_col"
df = data_frame(feature_col=feature_col, label_col=label_col)
df.loc[:, numeric_col] = np.random.randn(df.shape[0])
df.loc[:, categorical_col] = np.random.randint(df.shape[0])
imputer = SimpleImputer(
input_columns=[feature_col, numeric_col, categorical_col],
output_column=label_col,
output_path=test_dir)
imputer.fit_hpo(df, num_epochs=10)
def test_hpo_kwargs_only_support(test_dir, data_frame):
feature_col, label_col = "feature", "label"
numeric_col = "numeric_feature"
df = data_frame(feature_col=feature_col,
label_col=label_col)
df.loc[:, numeric_col] = np.random.randn(df.shape[0])
imputer = SimpleImputer(
input_columns=[feature_col, numeric_col],
output_column=label_col,
output_path=test_dir
)
imputer.fit_hpo(
df,
num_epochs=1,
patience=1,
weight_decay=[0.001],
batch_size=320,
num_hash_bucket_candidates=[3],
tokens_candidates=['words'],
numeric_latent_dim_candidates=[1],
numeric_hidden_layers_candidates=[1],
final_fc_hidden_units=[[1]],
learning_rate_candidates=[0.1],
normalize_numeric=False
)
def assert_val(col, value):
assert imputer.hpo.results[col].values[0] == value
assert_val('global:num_epochs', 1)
assert_val('global:patience', 1)
assert_val('global:weight_decay', 0.001)
assert_val('global:batch_size', 320)
assert_val(feature_col + ':max_tokens', 3)
assert_val(feature_col + ':tokens', ['words'])
assert_val(numeric_col + ':numeric_latent_dim', 1)
assert_val(numeric_col + ':numeric_hidden_layers', 1)
assert_val('global:final_fc_hidden_units', [1])
assert_val('global:learning_rate', 0.1)
def test_imputer_hpo_text(test_dir, data_frame):
"""
Tests SimpleImputer HPO with text data and categorical imputations
"""
feature_col = "string_feature"
label_col = "label"
n_samples = 1000
num_labels = 3
seq_len = 20
# generate some random data
df = data_frame(feature_col=feature_col,
label_col=label_col,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
df_train, df_test = random_split(df, [.8, .2])
output_path = os.path.join(test_dir, "tmp", "experiment_text_hpo")
imputer_string = SimpleImputer(
input_columns=[feature_col],
output_column=label_col,
output_path=output_path
)
hps = dict()
hps[feature_col] = {}
hps[feature_col]['type'] = ['string']
hps[feature_col]['tokens'] = [['words'], ['chars']]
hps['global'] = {}
hps['global']['final_fc_hidden_units'] = [[]]
hps['global']['learning_rate'] = [1e-3]
hps['global']['weight_decay'] = [0]
hps['global']['num_epochs'] = [30]
imputer_string.fit_hpo(df_train, hps=hps, num_epochs=10, num_evals=3)
assert max(imputer_string.hpo.results['f1_micro']) > 0.7
def test_hpo_num_evals_given_hps(test_dir, data_frame):
feature_col, label_col = "feature", "label"
# generate some random data
df = data_frame(feature_col=feature_col, label_col=label_col)
num_evals = 2
# assert that num_evals is an upper bound on the number of hpo runs
for n_max_tokens_to_try in range(1, 5):
imputer = SimpleImputer(
input_columns=[col for col in df.columns if col != label_col],
output_column=label_col,
output_path=test_dir)
hps = {feature_col: {'max_tokens': n_max_tokens_to_try * [10]}}
imputer.fit_hpo(df, hps=hps, num_evals=num_evals)
assert imputer.hpo.results.shape[0] == min(num_evals,
n_max_tokens_to_try)
def test_imputer_hpo_text():
"""
Tests SimpleImputer HPO with text data and categorical imputations
"""
feature_col = "string_feature"
label_col = "label"
n_samples = 1000
num_labels = 3
seq_len = 20
# generate some random data
df = generate_string_data_frame(feature_col=feature_col,
label_col=label_col,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
df_train, df_test = random_split(df, [.8, .2])
output_path = os.path.join(dir_path, "resources", "tmp",
"real_data_experiment_text_hpo")
imputer_string = SimpleImputer(
input_columns=[feature_col],
output_column=label_col,
output_path=output_path).fit_hpo(
train_df=df_train,
num_epochs=100,
patience=3,
num_hash_bucket_candidates=[2**10, 2**15],
tokens_candidates=['words'],
latent_dim_candidates=[10],
hpo_max_train_samples=1000)
imputer_string.predict(df_test)
assert f1_score(df_test[label_col],
df_test[label_col + '_imputed'],
average="weighted") > .7
shutil.rmtree(output_path)
def test_hpo_multiple_columns_only_one_used(test_dir, data_frame):
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
df.loc[:, feature_col + '_2'] = df.loc[:, feature_col]
imputer = SimpleImputer(input_columns=[feature_col],
output_column=label_col,
output_path=test_dir,
is_explainable=True)
hps = dict()
hps[feature_col] = {'max_tokens': [1024]}
hps['global'] = {}
hps['global']['num_epochs'] = [10]
imputer.fit_hpo(df, hps=hps)
assert imputer.hpo.results.shape[0] == 1
assert imputer.imputer.data_encoders[0].vectorizer.max_features == 1024
def test_hpo_single_column_encoder_parameter(test_dir, data_frame):
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
imputer = SimpleImputer(
input_columns=[col for col in df.columns if col != label_col],
output_column=label_col,
output_path=test_dir,
is_explainable=True)
hps = dict()
hps[feature_col] = {'max_tokens': [1024]}
hps['global'] = {}
hps['global']['num_epochs'] = [10]
imputer.fit_hpo(df, hps=hps)
assert imputer.hpo.results.shape[0] == 2
assert imputer.imputer.data_encoders[0].vectorizer.max_features == 1024
def test_imputer_hpo_numeric(test_dir):
"""
Tests SimpleImputer HPO for numeric data/imputation
"""
N = 200
numeric_data = np.random.uniform(-np.pi, np.pi, (N, ))
df = pd.DataFrame({
'x': numeric_data,
'**2': numeric_data**2 + np.random.normal(0, .1, (N, )),
})
df_train, df_test = random_split(df, [.8, .2])
output_path = os.path.join(test_dir, "tmp", "experiment_numeric_hpo")
imputer_numeric = SimpleImputer(input_columns=['x'],
output_column="**2",
output_path=output_path)
feature_col = 'x'
hps = {}
hps[feature_col] = {}
hps[feature_col]['type'] = ['numeric']
hps[feature_col]['numeric_latent_dim'] = [30]
hps[feature_col]['numeric_hidden_layers'] = [1]
hps['global'] = {}
hps['global']['final_fc_hidden_units'] = [[]]
hps['global']['learning_rate'] = [1e-3, 1e-4]
hps['global']['weight_decay'] = [0]
hps['global']['num_epochs'] = [200]
hps['global']['patience'] = [100]
hps['global']['concat_columns'] = [False]
imputer_numeric.fit_hpo(df_train, hps=hps)
results = imputer_numeric.hpo.results
assert results[results['mse'] == min(results['mse'])]['mse'].iloc[0] < .3
def test_imputer_categorical_heuristic(data_frame):
"""
Tests the heuristic used for checking whether a column is categorical
:param data_frame:
"""
feature_col = "string_feature"
label_col = "label"
n_samples = 1000
num_labels = 3
seq_len = 20
# generate some random data
df = data_frame(feature_col=feature_col,
label_col=label_col,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
assert SimpleImputer._is_categorical(df[feature_col]) == False
assert SimpleImputer._is_categorical(df[label_col]) == True
def test_explainable_simple_imputer(test_dir, data_frame):
label_col = 'label'
df = data_frame(n_samples=100, label_col=label_col)
output_path = os.path.join(test_dir, "tmp")
imputer = SimpleImputer(['features'],
label_col,
output_path=output_path,
is_explainable=True).fit(df)
assert imputer.is_explainable
assert isinstance(imputer.imputer.data_encoders[0],
column_encoders.TfIdfEncoder)
# explain should not raise an exception
_ = imputer.explain(df[label_col].unique()[0])
# explain_instance should not raise an exception
instance = pd.Series({'features': 'some feature text'})
_ = imputer.explain_instance(instance)
assert True
def test_hpo_explainable(test_dir, data_frame):
from sklearn.feature_extraction.text import HashingVectorizer, TfidfVectorizer
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
for explainable, vectorizer in [(False, HashingVectorizer),
(True, TfidfVectorizer)]:
imputer = SimpleImputer(input_columns=[feature_col],
output_column=label_col,
output_path=test_dir,
is_explainable=explainable).fit_hpo(df)
assert isinstance(imputer.imputer.data_encoders[0].vectorizer,
vectorizer)
def test_imputer_image_hpo():
"""
Tests SimpleImputer HPO with image data imputing a text column
"""
img_path = os.path.join(dir_path, "resources", "test_images")
os.makedirs(img_path, exist_ok=True)
colors = ['red', 'green', 'blue']
for color in colors:
create_test_image(os.path.join(img_path, color + ".png"), color)
n_samples = 32
color_labels = [random.choice(colors) for _ in range(n_samples)]
df = pd.DataFrame({"image_files": color_labels, "label": color_labels})
df['image_files'] = img_path + "/" + df['image_files'] + ".png"
output_path = os.path.join(dir_path, "resources", "tmp",
"experiment_images_hpo")
imputer_string = SimpleImputer(input_columns=['image_files'],
output_column="label",
output_path=output_path).fit_hpo(
train_df=df,
learning_rate=1e-3,
num_epochs=10,
patience=10,
test_split=.3,
weight_decay=.0,
num_hash_bucket_candidates=[2**10],
tokens_candidates=['words'],
latent_dim_candidates=[10, 100],
hpo_max_train_samples=1000)
def test_fit_resumes(test_dir, data_frame):
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
imputer = SimpleImputer(input_columns=[feature_col],
output_column=label_col,
output_path=test_dir)
assert imputer.imputer is None
imputer.fit(df)
first_fit_imputer = imputer.imputer
imputer.fit(df)
second_fit_imputer = imputer.imputer
assert first_fit_imputer == second_fit_imputer
def test_single_hpo(test_dir, data_frame):
feature_col, label_col = "feature", "label"
df = data_frame(feature_col=feature_col, label_col=label_col)
imputer = SimpleImputer(
input_columns=[col for col in df.columns if col != label_col],
output_column=label_col,
output_path=test_dir)
hps = dict()
hps[feature_col] = {'max_tokens': [1024]}
hps['global'] = {}
hps['global']['num_epochs'] = [10]
hps['string'] = {}
hps['categorical'] = {}
hps['numeric'] = {}
hpo = _HPO()
hpo.tune(train_df=df, hps=hps, simple_imputer=imputer)
assert hpo.results.shape[0] == 1
assert hpo.results[feature_col + ':max_tokens'].values[0] == 1024
assert hpo.results['global:num_epochs'].values[0] == 10
def test_explainable_simple_imputer_unfitted():
label_col = 'label'
imputer = SimpleImputer(['features'], label_col, is_explainable=True)
assert imputer.is_explainable
try:
imputer.explain('some class')
raise pytest.fail(
'imputer.explain should fail with an appropriate error message')
except ValueError as exception:
assert exception.args[0] == 'Need to call .fit() before'
instance = pd.Series({'features': 'some feature text'})
try:
imputer.explain_instance(instance)
raise pytest.fail(
'imputer.explain_instance should fail with an appropriate error message'
)
except ValueError as exception:
assert exception.args[0] == 'Need to call .fit() before'
def test_default_no_explainable_simple_imputer():
imputer = SimpleImputer(['features'], 'label')
assert not imputer.is_explainable
def test_simple_imputer_real_data_default_args(test_dir, data_frame):
"""
Tests SimpleImputer with default options
"""
feature_col = "string_feature"
label_col = "label"
n_samples = 2000
num_labels = 3
seq_len = 100
vocab_size = int(2**15)
# generate some random data
random_data = data_frame(feature_col=feature_col,
label_col=label_col,
vocab_size=vocab_size,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
df_train, df_test, df_val = random_split(random_data, [.8, .1, .1])
output_path = os.path.join(test_dir, "tmp", "real_data_experiment_simple")
df_train_cols_before = df_train.columns.tolist()
input_columns = [feature_col]
imputer = SimpleImputer(input_columns=input_columns,
output_column=label_col,
output_path=output_path).fit(train_df=df_train)
logfile = os.path.join(imputer.output_path, 'imputer.log')
assert os.path.exists(logfile)
assert os.path.getsize(logfile) > 0
assert imputer.output_path == output_path
assert imputer.imputer.data_featurizers[0].__class__ == BowFeaturizer
assert imputer.imputer.data_encoders[0].__class__ == BowEncoder
assert set(
imputer.imputer.data_encoders[0].input_columns) == set(input_columns)
assert set(imputer.imputer.label_encoders[0].input_columns) == set(
[label_col])
assert all([
after == before
for after, before in zip(df_train.columns, df_train_cols_before)
])
df_no_label_column = df_test.copy()
true_labels = df_test[label_col]
del (df_no_label_column[label_col])
df_test_cols_before = df_no_label_column.columns.tolist()
df_test_imputed = imputer.predict(df_no_label_column, inplace=True)
assert all([
after == before for after, before in zip(df_no_label_column.columns,
df_test_cols_before)
])
imputed_columns = df_test_cols_before + [
label_col + "_imputed", label_col + "_imputed_proba"
]
assert all([
after == before
for after, before in zip(df_test_imputed, imputed_columns)
])
f1 = f1_score(true_labels,
df_test_imputed[label_col + '_imputed'],
average="weighted")
assert f1 > .9
new_path = imputer.output_path + "-" + rand_string()
os.rename(imputer.output_path, new_path)
deserialized = SimpleImputer.load(new_path)
df_test = deserialized.predict(df_test,
imputation_suffix="_deserialized_imputed")
f1 = f1_score(df_test[label_col],
df_test[label_col + '_deserialized_imputed'],
average="weighted")
assert f1 > .9
retrained_simple_imputer = deserialized.fit(df_train, df_train)
df_train_imputed = retrained_simple_imputer.predict(df_train.copy(),
inplace=True)
f1 = f1_score(df_train[label_col],
df_train_imputed[label_col + '_imputed'],
average="weighted")
assert f1 > .9
metrics = retrained_simple_imputer.load_metrics()
assert f1 == metrics['weighted_f1']
def test_simple_imputer_no_string_column_name():
with pytest.raises(ValueError):
SimpleImputer([0], '1')
with pytest.raises(ValueError):
SimpleImputer(['0'], 1)
def test_hpo_all_input_types(test_dir, data_frame):
"""
Using sklearn advantages: parallelism, distributions of parameters, multiple cross-validation
"""
label_col = "label"
n_samples = 1000
num_labels = 3
seq_len = 12
# generate some random data
df = data_frame(feature_col="string_feature",
label_col=label_col,
num_labels=num_labels,
num_words=seq_len,
n_samples=n_samples)
# add categorical feature
df['categorical_feature'] = [
'foo' if r > .5 else 'bar' for r in np.random.rand(n_samples)
]
# add numerical feature
df['numeric_feature'] = np.random.rand(n_samples)
df_train, df_test = random_split(df, [.8, .2])
output_path = os.path.join(test_dir, "tmp",
"real_data_experiment_text_hpo")
imputer = SimpleImputer(input_columns=[
'string_feature', 'categorical_feature', 'numeric_feature'
],
output_column='label',
output_path=output_path)
# Define default hyperparameter choices for each column type (string, categorical, numeric)
hps = dict()
hps['global'] = {}
hps['global']['learning_rate'] = [3e-4]
hps['global']['weight_decay'] = [1e-8]
hps['global']['num_epochs'] = [5, 50]
hps['global']['patience'] = [5]
hps['global']['batch_size'] = [16]
hps['global']['final_fc_hidden_units'] = [[]]
hps['global']['concat_columns'] = [True, False]
hps['string_feature'] = {}
hps['string_feature']['max_tokens'] = [2**15]
hps['string_feature']['tokens'] = [['words', 'chars']]
hps['string_feature']['ngram_range'] = {}
hps['string_feature']['ngram_range']['words'] = [(1, 4), (2, 5)]
hps['string_feature']['ngram_range']['chars'] = [(2, 4), (3, 5)]
hps['categorical_feature'] = {}
hps['categorical_feature']['type'] = ['categorical']
hps['categorical_feature']['max_tokens'] = [2**15]
hps['categorical_feature']['embed_dim'] = [10]
hps['numeric_feature'] = {}
hps['numeric_feature']['normalize'] = [True]
hps['numeric_feature']['numeric_latent_dim'] = [10]
hps['numeric_feature']['numeric_hidden_layers'] = [1]
# user defined score function for hyperparameters
def calibration_check(true, predicted, confidence):
"""
expect kwargs: true, predicted, confidence
here we compute a calibration sanity check
"""
return (np.mean(true[confidence > .9] == predicted[confidence > .9]),
np.mean(true[confidence > .5] == predicted[confidence > .5]))
def coverage_check(true, predicted, confidence):
return np.mean(confidence > .9)
uds = [(calibration_check, 'calibration check'),
(coverage_check, 'coverage at 90')]
imputer.fit_hpo(df_train,
hps=hps,
user_defined_scores=uds,
num_evals=5,
hpo_run_name='test1_')
imputer.fit_hpo(df_train,
hps=hps,
user_defined_scores=uds,
num_evals=5,
hpo_run_name='test2_',
max_running_hours=1 / 3600)
results = imputer.hpo.results
assert results[results['global:num_epochs'] == 50]['f1_micro'].iloc[0] > \
results[results['global:num_epochs'] == 5]['f1_micro'].iloc[0]