def run_tabular_benchmarks(fast_benchmark,
subsample_size,
perf_threshold,
seed_val,
fit_args,
dataset_indices=None,
run_distill=False):
print("Running fit with args:")
print(fit_args)
# Each train/test dataset must be located in single directory with the given names.
train_file = 'train_data.csv'
test_file = 'test_data.csv'
EPS = 1e-10
# Information about each dataset in benchmark is stored in dict.
# performance_val = expected performance on this dataset (lower = better),should update based on previously run benchmarks
binary_dataset = {
'url':
'https://autogluon.s3.amazonaws.com/datasets/AdultIncomeBinaryClassification.zip',
'name': 'AdultIncomeBinaryClassification',
'problem_type': BINARY,
'label_column': 'class',
'performance_val': 0.129
} # Mixed types of features.
multi_dataset = {
'url':
'https://autogluon.s3.amazonaws.com/datasets/CoverTypeMulticlassClassification.zip',
'name': 'CoverTypeMulticlassClassification',
'problem_type': MULTICLASS,
'label_column': 'Cover_Type',
'performance_val': 0.032
} # big dataset with 7 classes, all features are numeric. Runs SLOW.
regression_dataset = {
'url':
'https://autogluon.s3.amazonaws.com/datasets/AmesHousingPriceRegression.zip',
'name': 'AmesHousingPriceRegression',
'problem_type': REGRESSION,
'label_column': 'SalePrice',
'performance_val': 0.076
} # Regression with mixed feature-types, skewed Y-values.
toyregres_dataset = {
'url': 'https://autogluon.s3.amazonaws.com/datasets/toyRegression.zip',
'name': 'toyRegression',
'problem_type': REGRESSION,
'label_column': 'y',
'performance_val': 0.183
}
# 1-D toy deterministic regression task with: heavy label+feature missingness, extra distraction column in test data
# List containing dicts for each dataset to include in benchmark (try to order based on runtimes)
datasets = [
toyregres_dataset, binary_dataset, regression_dataset, multi_dataset
]
if dataset_indices is not None: # only run some datasets
datasets = [datasets[i] for i in dataset_indices]
# Aggregate performance summaries obtained in previous benchmark run:
prev_perf_vals = [dataset['performance_val'] for dataset in datasets]
previous_avg_performance = np.mean(prev_perf_vals)
previous_median_performance = np.median(prev_perf_vals)
previous_worst_performance = np.max(prev_perf_vals)
# Run benchmark:
performance_vals = [0.0] * len(
datasets) # performance obtained in this run
directory_prefix = './datasets/'
with warnings.catch_warnings(record=True) as caught_warnings:
for idx in range(len(datasets)):
dataset = datasets[idx]
train_data, test_data = load_data(
directory_prefix=directory_prefix,
train_file=train_file,
test_file=test_file,
name=dataset['name'],
url=dataset['url'])
if seed_val is not None:
seed(seed_val)
np.random.seed(seed_val)
mx.random.seed(seed_val)
print("Evaluating Benchmark Dataset %s (%d of %d)" %
(dataset['name'], idx + 1, len(datasets)))
directory = directory_prefix + dataset['name'] + "/"
savedir = directory + 'AutogluonOutput/'
shutil.rmtree(
savedir, ignore_errors=True
) # Delete AutoGluon output directory to ensure previous runs' information has been removed.
label_column = dataset['label_column']
y_test = test_data[label_column]
test_data = test_data.drop(labels=[label_column], axis=1)
if fast_benchmark:
if subsample_size is None:
raise ValueError(
"fast_benchmark specified without subsample_size")
train_data = train_data.head(
subsample_size) # subsample for fast_benchmark
predictor = task.fit(train_data=train_data,
label=label_column,
output_directory=savedir,
**fit_args)
results = predictor.fit_summary(verbosity=4)
if predictor.problem_type != dataset['problem_type']:
warnings.warn(
"For dataset %s: Autogluon inferred problem_type = %s, but should = %s"
% (dataset['name'], predictor.problem_type,
dataset['problem_type']))
predictor = task.load(
savedir) # Test loading previously-trained predictor from file
y_pred = predictor.predict(test_data)
perf_dict = predictor.evaluate_predictions(y_true=y_test,
y_pred=y_pred,
auxiliary_metrics=True)
if dataset['problem_type'] != REGRESSION:
perf = 1.0 - perf_dict[
'accuracy_score'] # convert accuracy to error-rate
else:
perf = 1.0 - perf_dict[
'r2_score'] # unexplained variance score.
performance_vals[idx] = perf
print("Performance on dataset %s: %s (previous perf=%s)" %
(dataset['name'], performance_vals[idx],
dataset['performance_val']))
if (not fast_benchmark) and (
performance_vals[idx] >
dataset['performance_val'] * perf_threshold):
warnings.warn(
"Performance on dataset %s is %s times worse than previous performance."
% (dataset['name'], performance_vals[idx] /
(EPS + dataset['performance_val'])))
if run_distill:
predictor.distill(time_limits=60,
augment_args={'size_factor': 0.5})
# Summarize:
avg_perf = np.mean(performance_vals)
median_perf = np.median(performance_vals)
worst_perf = np.max(performance_vals)
for idx in range(len(datasets)):
print("Performance on dataset %s: %s (previous perf=%s)" %
(datasets[idx]['name'], performance_vals[idx],
datasets[idx]['performance_val']))
print("Average performance: %s" % avg_perf)
print("Median performance: %s" % median_perf)
print("Worst performance: %s" % worst_perf)
if not fast_benchmark:
if avg_perf > previous_avg_performance * perf_threshold:
warnings.warn(
"Average Performance is %s times worse than previously." %
(avg_perf / (EPS + previous_avg_performance)))
if median_perf > previous_median_performance * perf_threshold:
warnings.warn(
"Median Performance is %s times worse than previously." %
(median_perf / (EPS + previous_median_performance)))
if worst_perf > previous_worst_performance * perf_threshold:
warnings.warn(
"Worst Performance is %s times worse than previously." %
(worst_perf / (EPS + previous_worst_performance)))
print("Ran fit with args:")
print(fit_args)
# List all warnings again to make sure they are seen:
print("\n\n WARNINGS:")
for w in caught_warnings:
warnings.warn(w.message)
def load_predictor(model_directory: str) -> TabularPredictor:
predictor = TabularPrediction.load(model_directory)
predictor.save_space()
predictor.delete_models(models_to_keep='best', dry_run=False)
return predictor
file_path='https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv'
) # can be local CSV file as well, returns Pandas DataFrame
train_data = train_data.head(500) # subsample for faster demo
print(train_data.head())
label_column = 'class' # specifies which column do we want to predict
savedir = 'ag_models/' # where to save trained models
predictor = task.fit(train_data=train_data,
label=label_column,
output_directory=savedir)
# NOTE: Default settings above are intended to ensure reasonable runtime at the cost of accuracy. To maximize predictive accuracy, do this instead: predictor = task.fit(train_data=train_data, label=label_column, output_directory=savedir, presets='best_quality', eval_metric=YOUR_METRIC_NAME)
results = predictor.fit_summary()
# Inference time:
test_data = task.Dataset(
file_path='https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv'
) # another Pandas DataFrame
y_test = test_data[label_column]
test_data = test_data.drop(
labels=[label_column], axis=1
) # delete labels from test data since we wouldn't have them in practice
print(test_data.head())
predictor = task.load(
savedir
) # Unnecessary, we reload predictor just to demonstrate how to load previously-trained predictor from file
y_pred = predictor.predict(test_data)
perf = predictor.evaluate_predictions(y_true=y_test,
y_pred=y_pred,
auxiliary_metrics=True)
def __init__(self):
self.predictor_rank = task2.load(
'/content/common-alternusvera/PU/ag_predict')
self.predictor_sts = task.load(
'/content/common-alternusvera/PU/saved_dir')
