class AGLearner(object):
def __init__(self, path=None):
self.path = path
def fit(self, x, y):
''' '''
x = x if len(x.shape) > 1 else x[:, None]
y = y if len(y.shape) > 1 else y[:, None]
x_columns = ['x_%d' % i for i in range(x.shape[1])]
self.x_columns = x_columns
y_column = 'target'
columns = x_columns + [y_column]
train_data = pd.DataFrame(np.concatenate([x, y], axis=1),
columns=columns)
self._model = TabularPredictor(y_column, problem_type=problem_type, eval_metric=eval_metric, \
path=self.path, verbosity=verbosity, sample_weight=sample_weight, weight_evaluation=weight_evaluation, \
groups=groups, **kwargs).fit(train_data, **fit_kwargs)
def predict(self, x):
''' '''
assert hasattr(self, '_model'), 'The model has not been fitted yet'
x = x if len(x.shape) > 1 else x[:, None]
if not hasattr(self, 'x_columns'):
self.x_columns = ['x_%d' % i for i in range(x.shape[1])]
assert x.shape[1] == len(
self.x_columns
), 'x has a shape incompatible with training data'
data = pd.DataFrame(x, columns=self.x_columns)
y_pred = self._model.predict(data, as_pandas=False)
return y_pred
@property
def feature_importances_(self):
try:
importance_df = self._model.feature_importance()
importances = [
importance_df.at[col, 'importance']
for col in self.x_columns
]
return importances
except:
return []
def save(self, path):
self._model.save()
@classmethod
def load(cls, path):
learner = AGLearner(path=path)
learner._model = TabularPredictor.load(path)
return learner
def run_feature_permutation(predictor: TabularPredictor,
df_train: pd.DataFrame,
model_name: Union[str, None],
**kwargs) -> pd.DataFrame:
"""
Use feature permutation to derive feature importances from an AutoGluon
model. The AG documentation refers this website to explain feature
permutation: https://explained.ai/rf-importance/
"""
df_importance = predictor.feature_importance(df_train,
model=model_name,
num_shuffle_sets=kwargs['num_shuffle_sets'],
subsample_size=kwargs['subsample_size'])
#**kwargs)
return df_importance
def test_advanced_functionality():
fast_benchmark = True
dataset = {'url': 'https://autogluon.s3.amazonaws.com/datasets/AdultIncomeBinaryClassification.zip',
'name': 'AdultIncomeBinaryClassification',
'problem_type': BINARY}
label = 'class'
directory_prefix = './datasets/'
train_file = 'train_data.csv'
test_file = 'test_data.csv'
train_data, test_data = load_data(directory_prefix=directory_prefix, train_file=train_file, test_file=test_file, name=dataset['name'], url=dataset['url'])
if fast_benchmark: # subsample for fast_benchmark
subsample_size = 100
train_data = train_data.head(subsample_size)
test_data = test_data.head(subsample_size)
print(f"Evaluating Advanced Functionality on Benchmark Dataset {dataset['name']}")
directory = directory_prefix + 'advanced/' + dataset['name'] + "/"
savedir = directory + 'AutogluonOutput/'
shutil.rmtree(savedir, ignore_errors=True) # Delete AutoGluon output directory to ensure previous runs' information has been removed.
predictor = TabularPredictor(label=label, path=savedir).fit(train_data)
leaderboard = predictor.leaderboard(data=test_data)
extra_metrics = ['accuracy', 'roc_auc', 'log_loss']
leaderboard_extra = predictor.leaderboard(data=test_data, extra_info=True, extra_metrics=extra_metrics)
assert set(predictor.get_model_names()) == set(leaderboard['model'])
assert set(predictor.get_model_names()) == set(leaderboard_extra['model'])
assert set(leaderboard_extra.columns).issuperset(set(leaderboard.columns))
assert len(leaderboard) == len(leaderboard_extra)
assert set(leaderboard_extra.columns).issuperset(set(extra_metrics)) # Assert that extra_metrics are present in output
num_models = len(predictor.get_model_names())
feature_importances = predictor.feature_importance(data=test_data)
original_features = set(train_data.columns)
original_features.remove(label)
assert set(feature_importances.index) == original_features
assert set(feature_importances.columns) == {'importance', 'stddev', 'p_value', 'n', 'p99_high', 'p99_low'}
predictor.transform_features()
predictor.transform_features(data=test_data)
predictor.info()
assert predictor.get_model_names_persisted() == [] # Assert that no models were persisted during training
assert predictor.unpersist_models() == [] # Assert that no models were unpersisted
persisted_models = predictor.persist_models(models='all', max_memory=None)
assert set(predictor.get_model_names_persisted()) == set(persisted_models) # Ensure all models are persisted
assert predictor.persist_models(models='all', max_memory=None) == [] # Ensure that no additional models are persisted on repeated calls
unpersised_models = predictor.unpersist_models()
assert set(unpersised_models) == set(persisted_models)
assert predictor.get_model_names_persisted() == [] # Assert that all models were unpersisted
# Raise exception
with pytest.raises(NetworkXError):
predictor.persist_models(models=['UNKNOWN_MODEL_1', 'UNKNOWN_MODEL_2'])
assert predictor.get_model_names_persisted() == []
assert predictor.unpersist_models(models=['UNKNOWN_MODEL_1', 'UNKNOWN_MODEL_2']) == []
predictor.persist_models(models='all', max_memory=None)
predictor.save() # Save predictor while models are persisted: Intended functionality is that they won't be persisted when loaded.
predictor_loaded = TabularPredictor.load(predictor.path) # Assert that predictor loading works
leaderboard_loaded = predictor_loaded.leaderboard(data=test_data)
assert len(leaderboard) == len(leaderboard_loaded)
assert predictor_loaded.get_model_names_persisted() == [] # Assert that models were not still persisted after loading predictor
assert(predictor.get_model_full_dict() == dict())
predictor.refit_full()
assert(len(predictor.get_model_full_dict()) == num_models)
assert(len(predictor.get_model_names()) == num_models * 2)
for model in predictor.get_model_names():
predictor.predict(data=test_data, model=model)
predictor.refit_full() # Confirm that refit_models aren't further refit.
assert(len(predictor.get_model_full_dict()) == num_models)
assert(len(predictor.get_model_names()) == num_models * 2)
predictor.delete_models(models_to_keep=[]) # Test that dry-run doesn't delete models
assert(len(predictor.get_model_names()) == num_models * 2)
predictor.predict(data=test_data)
predictor.delete_models(models_to_keep=[], dry_run=False) # Test that dry-run deletes models
assert len(predictor.get_model_names()) == 0
assert len(predictor.leaderboard()) == 0
assert len(predictor.leaderboard(extra_info=True)) == 0
try:
predictor.predict(data=test_data)
except:
pass
else:
raise AssertionError('predictor.predict should raise exception after all models are deleted')
print('Tabular Advanced Functionality Test Succeeded.')
def train(args):
model_output_dir = f'{args.output_dir}/data'
is_distributed = len(args.hosts) > 1
host_rank = args.hosts.index(args.current_host)
dist_ip_addrs = args.hosts
dist_ip_addrs.pop(host_rank)
# Load training and validation data
print(f'Train files: {os.listdir(args.train)}')
train_data = __load_input_data(args.train)
# Extract column info
target = args.init_args['label']
columns = train_data.columns.tolist()
column_dict = {"columns": columns}
with open('columns.pkl', 'wb') as f:
pickle.dump(column_dict, f)
# Train models
args.init_args['path'] = args.model_dir
#args.fit_args.pop('label', None)
predictor = TabularPredictor(**args.init_args).fit(train_data,
**args.fit_args)
# Results summary
predictor.fit_summary(verbosity=3)
#model_summary_fname_src = os.path.join(predictor.output_directory, 'SummaryOfModels.html')
model_summary_fname_src = os.path.join(args.model_dir,
'SummaryOfModels.html')
model_summary_fname_tgt = os.path.join(model_output_dir,
'SummaryOfModels.html')
if os.path.exists(model_summary_fname_src):
shutil.copy(model_summary_fname_src, model_summary_fname_tgt)
# ensemble visualization
G = predictor._trainer.model_graph
remove = [node for node, degree in dict(G.degree()).items() if degree < 1]
G.remove_nodes_from(remove)
A = nx.nx_agraph.to_agraph(G)
A.graph_attr.update(rankdir='BT')
A.node_attr.update(fontsize=10)
for node in A.iternodes():
node.attr['shape'] = 'rectagle'
A.draw(os.path.join(model_output_dir, 'ensemble-model.png'),
format='png',
prog='dot')
# Optional test data
if args.test:
print(f'Test files: {os.listdir(args.test)}')
test_data = __load_input_data(args.test)
# Test data must be labeled for scoring
if target in test_data:
# Leaderboard on test data
print('Running model on test data and getting Leaderboard...')
leaderboard = predictor.leaderboard(test_data, silent=True)
print(format_for_print(leaderboard), end='\n\n')
leaderboard.to_csv(f'{model_output_dir}/leaderboard.csv',
index=False)
# Feature importance on test data
# Note: Feature importance must be calculated on held-out (test) data.
# If calculated on training data it will be biased due to overfitting.
if args.feature_importance:
print('Feature importance:')
# Increase rows to print feature importance
pd.set_option('display.max_rows', 500)
feature_importance_df = predictor.feature_importance(test_data)
print(feature_importance_df)
feature_importance_df.to_csv(
f'{model_output_dir}/feature_importance.csv', index=True)
# Classification report and confusion matrix for classification model
if predictor.problem_type in [BINARY, MULTICLASS]:
from sklearn.metrics import classification_report, confusion_matrix
X_test = test_data.drop(target, axis=1)
y_test_true = test_data[target]
y_test_pred = predictor.predict(X_test)
y_test_pred_prob = predictor.predict_proba(X_test,
as_multiclass=True)
report_dict = classification_report(
y_test_true,
y_test_pred,
output_dict=True,
labels=predictor.class_labels)
report_dict_df = pd.DataFrame(report_dict).T
report_dict_df.to_csv(
f'{model_output_dir}/classification_report.csv',
index=True)
cm = confusion_matrix(y_test_true,
y_test_pred,
labels=predictor.class_labels)
cm_df = pd.DataFrame(cm, predictor.class_labels,
predictor.class_labels)
sns.set(font_scale=1)
cmap = 'coolwarm'
sns.heatmap(cm_df, annot=True, fmt='d', cmap=cmap)
plt.title('Confusion Matrix')
plt.ylabel('true label')
plt.xlabel('predicted label')
plt.show()
plt.savefig(f'{model_output_dir}/confusion_matrix.png')
get_roc_auc(y_test_true, y_test_pred_prob,
predictor.class_labels,
predictor.class_labels_internal, model_output_dir)
else:
warnings.warn(
'Skipping eval on test data since label column is not included.'
)
# Files summary
print(f'Model export summary:')
print(f"/opt/ml/model/: {os.listdir('/opt/ml/model/')}")
models_contents = os.listdir('/opt/ml/model/models')
print(f"/opt/ml/model/models: {models_contents}")
print(f"/opt/ml/model directory size: {du('/opt/ml/model/')}\n")
train_data, **ag_fit_args)
# --------------------------------------------------------------- Inference
if args.test_dir:
test_file = get_input_path(args.test_dir)
test_data = TabularDataset(test_file)
# Predictions
y_pred_proba = predictor.predict_proba(test_data)
if config.get("output_prediction_format", "csv") == "parquet":
y_pred_proba.to_parquet(
f"{args.output_data_dir}/predictions.parquet")
else:
y_pred_proba.to_csv(f"{args.output_data_dir}/predictions.csv")
# Leaderboard
if config.get("leaderboard", False):
lb = predictor.leaderboard(test_data, silent=False)
lb.to_csv(f"{args.output_data_dir}/leaderboard.csv")
# Feature importance
if config.get("feature_importance", False):
feature_importance = predictor.feature_importance(test_data)
feature_importance.to_csv(
f"{args.output_data_dir}/feature_importance.csv")
else:
if config.get("leaderboard", False):
lb = predictor.leaderboard(silent=False)
lb.to_csv(f"{args.output_data_dir}/leaderboard.csv")
