def load(self, path): # load model self._model = TabularPrediction.load(path) # get the column name of label with open(os.path.join(path, "params.json"), "r") as f: params = json.load(f) self._label_column = params["label"]
def model_fn(model_dir): """ Load the gluon model. Called once when hosting service starts. :param: model_dir The directory where model files are stored. :return: a model (in this case an AutoGluon network) """ net = task.load(model_dir) return net
def model_fn(model_dir): """ Load the gluon model. Called once when hosting service starts. :param: model_dir The directory where model files are stored. :return: a model (in this case a Gluon network) """ print(f'Loading model from {model_dir} with contents {os.listdir(model_dir)}') net = task.load(model_dir, verbosity=True) return net
def model_fn(model_dir): """ Load the gluon model. Called once when hosting service starts. :param: model_dir The directory where model files are stored. :return: a model (in this case a Gluon network) and the column info. """ print(f'Loading model from {model_dir} with contents {os.listdir(model_dir)}') net = task.load(model_dir, verbosity=True) with open(f'{model_dir}/code/columns.pkl', 'rb') as f: column_dict = pickle.load(f) return net, column_dict
tuning_data=task.Dataset(df=df_valid), label="label", output_directory=output_dir, time_limits=args.walltime, hyperparameter_tune=True, auto_stack=True, excluded_model_types=excluded_model_types, ) else: _, (X_test, y_test) = load_data(use_test=True) print("Convert arrays to DataFrame...") df_test = convert_to_dataframe(X_test, y_test) print("Loading models...") predictor = task.load(output_dir, verbosity=4) print("Predicting...") t1 = time.time() y_pred = predictor.predict(task.Dataset(df=df_test)) t2 = time.time() y_test = df_test.label print("Evaluation predictions...") results = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True) print(results) test_scores_path = os.path.join(here, "test_scores.json")
""" Example script for predicting columns of tables, demonstrating simple use-case """ from autogluon import TabularPrediction as task # Training time: train_data = task.Dataset(file_path='https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv') # can be local CSV file as well, returns Pandas DataFrame train_data = train_data.head(500) # subsample for faster demo print(train_data.head()) label_column = 'class' # specifies which column do we want to predict savedir = 'ag_models/' # where to save trained models predictor = task.fit(train_data=train_data, label=label_column, output_directory=savedir) # NOTE: Default settings above are intended to ensure reasonable runtime at the cost of accuracy. To maximize predictive accuracy, do this instead: predictor = task.fit(train_data=train_data, label=label_column, output_directory=savedir, presets='best_quality', eval_metric=YOUR_METRIC_NAME) results = predictor.fit_summary() # Inference time: test_data = task.Dataset(file_path='https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv') # another Pandas DataFrame y_test = test_data[label_column] test_data = test_data.drop(labels=[label_column],axis=1) # delete labels from test data since we wouldn't have them in practice print(test_data.head()) predictor = task.load(savedir) # Unnecessary, we reload predictor just to demonstrate how to load previously-trained predictor from file y_pred = predictor.predict(test_data) perf = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True)
def run_tabular_benchmarks(fast_benchmark, subsample_size, perf_threshold, seed_val, fit_args, dataset_indices=None, run_distill=False): print("Running fit with args:") print(fit_args) # Each train/test dataset must be located in single directory with the given names. train_file = 'train_data.csv' test_file = 'test_data.csv' EPS = 1e-10 # Information about each dataset in benchmark is stored in dict. # performance_val = expected performance on this dataset (lower = better),should update based on previously run benchmarks binary_dataset = { 'url': 'https://autogluon.s3.amazonaws.com/datasets/AdultIncomeBinaryClassification.zip', 'name': 'AdultIncomeBinaryClassification', 'problem_type': BINARY, 'label_column': 'class', 'performance_val': 0.129 } # Mixed types of features. multi_dataset = { 'url': 'https://autogluon.s3.amazonaws.com/datasets/CoverTypeMulticlassClassification.zip', 'name': 'CoverTypeMulticlassClassification', 'problem_type': MULTICLASS, 'label_column': 'Cover_Type', 'performance_val': 0.032 } # big dataset with 7 classes, all features are numeric. Runs SLOW. regression_dataset = { 'url': 'https://autogluon.s3.amazonaws.com/datasets/AmesHousingPriceRegression.zip', 'name': 'AmesHousingPriceRegression', 'problem_type': REGRESSION, 'label_column': 'SalePrice', 'performance_val': 0.076 } # Regression with mixed feature-types, skewed Y-values. toyregres_dataset = { 'url': 'https://autogluon.s3.amazonaws.com/datasets/toyRegression.zip', 'name': 'toyRegression', 'problem_type': REGRESSION, 'label_column': 'y', 'performance_val': 0.183 } # 1-D toy deterministic regression task with: heavy label+feature missingness, extra distraction column in test data # List containing dicts for each dataset to include in benchmark (try to order based on runtimes) datasets = [ toyregres_dataset, binary_dataset, regression_dataset, multi_dataset ] if dataset_indices is not None: # only run some datasets datasets = [datasets[i] for i in dataset_indices] # Aggregate performance summaries obtained in previous benchmark run: prev_perf_vals = [dataset['performance_val'] for dataset in datasets] previous_avg_performance = np.mean(prev_perf_vals) previous_median_performance = np.median(prev_perf_vals) previous_worst_performance = np.max(prev_perf_vals) # Run benchmark: performance_vals = [0.0] * len( datasets) # performance obtained in this run directory_prefix = './datasets/' with warnings.catch_warnings(record=True) as caught_warnings: for idx in range(len(datasets)): dataset = datasets[idx] train_data, test_data = load_data( directory_prefix=directory_prefix, train_file=train_file, test_file=test_file, name=dataset['name'], url=dataset['url']) if seed_val is not None: seed(seed_val) np.random.seed(seed_val) mx.random.seed(seed_val) print("Evaluating Benchmark Dataset %s (%d of %d)" % (dataset['name'], idx + 1, len(datasets))) directory = directory_prefix + dataset['name'] + "/" savedir = directory + 'AutogluonOutput/' shutil.rmtree( savedir, ignore_errors=True ) # Delete AutoGluon output directory to ensure previous runs' information has been removed. label_column = dataset['label_column'] y_test = test_data[label_column] test_data = test_data.drop(labels=[label_column], axis=1) if fast_benchmark: if subsample_size is None: raise ValueError( "fast_benchmark specified without subsample_size") train_data = train_data.head( subsample_size) # subsample for fast_benchmark predictor = task.fit(train_data=train_data, label=label_column, output_directory=savedir, **fit_args) results = predictor.fit_summary(verbosity=0) if predictor.problem_type != dataset['problem_type']: warnings.warn( "For dataset %s: Autogluon inferred problem_type = %s, but should = %s" % (dataset['name'], predictor.problem_type, dataset['problem_type'])) predictor = task.load( savedir) # Test loading previously-trained predictor from file y_pred = predictor.predict(test_data) perf_dict = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True) if dataset['problem_type'] != REGRESSION: perf = 1.0 - perf_dict[ 'accuracy_score'] # convert accuracy to error-rate else: perf = 1.0 - perf_dict[ 'r2_score'] # unexplained variance score. performance_vals[idx] = perf print("Performance on dataset %s: %s (previous perf=%s)" % (dataset['name'], performance_vals[idx], dataset['performance_val'])) if (not fast_benchmark) and ( performance_vals[idx] > dataset['performance_val'] * perf_threshold): warnings.warn( "Performance on dataset %s is %s times worse than previous performance." % (dataset['name'], performance_vals[idx] / (EPS + dataset['performance_val']))) if run_distill: predictor.distill(time_limits=60, augment_args={'size_factor': 0.5}) # Summarize: avg_perf = np.mean(performance_vals) median_perf = np.median(performance_vals) worst_perf = np.max(performance_vals) for idx in range(len(datasets)): print("Performance on dataset %s: %s (previous perf=%s)" % (datasets[idx]['name'], performance_vals[idx], datasets[idx]['performance_val'])) print("Average performance: %s" % avg_perf) print("Median performance: %s" % median_perf) print("Worst performance: %s" % worst_perf) if not fast_benchmark: if avg_perf > previous_avg_performance * perf_threshold: warnings.warn( "Average Performance is %s times worse than previously." % (avg_perf / (EPS + previous_avg_performance))) if median_perf > previous_median_performance * perf_threshold: warnings.warn( "Median Performance is %s times worse than previously." % (median_perf / (EPS + previous_median_performance))) if worst_perf > previous_worst_performance * perf_threshold: warnings.warn( "Worst Performance is %s times worse than previously." % (worst_perf / (EPS + previous_worst_performance))) print("Ran fit with args:") print(fit_args) # List all warnings again to make sure they are seen: print("\n\n WARNINGS:") for w in caught_warnings: warnings.warn(w.message)
def test_tabularHPO(): # Aggregate performance summaries obtained in previous benchmark run: prev_perf_vals = [dataset['performance_val'] for dataset in datasets] previous_avg_performance = np.mean(prev_perf_vals) previous_median_performance = np.median(prev_perf_vals) previous_worst_performance = np.max(prev_perf_vals) # Run benchmark: performance_vals = [0.0] * len( datasets) # performance obtained in this run with warnings.catch_warnings(record=True) as caught_warnings: for idx in range(len(datasets)): seed(seed_val) np.random.seed(seed_val) mx.random.seed(seed_val) dataset = datasets[idx] print("Evaluating Benchmark Dataset %s (%d of %d)" % (dataset['name'], idx + 1, len(datasets))) directory = dataset['name'] + "/" train_file_path = directory + train_file test_file_path = directory + test_file if (not os.path.exists(train_file_path)) or ( not os.path.exists(test_file_path)): # fetch files from s3: print("%s data not found locally, so fetching from %s" % (dataset['name'], dataset['url'])) os.system("wget " + dataset['url'] + " -O temp.zip && unzip -o temp.zip && rm temp.zip") savedir = directory + 'AutogluonOutput/' shutil.rmtree( savedir, ignore_errors=True ) # Delete AutoGluon output directory to ensure previous runs' information has been removed. label_column = dataset['label_column'] train_data = task.Dataset(file_path=train_file_path) test_data = task.Dataset(file_path=test_file_path) y_test = test_data[label_column] test_data = test_data.drop(labels=[label_column], axis=1) if fast_benchmark: train_data = train_data.head( subsample_size) # subsample for fast_benchmark predictor = None # reset from last Dataset if fast_benchmark: predictor = task.fit(train_data=train_data, label=label_column, output_directory=savedir, hyperparameter_tune=hyperparameter_tune, hyperparameters=hyperparameters, time_limits=time_limits, num_trials=num_trials, verbosity=verbosity) else: predictor = task.fit(train_data=train_data, label=label_column, output_directory=savedir, hyperparameter_tune=hyperparameter_tune, verbosity=verbosity) results = predictor.fit_summary(verbosity=0) if predictor.problem_type != dataset['problem_type']: warnings.warn( "For dataset %s: Autogluon inferred problem_type = %s, but should = %s" % (dataset['name'], predictor.problem_type, dataset['problem_type'])) predictor = None # We delete predictor here to test loading previously-trained predictor from file predictor = task.load(savedir) y_pred = predictor.predict(test_data) perf_dict = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True) if dataset['problem_type'] != REGRESSION: perf = 1.0 - perf_dict[ 'accuracy_score'] # convert accuracy to error-rate else: perf = 1.0 - perf_dict[ 'r2_score'] # unexplained variance score. performance_vals[idx] = perf print("Performance on dataset %s: %s (previous perf=%s)" % (dataset['name'], performance_vals[idx], dataset['performance_val'])) if (not fast_benchmark) and ( performance_vals[idx] > dataset['performance_val'] * perf_threshold): warnings.warn( "Performance on dataset %s is %s times worse than previous performance." % (dataset['name'], performance_vals[idx] / (EPS + dataset['performance_val']))) # Summarize: avg_perf = np.mean(performance_vals) median_perf = np.median(performance_vals) worst_perf = np.max(performance_vals) for idx in range(len(datasets)): print("Performance on dataset %s: %s (previous perf=%s)" % (datasets[idx]['name'], performance_vals[idx], datasets[idx]['performance_val'])) print("Average performance: %s" % avg_perf) print("Median performance: %s" % median_perf) print("Worst performance: %s" % worst_perf) if not fast_benchmark: if avg_perf > previous_avg_performance * perf_threshold: warnings.warn( "Average Performance is %s times worse than previously." % (avg_perf / (EPS + previous_avg_performance))) if median_perf > previous_median_performance * perf_threshold: warnings.warn( "Median Performance is %s times worse than previously." % (median_perf / (EPS + previous_median_performance))) if worst_perf > previous_worst_performance * perf_threshold: warnings.warn( "Worst Performance is %s times worse than previously." % (worst_perf / (EPS + previous_worst_performance))) # List all warnings again to make sure they are seen: print("\n\n WARNINGS:") for w in caught_warnings: warnings.warn(w.message)
def load_model(cls) -> TabularPredictor: """Load AutoGluon Tabular task model for this instance, loading it if it's not already loaded.""" if cls.model is None: cls.model = task.load(model_path, verbosity=True) return cls.model
def recommend(readMePath): method, exception, sentences_train, sentences_test, features_train, features_test, refers_train, refers_test, \ abstract_train, abstract_test, labels_train, labels_test = preprocess("features.csv",readMePath, [0], [0]) labels_test = np.expand_dims(labels_test, axis=1) test_samples = np.concatenate([sentences_test, features_test, refers_test, abstract_test, labels_test], axis=-1) testData="test.csv" data1 = pd.DataFrame(test_samples) data1.columns = data1.columns.map(lambda x: 'test' if x == (data1.shape[1] - 1) else 'train') data1.to_csv(testData, index=False) label_column = 'test' test_data = task.Dataset(file_path='test.csv') y_test = test_data[label_column] # values to predict test_data_nolab = test_data.drop(labels=[label_column], axis=1) # delete label column to prove we're not cheating dir = 'agModels-predictClass_KXM' predictor = task.load(dir) predictor.save_space() predictor.delete_models(models_to_keep='best', dry_run=False) # y_pred = predictor.predict(test_data_nolab) # print(confusion_matrix(y_test, y_pred)) # print('accuracy: ', accuracy_score(y_test, y_pred)) # print('precision: ', precision_score(y_test, y_pred)) # print('recall: ', recall_score(y_test, y_pred)) # print('f1: ', f1_score(y_test, y_pred)) # print("=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x" + # "=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x") # recommendation_path = '/home/whu/data/zxq/py_recom/tmp1' recommendation_path = 'results' method_to_link = {} links = open('link.txt', 'r') for line in links: line = line.strip() if line: methods = line.split('->') for m in methods: if m: method_to_link[m] = line links.close() method = method.tolist() exception = exception.tolist() length = features_train.shape[1] + sentences_train.shape[1] + refers_train.shape[1] + abstract_train.shape[1] headers = ['train'] headers.extend(['train.' + str(i) for i in range(1, length)]) headers.append('test') target = open('resultLink.txt', 'w') index = 0 all_key = [] all_value = [] all_str = [] for i in range(features_test.shape[0]): s_train = {} # new_features(method[i], s_train) # @profile if method_to_link.get(method[i]) is not None: linking = method_to_link[method[i]].strip() split = linking.split('->') for idx, train_method in enumerate(split): if train_method: new_fea = np.array(features_test[i], copy=True) new_fea[1] = train_method.count('.') new_fea[2] = train_method.count('@') new_fea[3] = idx new_fea[4] = len(split) - idx conc = np.concatenate( [sentences_test[i], new_fea, refers_test[i], abstract_test[i], np.zeros(shape=(1,))], axis=0).T conc = np.expand_dims(conc, axis=1) frame = pd.DataFrame(conc.T, columns=headers) # frame.columns = frame.columns.map(lambda x: 'test' if x == frame.shape[1] - 1 else 'train') s_train[train_method.strip()] = frame else: con = np.concatenate( [sentences_test[i], features_test[i], refers_test[i], abstract_test[i], np.zeros(shape=(1,))], axis=0) con = np.expand_dims(con, axis=1) data_frame = pd.DataFrame( con.T, columns=headers) # data_frame.columns = data_frame.columns.map(lambda x: 'test' if x == data_frame.shape[1] - 1 else 'train') s_train[method[i]] = data_frame all_value.extend(list(s_train.values())) all_key.append(list(s_train.keys())) content = method[i] + '::' + exception[i].strip('\n') + '::' + str(labels_test[i][0]) + '::' all_str.append(content) tmp_result = pd.concat(all_value) tmp_result = predictor.predict_proba(tmp_result) j = 0 for i, key in enumerate(all_key): content = all_str[i] for iii, k in enumerate(key): content = content + str(k).strip('\n') + '|' + str(tmp_result[j]).strip('\n') + '->' j += 1 target.write(content + '\n') target.close()
def run_tabular_benchmarks(fast_benchmark, subsample_size, perf_threshold, seed_val, fit_args, dataset_indices=None): print("Running fit with args:") print(fit_args) # Each train/test dataset must be located in single directory with the given names. train_file = 'train_data.csv' test_file = 'test_data.csv' EPS = 1e-10 # Information about each dataset in benchmark is stored in dict. # performance_val = expected performance on this dataset (lower = better),should update based on previously run benchmarks binary_dataset = {'url': 'https://autogluon.s3-us-west-2.amazonaws.com/datasets/AdultIncomeBinaryClassification.zip', 'name': 'AdultIncomeBinaryClassification', 'problem_type': BINARY, 'label_column': 'class', 'performance_val': 0.129} # Mixed types of features. multi_dataset = {'url': 'https://autogluon.s3-us-west-2.amazonaws.com/datasets/CoverTypeMulticlassClassification.zip', 'name': 'CoverTypeMulticlassClassification', 'problem_type': MULTICLASS, 'label_column': 'Cover_Type', 'performance_val': 0.032} # big dataset with 7 classes, all features are numeric. Runs SLOW. regression_dataset = {'url': 'https://autogluon.s3-us-west-2.amazonaws.com/datasets/AmesHousingPriceRegression.zip', 'name': 'AmesHousingPriceRegression', 'problem_type': REGRESSION, 'label_column': 'SalePrice', 'performance_val': 0.076} # Regression with mixed feature-types, skewed Y-values. toyregres_dataset = {'url': 'https://autogluon.s3-us-west-2.amazonaws.com/datasets/toyRegression.zip', 'name': 'toyRegression', 'problem_type': REGRESSION, 'label_column': 'y', 'performance_val': 0.183} # 1-D toy deterministic regression task with: heavy label+feature missingness, extra distraction column in test data toyclassif_dataset = {'url': 'https://autogluon.s3-us-west-2.amazonaws.com/datasets/toyClassification.zip', 'name': 'toyClassification', 'problem_type': MULTICLASS, 'label_column': 'y', 'performance_val': 0.436} # 2-D toy noisy, imbalanced 4-class classification task with: feature missingness, out-of-vocabulary feature categories in test data, out-of-vocabulary labels in test data, training column missing from test data, extra distraction columns in test data # toyclassif_dataset should produce 3 warnings: # UserWarning: These columns from this dataset were not present in the training dataset (AutoGluon will ignore them): ['distractioncolumn1', 'distractioncolumn2'] # UserWarning: The columns listed below from the training data are no longer in the given dataset. (AutoGluon will proceed assuming their values are missing, but you should remove these columns from training dataset and train a new model): ['lostcolumn'] # UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. # List containing dicts for each dataset to include in benchmark (try to order based on runtimes) datasets = [toyregres_dataset, toyclassif_dataset, binary_dataset, regression_dataset, multi_dataset] if dataset_indices is not None: # only run some datasets datasets = [datasets[i] for i in dataset_indices] # Aggregate performance summaries obtained in previous benchmark run: prev_perf_vals = [dataset['performance_val'] for dataset in datasets] previous_avg_performance = np.mean(prev_perf_vals) previous_median_performance = np.median(prev_perf_vals) previous_worst_performance = np.max(prev_perf_vals) # Run benchmark: performance_vals = [0.0] * len(datasets) # performance obtained in this run directory_prefix = './datasets/' if not os.path.exists(directory_prefix): os.mkdir(directory_prefix) with warnings.catch_warnings(record=True) as caught_warnings: for idx in range(len(datasets)): if seed_val is not None: seed(seed_val) np.random.seed(seed_val) mx.random.seed(seed_val) dataset = datasets[idx] print("Evaluating Benchmark Dataset %s (%d of %d)" % (dataset['name'], idx+1, len(datasets))) directory = directory_prefix + dataset['name'] + "/" train_file_path = directory + train_file test_file_path = directory + test_file if (not os.path.exists(train_file_path)) or (not os.path.exists(test_file_path)): # fetch files from s3: print("%s data not found locally, so fetching from %s" % (dataset['name'], dataset['url'])) zip_name = ag.download(dataset['url'], directory_prefix) ag.unzip(zip_name, directory_prefix) os.remove(zip_name) savedir = directory + 'AutogluonOutput/' shutil.rmtree(savedir, ignore_errors=True) # Delete AutoGluon output directory to ensure previous runs' information has been removed. label_column = dataset['label_column'] train_data = task.Dataset(file_path=train_file_path) test_data = task.Dataset(file_path=test_file_path) y_test = test_data[label_column] test_data = test_data.drop(labels=[label_column], axis=1) if fast_benchmark: if subsample_size is None: raise ValueError("fast_benchmark specified without subsample_size") train_data = train_data.head(subsample_size) # subsample for fast_benchmark predictor = None # reset from last Dataset predictor = task.fit(train_data=train_data, label=label_column, output_directory=savedir, **fit_args) results = predictor.fit_summary(verbosity=0) if predictor.problem_type != dataset['problem_type']: warnings.warn("For dataset %s: Autogluon inferred problem_type = %s, but should = %s" % (dataset['name'], predictor.problem_type, dataset['problem_type'])) predictor = None # We delete predictor here to test loading previously-trained predictor from file predictor = task.load(savedir) y_pred = predictor.predict(test_data) perf_dict = predictor.evaluate_predictions(y_true=y_test, y_pred=y_pred, auxiliary_metrics=True) if dataset['problem_type'] != REGRESSION: perf = 1.0 - perf_dict['accuracy_score'] # convert accuracy to error-rate else: perf = 1.0 - perf_dict['r2_score'] # unexplained variance score. performance_vals[idx] = perf print("Performance on dataset %s: %s (previous perf=%s)" % (dataset['name'], performance_vals[idx], dataset['performance_val'])) if (not fast_benchmark) and (performance_vals[idx] > dataset['performance_val'] * perf_threshold): warnings.warn("Performance on dataset %s is %s times worse than previous performance." % (dataset['name'], performance_vals[idx]/(EPS+dataset['performance_val']))) # Summarize: avg_perf = np.mean(performance_vals) median_perf = np.median(performance_vals) worst_perf = np.max(performance_vals) for idx in range(len(datasets)): print("Performance on dataset %s: %s (previous perf=%s)" % (datasets[idx]['name'], performance_vals[idx], datasets[idx]['performance_val'])) print("Average performance: %s" % avg_perf) print("Median performance: %s" % median_perf) print("Worst performance: %s" % worst_perf) if not fast_benchmark: if avg_perf > previous_avg_performance * perf_threshold: warnings.warn("Average Performance is %s times worse than previously." % (avg_perf/(EPS+previous_avg_performance))) if median_perf > previous_median_performance * perf_threshold: warnings.warn("Median Performance is %s times worse than previously." % (median_perf/(EPS+previous_median_performance))) if worst_perf > previous_worst_performance * perf_threshold: warnings.warn("Worst Performance is %s times worse than previously." % (worst_perf/(EPS+previous_worst_performance))) print("Ran fit with args:") print(fit_args) # List all warnings again to make sure they are seen: print("\n\n WARNINGS:") for w in caught_warnings: warnings.warn(w.message)
# TODO predictor = task.fit(train_data=train_data, label='test', output_directory=dir, auto_stack=True, time_limits=1800) results = predictor.fit_summary() y_test = test_data[label_column] # values to predict test_data_nolab = test_data.drop( labels=[label_column], axis=1) # delete label column to prove we're not cheating #print(test_data_nolab.head()) predictor = task.load(dir) y_pred = predictor.predict(test_data_nolab) print("acorss:", mapping[t_id]) print("Predictions: ", y_pred) print(confusion_matrix(y_test, y_pred)) print('accuracy: ', accuracy_score(y_test, y_pred)) print('precision: ', precision_score(y_test, y_pred)) print('recall: ', recall_score(y_test, y_pred)) print('f1: ', f1_score(y_test, y_pred)) print( "=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x" + "=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x=x" ) call_link_path = '../dataset/calling-link-1'
data1 = pd.DataFrame(test_samples) data1.columns = data1.columns.map(lambda x: 'test' if x == (data1.shape[1] - 1) else 'train') data1.to_csv(testData, index=False) import pandas as pd import autogluon as ag from autogluon import TabularPrediction as task from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, f1_score #autogluon label_column = 'test' dir = 'agModels-predictClass_KXM' predictor = task.load( 'agModels-predictClass_KXM' ) # unnecessary, just demonstrates how to load previously-trained predictor from file train_data = task.Dataset(file_path=trainData) test_data = task.Dataset(file_path=testData) # TODO predictor = task.fit(train_data=train_data, label='test', output_directory=dir, auto_stack=True, time_limits=600) results = predictor.fit_summary() y_test = test_data[label_column] # values to predict test_data_nolab = test_data.drop( labels=[label_column],
def run_tabular_benchmarks(subsample_size, unlabeled_subsample_size, perf_threshold, seed_val, fit_args, dataset_indices=None, run_distill=False): print("Running fit with args:") print(fit_args) # Each train/test dataset must be located in single directory with the given names. train_file = 'train_data.csv' test_file = 'test_data.csv' EPS = 1e-10 # Information about each dataset in benchmark is stored in dict. # performance_val = expected performance on this dataset (lower = better),should update based on previously run benchmarks binary_dataset = { 'url': 'https://autogluon.s3.amazonaws.com/datasets/AdultIncomeBinaryClassification.zip', 'name': 'AdultIncomeBinaryClassification', 'problem_type': BINARY, 'label_column': 'class', 'performance_val': 0.129 } # Mixed types of features. multi_dataset = { 'url': 'https://autogluon.s3.amazonaws.com/datasets/CoverTypeMulticlassClassification.zip', 'name': 'CoverTypeMulticlassClassification', 'problem_type': MULTICLASS, 'label_column': 'Cover_Type', 'performance_val': 0.032 } # big dataset with 7 classes, all features are numeric. Runs SLOW. # List containing dicts for each dataset to include in benchmark (try to order based on runtimes) datasets = [binary_dataset] #multi_dataset] if dataset_indices is not None: # only run some datasets datasets = [datasets[i] for i in dataset_indices] # Aggregate performance summaries obtained in previous benchmark run: prev_perf_vals = [dataset['performance_val'] for dataset in datasets] previous_avg_performance = np.mean(prev_perf_vals) previous_median_performance = np.median(prev_perf_vals) previous_worst_performance = np.max(prev_perf_vals) # Run benchmark: sup_performance_vals = [0.0] * len( datasets) # performance obtained in this run semi_sup_performance_vals = [0.0] * len(datasets) directory_prefix = './datasets/' with warnings.catch_warnings(record=True) as caught_warnings: for idx in range(len(datasets)): dataset = datasets[idx] data, test_data = load_data(directory_prefix=directory_prefix, train_file=train_file, test_file=test_file, name=dataset['name'], url=dataset['url']) if seed_val is not None: seed(seed_val) np.random.seed(seed_val) mx.random.seed(seed_val) print("Evaluating Benchmark Dataset %s (%d of %d)" % (dataset['name'], idx + 1, len(datasets))) directory = directory_prefix + dataset['name'] + "/" savedir = directory + 'AutogluonOutput/' shutil.rmtree( savedir, ignore_errors=True ) # Delete AutoGluon output directory to ensure previous runs' information has been removed. label_column = dataset['label_column'] test_data = test_data.sample(frac=1).head(10000) y_test = test_data[label_column] test_data = test_data.drop(labels=[label_column], axis=1) if subsample_size is None: raise ValueError("subsample_size not specified") train_data = data.head(subsample_size) unlabeled_data = data.head(unlabeled_subsample_size).drop( columns=[label_column]) custom_hyperparameters = {"Transf": {}} if supervised: predictor = task.fit(train_data=train_data, label=label_column, hyperparameters=custom_hyperparameters, problem_type=dataset['problem_type'], output_directory=savedir, **fit_args) results = predictor.fit_summary(verbosity=0) if predictor.problem_type != dataset['problem_type']: warnings.warn( "For dataset %s: Autogluon inferred problem_type = %s, but should = %s" % (dataset['name'], predictor.problem_type, dataset['problem_type'])) predictor = task.load( savedir ) # Test loading previously-trained predictor from file y_pred = predictor.predict(test_data) perf_dict = predictor.evaluate_predictions( y_true=y_test, y_pred=y_pred, auxiliary_metrics=True) if dataset['problem_type'] != REGRESSION: perf = 1.0 - perf_dict[ 'accuracy_score'] # convert accuracy to error-rate else: perf = 1.0 - perf_dict[ 'r2_score'] # unexplained variance score. sup_performance_vals[idx] = perf print("Performance on dataset %s: %s (previous perf=%s)" % (dataset['name'], sup_performance_vals[idx], dataset['performance_val'])) if run_distill: predictor.distill(time_limits=60, augment_args={'size_factor': 0.5}) if semi_supervised: predictor = task.fit(train_data=train_data, label=label_column, problem_type=dataset['problem_type'], unlabeled_data=unlabeled_data, hyperparameters=custom_hyperparameters, output_directory=savedir, **fit_args) results = predictor.fit_summary(verbosity=0) if predictor.problem_type != dataset['problem_type']: warnings.warn( "For dataset %s: Autogluon inferred problem_type = %s, but should = %s" % (dataset['name'], predictor.problem_type, dataset['problem_type'])) predictor = task.load( savedir ) # Test loading previously-trained predictor from file y_pred = predictor.predict(test_data) perf_dict = predictor.evaluate_predictions( y_true=y_test, y_pred=y_pred, auxiliary_metrics=True) if dataset['problem_type'] != REGRESSION: perf = 1.0 - perf_dict[ 'accuracy_score'] # convert accuracy to error-rate else: perf = 1.0 - perf_dict[ 'r2_score'] # unexplained variance score. semi_sup_performance_vals[idx] = perf print("Performance on dataset %s: %s (previous perf=%s)" % (dataset['name'], semi_sup_performance_vals[idx], dataset['performance_val'])) if run_distill: predictor.distill(time_limits=60, augment_args={'size_factor': 0.5}) # Summarize: avg_perf = np.mean(sup_performance_vals) median_perf = np.median(sup_performance_vals) worst_perf = np.max(sup_performance_vals) for idx in range(len(datasets)): print("Performance on dataset %s: %s (previous perf=%s)" % (datasets[idx]['name'], sup_performance_vals[idx], datasets[idx]['performance_val'])) print("Supervised Average performance: %s" % avg_perf) print("Supervised Median performance: %s" % median_perf) print("Supervised Worst performance: %s" % worst_perf) avg_perf = np.mean(semi_sup_performance_vals) median_perf = np.median(semi_sup_performance_vals) worst_perf = np.max(semi_sup_performance_vals) for idx in range(len(datasets)): print("Performance on dataset %s: %s (previous perf=%s)" % (datasets[idx]['name'], semi_sup_performance_vals[idx], datasets[idx]['performance_val'])) print("Semi-supervised Average performance: %s" % avg_perf) print("Semi-supervised Median performance: %s" % median_perf) print("Semi-supervised Worst performance: %s" % worst_perf) print("Ran fit with args:") print(fit_args) # List all warnings again to make sure they are seen: print("\n\n WARNINGS:") for w in caught_warnings: warnings.warn(w.message)