def get_data(data_path='/adult/dataset_183_adult.csv', continuous_columns = [0, 2, 4, 10, 11, 12], sensitive_attribute = "sex", limit = 1000): df = pd.read_csv(Config.get('data_path') + data_path, delimiter=',', header=0) y = df['class'] del df['class'] X = df one_hot = True X_train, X_test, y_train, y_test = train_test_split(X.values[0:limit,:], y.values[0:limit], test_size=0.5, random_state=42) sensitive_attribute_id = -1 for c_i in range(len(df.columns)): if str(df.columns[c_i]) == sensitive_attribute: sensitive_attribute_id = c_i break categorical_features = list(set(list(range(X_train.shape[1]))) - set(continuous_columns)) cat_sensitive_attribute_id = -1 for c_i in range(len(categorical_features)): if categorical_features[c_i] == sensitive_attribute_id: cat_sensitive_attribute_id = c_i break xshape = X_train.shape[1] if one_hot: ct = ColumnTransformer([("onehot", OneHotEncoder(handle_unknown='ignore', sparse=False), categorical_features)]) scale = ColumnTransformer([("scale", MinMaxScaler(), continuous_columns)]) pipeline = FeatureUnion([("o", ct),("s", scale)]) X_train = pipeline.fit_transform(X_train) xshape = X_train.shape[1] X_test = pipeline.transform(X_test) names = ct.get_feature_names() for c in continuous_columns: names.append(str(X.columns[c])) print(names) sensitive_ids = [] all_names = ct.get_feature_names() for fname_i in range(len(all_names)): if all_names[fname_i].startswith('onehot__x' + str(cat_sensitive_attribute_id) + '_'): sensitive_ids.append(fname_i) #pickle.dump(names, open("/home/felix/phd/ranking_exeriments/names.p", "wb")) le = preprocessing.LabelEncoder() le.fit(y_train) y_train = le.fit_transform(y_train) y_test = le.transform(y_test) return X_train, X_test, y_train, y_test, names, sensitive_ids
def evaluate_candidates(self, candidates): self.preprocessed_folds = [] for train, test in StratifiedKFold(n_splits=10, random_state=42).split( self.dataset.splitted_values['train'], self.current_target): self.preprocessed_folds.append((train, test)) pool = mp.Pool(processes=int(Config.get("parallelism"))) results = pool.map(self.evaluate_single_candidate, candidates) return results
def read(self): openML_path = Config.get('openml.path') info_frame = pd.read_csv(openML_path + "/info.csv") assert info_frame[info_frame['name'] == self.name][ 'MLType'].values == 'classification', "it is not a classification task" #get schema and target file = open(openML_path + "/data/" + self.name + "_columns.csv", mode='r') json_schema = file.read() file.close() schema = json.loads(json_schema) names = [s['name'] for s in schema] self.dataframe = pd.read_csv( openML_path + "/data/" + self.name + ".csv", ) self.target_column_id = np.where( self.dataframe.columns == 'target')[0][0] # get target self.target_values = self.dataframe[self.dataframe.columns[ self.target_column_id]].values self.dataframe.drop(self.dataframe.columns[self.target_column_id], axis=1, inplace=True) # get split of the data self.splitter.get_splitted_ids(self.dataframe, self.target_values) self.splitted_values = {} self.splitted_target = {} self.splitted_target['train'], self.splitted_target[ 'valid'], self.splitted_target[ 'test'] = self.splitter.materialize_target(self.target_values) self.splitted_values['train'], self.splitted_values[ 'valid'], self.splitted_values[ 'test'] = self.splitter.materialize_values(self.dataframe) for attribute_i in range(self.dataframe.shape[1]): properties = self.derive_properties( attribute_i, self.dataframe[self.dataframe.columns[attribute_i]].values) self.raw_features.append( RawFeature(self.dataframe.columns[attribute_i], attribute_i, properties)) return self.raw_features
from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from mlxtend.feature_selection import SequentialFeatureSelector as SFS import multiprocessing as mp import itertools from sklearn.ensemble import RandomForestRegressor import scipy.special import seaborn as sns import matplotlib.pyplot as plt from fastsklearnfeature.configuration.Config import Config #X_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.data', delimiter=' ', header=None).values[:,0:500] [0:100,:] #y_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values [0:100] X_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.data', delimiter=' ', header=None).values[:,0:10000][0:100,:] y_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.labels', delimiter=' ', header=None).values[0:100] data_name = 'ARCENE_sample' # generate grid complexity_grid = np.arange(1, X_train.shape[1]+1) max_acc = 1.0 accuracy_grid = np.arange(0.0, max_acc, max_acc / 100.0) def get_estimated_runtimes(old_model = "/tmp/model11_hyperopt.p"): grid = list(itertools.product(complexity_grid, accuracy_grid)) meta_X_data = np.matrix(grid)
''' data = pd.read_csv(Config.get('data_path') + '/breastTumor/breastTumor.csv', delimiter=',', header=0) y = data['binaryClass'].values X = data[data.columns.difference(['binaryClass'])].values data_name = 'breastTumor' one_hot = True ''' ''' data = pd.read_csv(Config.get('data_path') + '/promoters/dataset_106_molecular-biology_promoters.csv', delimiter=',', header=0) y = data['class'].values X = data[data.columns.difference(['class', 'instance'])].values data_name = 'promoters' one_hot = True ''' X = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.data', delimiter=' ', header=None).values[:, 0:500] y = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values data_name = 'madelon' one_hot = False ''' X_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.data', delimiter=' ', header=None).values[:,0:10000] y_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.labels', delimiter=' ', header=None).values data_name = 'ARCENE' one_hot = False ''' print(X.shape)
if __name__ == '__main__': #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_27_colic_horse.csv", 22) #dataset = ("/home/felix/datasets/ExploreKit/csv/phpAmSP4g_cancer.csv", 30) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_29_credit-a_credit.csv", 15) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_37_diabetes_diabetes.csv", 8) #dataset = (Config.get('data_path') + "/phpn1jVwe_mammography.csv", 6) #dataset = (Config.get('data_path') + "/dataset_23_cmc_contraceptive.csv", 9) #dataset = (Config.get('data_path') + "/dataset_31_credit-g_german_credit.csv", 20) #dataset = (Config.get('data_path') + '/dataset_53_heart-statlog_heart.csv', 13) #dataset = (Config.get('data_path') + '/ILPD.csv', 10) #dataset = (Config.get('data_path') + '/iris.data', 4) #dataset = (Config.get('data_path') + '/data_banknote_authentication.txt', 4) #dataset = (Config.get('data_path') + '/ecoli.data', 8) #dataset = (Config.get('data_path') + '/breast-cancer.data', 0) dataset = (Config.get('data_path') + '/transfusion.data', 4) #dataset = (Config.get('data_path') + '/test_categorical.data', 4) #dataset = ('../configuration/resources/data/transfusion.data', 4) #dataset = (Config.get('data_path') + '/wine.data', 0) #dataset = (Config.get('data_path') + '/house_price.csv', 79) #dataset = (Config.get('data_path') + '/synthetic_data.csv', 3) from fastsklearnfeature.reader.OnlineOpenMLReader import OnlineOpenMLReader from fastsklearnfeature.feature_selection.evaluation.openMLdict import openMLname2task # task_id = openMLname2task['transfusion'] #interesting # task_id = openMLname2task['iris'] # task_id = openMLname2task['ecoli'] # task_id = openMLname2task['breast cancer']
#y_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values [0:100] ''' X_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.data', delimiter=' ', header=None).values[:,0:10000] y_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.labels', delimiter=' ', header=None).values data_name = 'ARCENE' my_path = "/home/felix/phd/feature_constraints/experiments_arcene/" onehot = False ''' ''' data = pd.read_csv(Config.get('data_path') + '/musk/musk.csv', delimiter=',', header=0) y_train = data['class'] X_train = data[data.columns.difference(['class', 'ID', 'molecule_name', 'conformation_name'])].values data_name = 'musk' ''' data = pd.read_csv(Config.get('data_path') + '/breastTumor/breastTumor.csv', delimiter=',', header=0) y_train = data['binaryClass'].values X_train = data[data.columns.difference(['binaryClass'])].values data_name = 'breastTumor' my_path = "/home/felix/phd/feature_constraints/experiments_actual_tumor/" onehot = True ''' data = pd.read_csv(Config.get('data_path') + '/promoters/dataset_106_molecular-biology_promoters.csv', delimiter=',', header=0) y_train = data['class'].values X_train = data[data.columns.difference(['class', 'instance'])].values data_name = 'promoters' my_path = "/home/felix/phd/feature_constraints/experiments_promoters/" onehot = True '''
self.generate() #starting_feature_matrix = self.create_starting_features() self.generate_target() print([r.name for r in self.dataset.raw_features]) plain_attributes = CandidateFeature(IdentityTransformation(len(self.dataset.raw_features)), self.dataset.raw_features) self.evaluate_candidates([plain_attributes]) #statlog_heart.csv=/home/felix/datasets/ExploreKit/csv/dataset_53_heart-statlog_heart.csv #statlog_heart.target=13 if __name__ == '__main__': dataset = (Config.get('statlog_heart.csv'), 13) selector = SissoExperiment(dataset) selector.run()
from sklearn.feature_selection import f_classif from sklearn.feature_selection import chi2 from sklearn.model_selection import cross_val_score from fastsklearnfeature.interactiveAutoML.fair_measure import true_positive_rate_score from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.robust_measure import robust_score from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.robust_measure import robust_score_test import fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.my_global_variable as my_global_variable sensitive_attribute = "sex" n_estimators = 5 df = pd.read_csv(Config.get('data_path') + '/adult/dataset_183_adult.csv', delimiter=',', header=0) y = df['class'] del df['class'] X = df one_hot = True limit = 1000 X_train, X_test, y_train, y_test = train_test_split(X.values[0:limit,:], y.values[0:limit], test_size=0.5, random_state=42) from sklearn import preprocessing le = preprocessing.LabelEncoder() le.fit(y_train) y_train = le.fit_transform(y_train) y_test = le.transform(y_test)
from sklearn.model_selection import cross_val_score from sklearn.preprocessing import MinMaxScaler import fastsklearnfeature.interactiveAutoML.feature_selection.WrapperBestK as wrap from sklearn.ensemble import ExtraTreesClassifier from hyperopt.fmin import generate_trials_to_calculate ''' data = pd.read_csv(Config.get('data_path') + '/breastTumor/breastTumor.csv', delimiter=',', header=0) y = data['binaryClass'].values X = data[data.columns.difference(['binaryClass'])].values data_name = 'breastTumor' one_hot = True ''' data = pd.read_csv(Config.get('data_path') + '/promoters/dataset_106_molecular-biology_promoters.csv', delimiter=',', header=0) y = data['class'].values X = data[data.columns.difference(['class', 'instance'])].values data_name = 'promoters' one_hot = True ''' X_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.data', delimiter=' ', header=None).values[:,0:500] y_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values data_name = 'madelon' one_hot = False ''' ''' X_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.data', delimiter=' ', header=None).values[:,0:10000]
number_instances = [] number_attributes = [] number_features = [] def get_class_attribute_name(df): for i in range(len(df.columns)): if str(df.columns[i]).startswith('class@'): return str(df.columns[i]) def get_sensitive_attribute_id(df, sensitive_attribute_name): for i in range(len(df.columns)): if str(df.columns[i]) == sensitive_attribute_name: return i with open(Config.get('data_path') + "/downloaded_arff/" + "42132.arff") as f: df = a2p.load(f) print(df.columns) print(df.head()) #df['TotalCharges@REAL'] = pd.to_numeric(df['TotalCharges@STRING'], errors='coerce') df = df.drop(columns=['geolocation@STRING']) df = df.drop(columns=['seqid@STRING']) df = df.drop(columns=['date_of_stop@STRING']) df = df.drop(columns=['time_of_stop@STRING']) df = df.drop(columns=['description@STRING']) df = df.drop(columns=['location@STRING']) df.rename(columns={'race@{ASIAN,BLACK,HISPANIC,NATIVE AMERICAN,OTHER,WHITE}': 'race@{BLACKLIVESMATTER,OTHER}'}, inplace=True)
from sklearn.model_selection import cross_val_score from sklearn.preprocessing import MinMaxScaler import fastsklearnfeature.interactiveAutoML.feature_selection.WrapperBestK as wrap from sklearn.ensemble import ExtraTreesClassifier ''' data = pd.read_csv(Config.get('data_path') + '/breastTumor/breastTumor.csv', delimiter=',', header=0) y = data['binaryClass'].values X = data[data.columns.difference(['binaryClass'])].values data_name = 'breastTumor' one_hot = True ''' data = pd.read_csv(Config.get('data_path') + '/promoters/dataset_106_molecular-biology_promoters.csv', delimiter=',', header=0) y = data['class'].values X = data[data.columns.difference(['class', 'instance'])].values data_name = 'promoters' one_hot = True ''' X_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.data', delimiter=' ', header=None).values[:,0:500] y_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values data_name = 'madelon' one_hot = False '''
def run(self): self.global_starting_time = time.time() # generate all candidates self.generate() for raw_f in self.raw_features: raw_f.properties['type'] = 'float' #starting_feature_matrix = self.create_starting_features() self.generate_target() myfolds = copy.deepcopy(list(self.preprocessed_folds)) R_w = 15000 max_iterations = 15 #15 threshold_f = 0.001 epsilon_w = 0.01 threshold_w = 0.0 all_features = self.produce_features() print(len(all_features)) self.base_features = CandidateFeature( IdentityTransformation(len(self.raw_features)), self.raw_features) results = {} for i in range(max_iterations): print("base features: " + str(self.base_features)) results[i] = self.evaluate_candidates([self.base_features], myfolds)[0] print(results[i]) print(results[i].runtime_properties) feature_scores = self.evaluate_ranking(all_features) ids = np.argsort(np.array(feature_scores) * -1) print(feature_scores) best_improvement_so_far = np.NINF best_Feature_So_Far = None evaluated_candidate_features = 0 for f_i in range(len(feature_scores)): if feature_scores[ids[f_i]] < threshold_f: break current_feature_set = CandidateFeature( IdentityTransformation(2), [self.base_features, all_features[ids[f_i]]]) print(current_feature_set) result = self.evaluate_candidates([current_feature_set], myfolds)[0] evaluated_candidate_features += 1 improvement = result.runtime_properties['score'] - results[ i].runtime_properties['score'] print("Candidate: " + str(all_features[ids[f_i]]) + " score: " + str(result.runtime_properties['score']) + " info: " + str(feature_scores[ids[f_i]])) print("improvement: " + str(improvement)) if improvement > best_improvement_so_far: best_improvement_so_far = improvement best_Feature_So_Far = result results[i] = best_Feature_So_Far results[i].runtime_properties[ 'score_improvement'] = improvement results[i].runtime_properties[ 'info_gain'] = feature_scores[ids[f_i]] results[i].runtime_properties['global time'] = time.time( ) - self.global_starting_time pickle.dump( results, open( Config.get("tmp.folder") + "/explorekit_results.p", "wb")) if improvement >= epsilon_w: break if evaluated_candidate_features >= R_w: break if best_improvement_so_far > threshold_w: self.base_features = best_Feature_So_Far else: return self.base_features all_features_new = [] for i in range(len(feature_scores)): if feature_scores[i] >= 0: all_features_new.append(all_features[i]) all_features = all_features_new return results
def evaluate_ranking(self, candidates): self.preprocessed_folds = [] pool = mp.Pool(processes=int(Config.get("parallelism"))) results = pool.map(self.get_info_gain_of_feature, candidates) return results
import openml from fastsklearnfeature.configuration.Config import Config import pickle import numpy as np import random from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import OneHotEncoder from sklearn.compose import ColumnTransformer from fastsklearnfeature.configuration.Config import Config from sklearn import preprocessing import openml from sklearn.pipeline import FeatureUnion from sklearn.model_selection import train_test_split openml.config.apikey = Config.get('openML.apikey') unique_data = {} for _, data_info in openml.datasets.list_datasets().items(): if 'status' in data_info and data_info['status'] == 'active' \ and 'NumberOfClasses' in data_info and data_info['NumberOfClasses'] == 2 \ and 'NumberOfInstances' in data_info and data_info['NumberOfInstances'] > 250: try: dataset = openml.datasets.get_dataset(data_info['did']) print(data_info) continuous_columns = [] categorical_features = []
def get_fair_data1(dataset_key=None): map_dataset = {} map_dataset['31'] = 'foreign_worker@{yes,no}' map_dataset['802'] = 'sex@{female,male}' map_dataset['1590'] = 'sex@{Female,Male}' map_dataset['1461'] = 'AGE@{True,False}' map_dataset['42193'] = 'race_Caucasian@{0,1}' map_dataset['1480'] = 'V2@{Female,Male}' # map_dataset['804'] = 'Gender@{0,1}' map_dataset['42178'] = 'gender@STRING' map_dataset['981'] = 'Gender@{Female,Male}' map_dataset['40536'] = 'samerace@{0,1}' map_dataset['40945'] = 'sex@{female,male}' map_dataset['451'] = 'Sex@{female,male}' # map_dataset['945'] = 'sex@{female,male}' map_dataset['446'] = 'sex@{Female,Male}' map_dataset['1017'] = 'sex@{0,1}' map_dataset['957'] = 'Sex@{0,1,4}' map_dataset['41430'] = 'SEX@{True,False}' map_dataset['1240'] = 'sex@{Female,Male}' map_dataset['1018'] = 'sex@{Female,Male}' # map_dataset['55'] = 'SEX@{male,female}' map_dataset['38'] = 'sex@{F,M}' map_dataset['1003'] = 'sex@{male,female}' map_dataset['934'] = 'race@{black,white}' number_instances = [] number_attributes = [] number_features = [] def get_class_attribute_name(df): for i in range(len(df.columns)): if str(df.columns[i]).startswith('class@'): return str(df.columns[i]) def get_sensitive_attribute_id(df, sensitive_attribute_name): for i in range(len(df.columns)): if str(df.columns[i]) == sensitive_attribute_name: return i key = dataset_key if type(dataset_key) == type(None): key = list(map_dataset.keys())[random.randint(0, len(map_dataset) - 1)] value = map_dataset[key] with open(Config.get('data_path') + "/downloaded_arff/" + str(key) + ".arff") as f: df = a2p.load(f) print("dataset: " + str(key)) number_instances.append(df.shape[0]) number_attributes.append(df.shape[1]) y = copy.deepcopy(df[get_class_attribute_name(df)]) X = df.drop(columns=[get_class_attribute_name(df)]) categorical_features = [] continuous_columns = [] for type_i in range(len(X.columns)): if X.dtypes[type_i] == object: categorical_features.append(type_i) else: continuous_columns.append(type_i) sensitive_attribute_id = get_sensitive_attribute_id(X, value) print(sensitive_attribute_id) X_datat = X.values for x_i in range(X_datat.shape[0]): for y_i in range(X_datat.shape[1]): if type(X_datat[x_i][y_i]) == type(None): if X.dtypes[y_i] == object: X_datat[x_i][y_i] = 'missing' else: X_datat[x_i][y_i] = np.nan X_train, X_test, y_train, y_test = train_test_split(X_datat, y.values.astype('str'), test_size=0.5, random_state=42, stratify=y.values.astype('str')) ''' X_train, X_test, y_train, y_test = train_test_split(X_datat[0:200,:], y.values[0:200].astype('str'), test_size=0.5, random_state=42, stratify=y.values[0:200].astype('str')) ''' cat_sensitive_attribute_id = -1 for c_i in range(len(categorical_features)): if categorical_features[c_i] == sensitive_attribute_id: cat_sensitive_attribute_id = c_i break my_transformers = [] if len(categorical_features) > 0: ct = ColumnTransformer( [("onehot", OneHotEncoder(handle_unknown='ignore', sparse=False), categorical_features)]) my_transformers.append(("o", ct)) if len(continuous_columns) > 0: scale = ColumnTransformer([("scale", Pipeline( [('impute', SimpleImputer(missing_values=np.nan, strategy='mean')), ('scale', MinMaxScaler())]), continuous_columns)]) my_transformers.append(("s", scale)) pipeline = FeatureUnion(my_transformers) pipeline.fit(X_train) X_train = pipeline.transform(X_train) X_test = pipeline.transform(X_test) number_features.append(X_train.shape[1]) all_columns = [] for ci in range(len(X.columns)): all_columns.append(str(X.columns[ci]).split('@')[0]) X.columns = all_columns names = ct.get_feature_names() for c in continuous_columns: names.append(str(X.columns[c])) for n_i in range(len(names)): if names[n_i].startswith('onehot__x'): tokens = names[n_i].split('_') category = '' for ti in range(3, len(tokens)): category += '_' + tokens[ti] cat_id = int(names[n_i].split('_')[2].split('x')[1]) names[n_i] = str(X.columns[categorical_features[cat_id]]) + category print(names) sensitive_ids = [] all_names = ct.get_feature_names() for fname_i in range(len(all_names)): if all_names[fname_i].startswith('onehot__x' + str(cat_sensitive_attribute_id) + '_'): sensitive_ids.append(fname_i) le = preprocessing.LabelEncoder() le.fit(y_train) y_train = le.fit_transform(y_train) y_test = le.transform(y_test) return X_train, X_test, y_train, y_test, names, sensitive_ids, key, sensitive_attribute_id
import itertools from sklearn.ensemble import RandomForestRegressor import scipy.special import seaborn as sns import matplotlib.pyplot as plt from fastsklearnfeature.configuration.Config import Config #X_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.data', delimiter=' ', header=None).values[:,0:500] [0:100,:] #y_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values [0:100] ''' X_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.data', delimiter=' ', header=None).values[:,0:10000][0:100,:] y_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.labels', delimiter=' ', header=None).values[0:100] data_name = 'ARCENE_sample' ''' X_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.data', delimiter=' ', header=None).values[:, 0:10000][0:1000, :] y_train = pd.read_csv(Config.get('data_path') + '/ARCENE/arcene_train.labels', delimiter=' ', header=None).values[0:1000] data_name = 'ARCENE_sample1k' # generate grid complexity_grid = np.arange(1, X_train.shape[1] + 1) max_acc = 1.0 accuracy_grid = np.arange(0.0, max_acc, max_acc / 100.0) def get_estimated_runtimes(old_model="/tmp/model11_hyperopt.p"):
def run_strategy(strategy_method, ranking_id, strategy_id): data_infos = pickle.load( open( Config.get('data_path') + '/openml_data/fitting_datasets.pickle', 'rb')) time_limit = 60 * 20 meta_classifier = RandomForestRegressor(n_estimators=1000) X_train_meta_classifier = [] y_train_meta_classifier = [] cv_splitter = StratifiedKFold(5, random_state=42) auc_scorer = make_scorer(roc_auc_score, greater_is_better=True, needs_threshold=True) acc_value_list = [] fair_value_list = [] robust_value_list = [] success_value_list = [] runtime_value_list = [] dataset_did_list = [] dataset_sensitive_attribute_list = [] while True: X_train, X_test, y_train, y_test, names, sensitive_ids, data_did, sensitive_attribute_id = get_data_openml( data_infos) #run on tiny sample X_train_tiny, _, y_train_tiny, _ = train_test_split(X_train, y_train, train_size=100, random_state=42, stratify=y_train) fair_train_tiny = make_scorer( true_positive_rate_score, greater_is_better=True, sensitive_data=X_train_tiny[:, sensitive_ids[0]]) def objective(hps): print(hps) try: cv_k = 1.0 cv_privacy = hps['privacy'] model = LogisticRegression() if type(cv_privacy) == type(None): cv_privacy = X_train_tiny.shape[0] else: model = models.LogisticRegression(epsilon=cv_privacy) robust_scorer = make_scorer(robust_score, greater_is_better=True, X=X_train_tiny, y=y_train_tiny, model=model, feature_selector=None, scorer=auc_scorer) cv = GridSearchCV(model, param_grid={'C': [1.0]}, scoring={ 'AUC': auc_scorer, 'Fairness': fair_train_tiny, 'Robustness': robust_scorer }, refit=False, cv=cv_splitter) cv.fit(X_train_tiny, pd.DataFrame(y_train_tiny)) cv_acc = cv.cv_results_['mean_test_AUC'][0] cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0] cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0] small_start_time = time.time() cv = GridSearchCV(model, param_grid={'C': [1.0]}, scoring={ 'AUC': auc_scorer, 'Fairness': fair_train_tiny, 'Robustness': robust_scorer }, refit=False, cv=cv_splitter) cv.fit(X_train_tiny, pd.DataFrame(y_train_tiny)) cv_acc = cv.cv_results_['mean_test_AUC'][0] cv_fair = 1.0 - cv.cv_results_['mean_test_Fairness'][0] cv_robust = 1.0 - cv.cv_results_['mean_test_Robustness'][0] # construct feature vector feature_list = [] # user-specified constraints feature_list.append(hps['accuracy']) feature_list.append(hps['fairness']) feature_list.append(hps['k']) feature_list.append(hps['k'] * X_train.shape[1]) feature_list.append(hps['robustness']) feature_list.append(cv_privacy) feature_list.append(hps['search_time']) # differences to sample performance feature_list.append(cv_acc - hps['accuracy']) feature_list.append(cv_fair - hps['fairness']) feature_list.append(cv_k - hps['k']) feature_list.append((cv_k - hps['k']) * X_train.shape[1]) feature_list.append(cv_robust - hps['robustness']) feature_list.append(time.time() - small_start_time) # privacy constraint is always satisfied => difference always zero => constant => unnecessary # metadata features feature_list.append(X_train.shape[0]) # number rows feature_list.append(X_train.shape[1]) # number columns features = np.array(feature_list) #predict the best model and calculate uncertainty loss = 0 if hasattr(meta_classifier, 'estimators_'): predictions = [] for tree in range(len(meta_classifier.estimators_)): predictions.append( meta_classifier.estimators_[tree].predict( [features])[0]) stddev = np.std(np.array(predictions), axis=0) print('stddev: ' + str(stddev)) loss = (stddev**2) * -1 return { 'loss': loss, 'status': STATUS_OK, 'features': features } except: return {'loss': np.inf, 'status': STATUS_OK} space = { 'k': hp.choice('k_choice', [(1.0), (hp.uniform('k_specified', 0, 1))]), 'accuracy': hp.uniform('accuracy_specified', 0.5, 1), 'fairness': hp.choice('fairness_choice', [(0.0), (hp.uniform('fairness_specified', 0, 1))]), 'privacy': hp.choice('privacy_choice', [(None), (hp.lognormal('privacy_specified', 0, 1))]), 'robustness': hp.choice('robustness_choice', [(0.0), (hp.uniform('robustness_specified', 0, 1))]), 'search_time': hp.uniform('search_time_specified', 10, time_limit), # in seconds } trials = Trials() runs_per_dataset = 0 i = 1 while True: fmin(objective, space=space, algo=tpe.suggest, max_evals=i, trials=trials) i += 1 if trials.trials[-1]['result']['loss'] == np.inf: break #break, once convergence tolerance is reached and generate new dataset if trials.trials[-1]['result']['loss'] == 0 or i % 20 == 0: best_trial = trials.trials[-1] if i % 20 == 0: best_trial = trials.best_trial most_uncertain_f = best_trial['misc']['vals'] #print(most_uncertain_f) min_accuracy = most_uncertain_f['accuracy_specified'][0] min_fairness = 0.0 if most_uncertain_f['fairness_choice'][0]: min_fairness = most_uncertain_f['fairness_specified'][0] min_robustness = 0.0 if most_uncertain_f['robustness_choice'][0]: min_robustness = most_uncertain_f['robustness_specified'][ 0] max_number_features = X_train.shape[1] if most_uncertain_f['k_choice'][0]: max_number_features = most_uncertain_f['k_specified'][0] max_search_time = most_uncertain_f['search_time_specified'][0] # Execute each search strategy with a given time limit (in parallel) # maybe run multiple times to smooth stochasticity model = LogisticRegression() if most_uncertain_f['privacy_choice'][0]: model = models.LogisticRegression( epsilon=most_uncertain_f['privacy_specified'][0]) rankings = [variance, chi2_score_wo] # simple rankings rankings.append( partial(model_score, estimator=ExtraTreesClassifier( n_estimators=1000))) # accuracy ranking rankings.append( partial(robustness_score, model=model, scorer=auc_scorer)) # robustness ranking rankings.append( partial(fairness_score, estimator=ExtraTreesClassifier(n_estimators=1000), sensitive_ids=sensitive_ids)) # fairness ranking selected_rankings = rankings if type(ranking_id) != type(None): selected_rankings = [rankings[ranking_id]] result = strategy_method( X_train, X_test, y_train, y_test, names, sensitive_ids, ranking_functions=selected_rankings, clf=model, min_accuracy=min_accuracy, min_fairness=min_fairness, min_robustness=min_robustness, max_number_features=max_number_features, max_search_time=max_search_time, cv_splitter=cv_splitter) # append ml data X_train_meta_classifier.append( best_trial['result']['features']) y_train_meta_classifier.append(result['time']) try: meta_classifier.fit(np.array(X_train_meta_classifier), y_train_meta_classifier) except: pass #pickle everything and store it one_big_object = {} one_big_object['features'] = X_train_meta_classifier #one_big_object['best_strategy'] = y_train_meta_classifier runtime_value_list.append(result['time']) acc_value_list.append(result['cv_acc']) fair_value_list.append(result['cv_fair']) robust_value_list.append(result['cv_robust']) success_value_list.append(result['success']) dataset_did_list.append(data_did) dataset_sensitive_attribute_list.append(sensitive_attribute_id) one_big_object['times_value'] = runtime_value_list one_big_object['acc_value'] = acc_value_list one_big_object['fair_value'] = fair_value_list one_big_object['robust_value'] = robust_value_list one_big_object['success_value'] = success_value_list one_big_object['dataset_id'] = dataset_did_list one_big_object[ 'sensitive_attribute_id'] = dataset_sensitive_attribute_list pickle.dump( one_big_object, open( '/tmp/metalearning_data' + str(strategy_id) + '.pickle', 'wb')) trials = Trials() i = 1 runs_per_dataset += 1 break
from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.metalearning.strategies.exhaustive import exhaustive from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.metalearning.strategies.forward_floating_selection import forward_selection from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.metalearning.strategies.backward_floating_selection import backward_selection from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.metalearning.strategies.forward_floating_selection import forward_floating_selection from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.metalearning.strategies.backward_floating_selection import backward_floating_selection from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.metalearning.strategies.recursive_feature_elimination import recursive_feature_elimination #static constraints: fairness, number of features (absolute and relative), robustness, privacy, accuracy from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.bench_utils import get_fair_data1 from concurrent.futures import TimeoutError from pebble import ProcessPool, ProcessExpired #load list of viable datasets data_infos = pickle.load(open(Config.get('data_path') + '/openml_data/fitting_datasets.pickle', 'rb')) current_run_time_id = time.time() time_limit = 60 * 60 * 3 n_jobs = 20 number_of_runs = 1 X_train_meta_classifier = [] y_train_meta_classifier = [] ranking_scores_info = [] acc_value_list = [] fair_value_list = []
for i in range(len(df.columns)): if str(df.columns[i]).startswith('class@'): return str(df.columns[i]) def get_sensitive_attribute_id(df, sensitive_attribute_name): for i in range(len(df.columns)): if str(df.columns[i]) == sensitive_attribute_name: return i random_number = 42 for key, value in map_dataset.items(): with open( Config.get('data_path') + "/downloaded_arff/" + str(key) + ".arff") as f: df = a2p.load(f) print("dataset: " + str(key)) name_dataset = openml.datasets.get_dataset(dataset_id=int(key), download_data=False).name number_instances = df.shape[0] number_attributes = df.shape[1] y = copy.deepcopy(df[get_class_attribute_name(df)]) X = df.drop(columns=[get_class_attribute_name(df)]) categorical_features = []
hp.uniform('informative_specified', 0, 1), 'n_redundant': hp.uniform('redundant_specified', 0, 1), 'n_repeated': hp.uniform('repeated_specified', 0, 1), 'n_useless': hp.uniform('useless_specified', 0, 1), 'n_clusters_per_class': hp.randint('clusters_specified', 1, 10), } configurations = [] try: configurations = pickle.load( open( Config.get('data_path') + "/scaling_configurations_samples/scaling_configurations_models.pickle", "rb")) except: while len(configurations) < 100: my_config = hyperopt.pyll.stochastic.sample(space) try: generate_data(100, 50, my_config, 0) configurations.append(my_config) except: continue pickle.dump( configurations, open( Config.get('data_path') +
from sklearn.model_selection import train_test_split from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import OneHotEncoder from sklearn.compose import ColumnTransformer #static constraints: fairness, number of features (absolute and relative), robustness, privacy, accuracy from fastsklearnfeature.interactiveAutoML.new_bench.multiobjective.bench_utils import get_fair_data1 from concurrent.futures import TimeoutError from pebble import ProcessPool, ProcessExpired from arff2pandas import a2p #load list of viable datasets data_infos = pickle.load( open( Config.get('data_path') + '/openml_data/fitting_datasets.pickle', 'rb')) current_run_time_id = time.time() time_limit = 60 * 60 * 3 n_jobs = 20 number_of_runs = 1 X_train_meta_classifier = [] y_train_meta_classifier = [] ranking_scores_info = [] acc_value_list = [] fair_value_list = []
print("(" + str(r + 1) + "," + str(results[r]['test_score']) + ")") #new_scores = [r['test_score'] for r in results] #best_id = np.argmax(new_scores) #print(results[best_id]) #statlog_heart.csv=/home/felix/datasets/ExploreKit/csv/dataset_53_heart-statlog_heart.csv #statlog_heart.target=13 if __name__ == '__main__': #dataset = (Config.get('statlog_heart.csv'), int(Config.get('statlog_heart.target'))) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_27_colic_horse.csv", 22) #dataset = ("/home/felix/datasets/ExploreKit/csv/phpAmSP4g_cancer.csv", 30) # dataset = ("/home/felix/datasets/ExploreKit/csv/phpOJxGL9_indianliver.csv", 10) # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_29_credit-a_credit.csv", 15) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_37_diabetes_diabetes.csv", 8) # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_31_credit-g_german_credit.csv", 20) # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_23_cmc_contraceptive.csv", 9) # dataset = ("/home/felix/datasets/ExploreKit/csv/phpn1jVwe_mammography.csv", 6) dataset = (Config.get('transfusion.csv'), 4) selector = ExploreKitSelection_iterative_search(dataset) #selector = ExploreKitSelection(dataset, KNeighborsClassifier(), {'n_neighbors': np.arange(3,10), 'weights': ['uniform','distance'], 'metric': ['minkowski','euclidean','manhattan']}) results = selector.run() pickle.dump(results, open("/tmp/all_data_iterations.p", "wb"))
from sklearn.metrics import precision_score from sklearn.feature_selection import SelectKBest from sklearn.decomposition import PCA from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from mlxtend.feature_selection import SequentialFeatureSelector as SFS import multiprocessing as mp import itertools from sklearn.ensemble import RandomForestRegressor import scipy.special import seaborn as sns import matplotlib.pyplot as plt from fastsklearnfeature.configuration.Config import Config X_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.data', delimiter=' ', header=None).values[:,0:500] [0:100,:] y_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values [0:100] # generate grid complexity_grid = np.arange(1, X_train.shape[1]+1) max_acc = 1.0 accuracy_grid = np.arange(0.0, max_acc, max_acc / len(complexity_grid)) def get_estimated_runtimes(old_model = "/tmp/model11_hyperopt.p"): grid = list(itertools.product(complexity_grid, accuracy_grid)) meta_X_data = np.matrix(grid) al_model = pickle.load(open(old_model, "rb")) runtime_predictions = al_model.predict(meta_X_data)
def filter_failing_in_parallel(self): pool = mp.Pool(processes=int(Config.get("parallelism"))) results = pool.map(self.filter_candidate, self.candidates) return list(itertools.chain(*results))
if rep != None: all_representations.append(rep) all_seen_representations.append(rep) if __name__ == '__main__': #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_27_colic_horse.csv", 22) #dataset = ("/home/felix/datasets/ExploreKit/csv/phpAmSP4g_cancer.csv", 30) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_29_credit-a_credit.csv", 15) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_37_diabetes_diabetes.csv", 8) #dataset = (Config.get('data_path') + "/phpn1jVwe_mammography.csv", 6) #dataset = (Config.get('data_path') + "/dataset_23_cmc_contraceptive.csv", 9) #dataset = (Config.get('data_path') + "/dataset_31_credit-g_german_credit.csv", 20) dataset = (Config.get('data_path') + '/dataset_53_heart-statlog_heart.csv', 13) #dataset = (Config.get('data_path') + '/ILPD.csv', 10) #dataset = (Config.get('data_path') + '/iris.data', 4) #dataset = (Config.get('data_path') + '/data_banknote_authentication.txt', 4) #dataset = (Config.get('data_path') + '/ecoli.data', 8) #dataset = (Config.get('data_path') + '/breast-cancer.data', 0) #dataset = (Config.get('data_path') + '/transfusion.data', 4) #dataset = (Config.get('data_path') + '/test_categorical.data', 4) #dataset = ('../configuration/resources/data/transfusion.data', 4) #dataset = (Config.get('data_path') + '/wine.data', 0) #dataset = (Config.get('data_path') + '/house_price.csv', 79) #dataset = (Config.get('data_path') + '/synthetic_data.csv', 3)
start_time = time.time() results = self.evaluate_candidates( np.array(self.candidates)[ranked_selected_candidate_ids]) print("evaluation time: " + str((time.time() - start_time) / 60) + " min") return start_score, results, ranked_selected_candidate_ids #statlog_heart.csv=/home/felix/datasets/ExploreKit/csv/dataset_53_heart-statlog_heart.csv #statlog_heart.target=13 if __name__ == '__main__': dataset = (Config.get('statlog_heart.csv'), int(Config.get('statlog_heart.target'))) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_27_colic_horse.csv", 22) #dataset = ("/home/felix/datasets/ExploreKit/csv/phpAmSP4g_cancer.csv", 30) # dataset = ("/home/felix/datasets/ExploreKit/csv/phpOJxGL9_indianliver.csv", 10) # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_29_credit-a_credit.csv", 15) #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_37_diabetes_diabetes.csv", 8) # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_31_credit-g_german_credit.csv", 20) # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_23_cmc_contraceptive.csv", 9) # dataset = ("/home/felix/datasets/ExploreKit/csv/phpn1jVwe_mammography.csv", 6) selector = ExploreKitSelection(dataset) #selector = ExploreKitSelection(dataset, KNeighborsClassifier(), {'n_neighbors': np.arange(3,10), 'weights': ['uniform','distance'], 'metric': ['minkowski','euclidean','manhattan']}) start_score, results, ids = selector.run()
from sklearn.metrics import recall_score from sklearn.metrics import precision_score from sklearn.feature_selection import SelectKBest from sklearn.decomposition import PCA from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from mlxtend.feature_selection import SequentialFeatureSelector as SFS import multiprocessing as mp import itertools from sklearn.ensemble import RandomForestRegressor import scipy.special import seaborn as sns import matplotlib.pyplot as plt from fastsklearnfeature.configuration.Config import Config X_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.data', delimiter=' ', header=None).values[:, 0:500][0:100, :] y_train = pd.read_csv(Config.get('data_path') + '/madelon/madelon_train.labels', delimiter=' ', header=None).values[0:100] name = 'hyperopt' # generate grid complexity_grid = np.arange(1, X_train.shape[1] + 1) max_acc = 0.8 accuracy_grid = np.arange(0.0, max_acc, max_acc / len(complexity_grid))
from sklearn.feature_selection import mutual_info_classif from sklearn.feature_selection import f_classif from sklearn.feature_selection import chi2 from sklearn.tree import DecisionTreeClassifier from fastsklearnfeature.configuration.Config import Config from skrebate import ReliefF from fastsklearnfeature.interactiveAutoML.feature_selection.fcbf_package import my_fisher_score sensitive_attribute = "sex" n_estimators = 5 df = pd.read_csv(Config.get('data_path') + '/adult/dataset_183_adult.csv', delimiter=',', header=0) y = df['class'] del df['class'] X = df one_hot = True limit = 1000 X_train, X_test, y_train, y_test = train_test_split(X.values[0:limit, :], y.values[0:limit], test_size=0.5, random_state=42) xshape = X_train.shape[1] encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
self.splitted_values = {} self.splitted_target= {} self.splitted_target['train'], self.splitted_target['valid'], self.splitted_target['test'] = self.splitter.materialize_target(self.target_values) self.splitted_values['train'], self.splitted_values['valid'],self.splitted_values['test'] = self.splitter.materialize_values(self.dataframe) for attribute_i in range(self.dataframe.shape[1]): rf = RawFeature(self.dataframe.columns[attribute_i], attribute_i, {}) rf.derive_properties(self.dataframe[self.dataframe.columns[attribute_i]].values) self.raw_features.append(rf) return self.raw_features if __name__ == '__main__': from fastsklearnfeature.splitting.RandomSplitter import RandomSplitter from fastsklearnfeature.configuration.Config import Config s = RandomSplitter() dataset = (Config.get('data_path') + '/house_price.csv', 79) r = Reader(dataset[0], dataset[1], s) r.read() for rf in r.raw_features: print(str(rf) + ": " + str(rf.properties))