def evaluate(candidate: CandidateFeature, classifier, grid_search_parameters, preprocessed_folds, score, train_data, current_target, train_X_all, train_y_all_target, test_data, test_target): pipeline = Pipeline([('features', FeatureUnion( [ (candidate.get_name(), candidate.pipeline) ])), ('classifier', classifier()) ]) refit = False if Config.get_default('score.test', 'False') == 'True' and not Config.get_default('instance.selection', 'False') == 'True': refit = True clf = GridSearchCV(pipeline, grid_search_parameters, cv=preprocessed_folds, scoring=score, iid=False, error_score='raise', refit=refit) clf.fit(train_data, current_target) #dataset.splitted_values['train'] candidate.runtime_properties['score'] = clf.best_score_ candidate.runtime_properties['hyperparameters'] = clf.best_params_ if Config.get_default('score.test', 'False') == 'True': if Config.get_default('instance.selection', 'False') == 'True': clf = GridSearchCV(pipeline, grid_search_parameters, cv=preprocessed_folds, scoring=score, iid=False, error_score='raise', refit=True) clf.fit(train_X_all, train_y_all_target) candidate.runtime_properties['test_score'] = clf.score(test_data, test_target) #self.dataset.splitted_values['test'] else: candidate.runtime_properties['test_score'] = 0.0 return candidate
def materialize_raw_features(self, candidate): train_transformed = [None] * len(self.preprocessed_folds) test_transformed = [None] * len(self.preprocessed_folds) # test training_all = None one_test_set_transformed = None candidate.fit(self.dataset.splitted_values['train']) raw_feature = candidate.transform(self.dataset.splitted_values['train']) for fold in range(len(self.preprocessed_folds)): train_transformed[fold] = raw_feature[self.preprocessed_folds[fold][0]] test_transformed[fold] = raw_feature[self.preprocessed_folds[fold][1]] if Config.get_default('score.test', 'False') == 'True': training_all = raw_feature if Config.get_default('instance.selection', 'False') == 'True': candidate.fit(self.train_X_all) training_all = candidate.transform(self.train_X_all) one_test_set_transformed = candidate.transform(self.dataset.splitted_values['test']) candidate.runtime_properties['training_all'] = training_all candidate.runtime_properties['one_test_set_transformed'] = one_test_set_transformed candidate.runtime_properties['train_transformed'] = train_transformed candidate.runtime_properties['test_transformed'] = test_transformed
def __init__( self, dataset_config, classifier=XGBClassifier, grid_search_parameters={ 'classifier__min_child_weight': [1, 5, 10], 'classifier__gamma': [0.5, 1, 1.5, 2, 5], 'classifier__subsample': [0.6, 0.8, 1.0], 'classifier__colsample_bytree': [0.1, 0.2, 0.4, 0.6, 0.8, 1.0], 'classifier__max_depth': [3, 4, 5], 'classifier__learning_rate': [0.02], 'classifier__n_estimators': [600], 'classifier__objective': ['binary:logistic'], 'classifier__silent': [True], 'classifier__n_jobs': [int(Config.get_default("parallelism", mp.cpu_count()))] }, transformation_producer=get_transformation_for_feature_space, score=make_scorer(f1_score, average='micro'), reader=None, folds=10, max_complexity=3): self.dataset_config = dataset_config self.classifier = classifier self.grid_search_parameters = grid_search_parameters self.transformation_producer = transformation_producer self.score = score self.reader = reader self.folds = folds self.max_complexity = max_complexity
def create_pipeline(self): #parent_features = FeatureUnion([(p.get_name(), p.pipeline) for p in self.parents], n_jobs=Config.get('feature.union.parallelism')) if len(self.parents) > 1: parent_features = FeatureUnion([(p.get_name() + str(time.time()), copy.deepcopy(p.pipeline)) for p in self.parents]) else: parent_features = copy.deepcopy(self.parents[0].pipeline) memory = None if Config.get_default('pipeline.caching', 'False') == 'True': memory = "/dev/shm" pipeline = None if isinstance(parent_features, Pipeline): parent_features.steps.append( (self.transformation.name + str(time.time()), self.transformation)) pipeline = parent_features else: pipeline = Pipeline( [('parents', parent_features), (self.transformation.name, self.transformation)], memory=memory) return pipeline
def generate(self, seed=42): if type(self.reader) == type(None): s = None if isinstance(self.classifier(), ClassifierMixin): s = Splitter(train_fraction=[0.6, 10000000], valid_fraction=0.0, test_fraction=0.4, seed=seed) elif isinstance(self.classifier(), RegressorMixin): s = RandomSplitter(train_fraction=[0.6, 10000000], valid_fraction=0.0, test_fraction=0.4, seed=seed) else: pass self.dataset = Reader(self.dataset_config[0], self.dataset_config[1], s) else: self.dataset = self.reader self.raw_features = self.dataset.read() print("training:" + str(len(self.dataset.splitted_target['train']))) print("test:" + str(len(self.dataset.splitted_target['test']))) if Config.get_default('instance.selection', 'False') == 'True': self.train_X_all = copy.deepcopy(self.dataset.splitted_values['train']) self.train_y_all = copy.deepcopy(self.dataset.splitted_target['train']) self.dataset.splitted_values['train'], self.dataset.splitted_target['train'] = sample_data_by_cnn(self.dataset.splitted_values['train'], self.dataset.splitted_target['train']) print("training:" + str(len(self.dataset.splitted_target['train']))) else: self.train_X_all = self.dataset.splitted_values['train'] self.train_y_all = self.dataset.splitted_target['train']
def generate_target(self): current_target = self.dataset.splitted_target['train'] if isinstance(self.classifier(), ClassifierMixin): label_encoder = LabelEncoder() label_encoder.fit(current_target) current_target = label_encoder.transform(current_target) self.test_target = None self.train_y_all_target = None if Config.get_default('score.test', 'False') == 'True': self.test_target = label_encoder.transform( self.dataset.splitted_target['test']) self.train_y_all_target = label_encoder.transform( self.train_y_all) self.preprocessed_folds = [] for train, test in StratifiedKFold( n_splits=self.folds, random_state=42).split( self.dataset.splitted_values['train'], current_target): self.preprocessed_folds.append((train, test)) elif isinstance(self.classifier(), RegressorMixin): if Config.get_default('score.test', 'False') == 'True': self.test_target = self.dataset.splitted_target['test'] self.train_y_all_target = self.train_y_all self.preprocessed_folds = [] for train, test in KFold(n_splits=self.folds, random_state=42).split( self.dataset.splitted_values['train'], current_target): self.preprocessed_folds.append((train, test)) else: pass self.target_train_folds = [None] * self.folds self.target_test_folds = [None] * self.folds for fold in range(len(self.preprocessed_folds)): self.target_train_folds[fold] = current_target[ self.preprocessed_folds[fold][0]] self.target_test_folds[fold] = current_target[ self.preprocessed_folds[fold][1]]
def evaluate_candidates( candidates: List[CandidateFeature]) -> List[CandidateFeature]: my_globale_module.candidate_list_global = candidates with mp.Pool(processes=int( Config.get_default("parallelism", mp.cpu_count()))) as pool: my_function = evaluate_catch candidates_ids = list(range(len(candidates))) if Config.get_default("show_progess", 'True') == 'True': results = [] for x in tqdm.tqdm(pool.imap_unordered(my_function, candidates_ids), total=len(candidates_ids)): results.append(x) else: results = pool.map(my_function, candidates_ids) return results
def create_pipeline(self): memory = None if Config.get_default('pipeline.caching', 'False') == 'True': memory = "/dev/shm" pipeline = Pipeline( [(self.name, ColumnTransformer( [('identity', FunctionTransformer( identity, validate=False), [self.column_id])]))], memory=memory) return pipeline
def nested_cv_score_parallel( candidates: List[CandidateFeature], splitted_values_train, splitted_target_train, n_jobs: int = int(Config.get_default("parallelism", mp.cpu_count())) ) -> List[CandidateFeature]: nested_my_globale_module.candidate_list_global = candidates nested_my_globale_module.splitted_values_train = splitted_values_train nested_my_globale_module.splitted_target_train = splitted_target_train with mp.Pool(processes=n_jobs) as pool: my_function = run_nested_cross_validation_global candidates_ids = list(range(len(candidates))) if Config.get_default("show_progess", 'True') == 'True': results = [] for x in tqdm.tqdm(pool.imap_unordered(my_function, candidates_ids), total=len(candidates_ids)): results.append(x) else: results = pool.map(my_function, candidates_ids) return results
def generate_target(self): current_target = self.dataset.splitted_target['train'] label_encoder = LabelEncoder() label_encoder.fit(current_target) self.current_target = label_encoder.transform(current_target) if Config.get_default('score.test', 'False') == 'True': self.test_target = label_encoder.transform(self.dataset.splitted_target['test']) self.train_y_all_target = label_encoder.transform(self.train_y_all) self.preprocessed_folds = [] for train, test in StratifiedKFold(n_splits=self.folds, random_state=42).split(self.dataset.splitted_values['train'], self.current_target): self.preprocessed_folds.append((train, test))
def grid_search(train_transformed, test_transformed, training_all, one_test_set_transformed, grid_search_parameters, score, classifier, target_train_folds, target_test_folds, train_y_all_target, test_target): hyperparam_to_score_list = {} my_keys = list(grid_search_parameters.keys()) for parameter_combination in itertools.product( *[grid_search_parameters[k] for k in my_keys]): parameter_set = hashabledict(zip(my_keys, parameter_combination)) hyperparam_to_score_list[parameter_set] = [] for fold in range(len(train_transformed)): clf = classifier(**parameter_set) clf.fit(train_transformed[fold], target_train_folds[fold]) y_pred = clf.predict(test_transformed[fold]) hyperparam_to_score_list[parameter_set].append( score._sign * score._score_func(target_test_folds[fold], y_pred, **score._kwargs)) best_param = None best_mean_cross_val_score = -float("inf") best_score_list = [] for parameter_config, score_list in hyperparam_to_score_list.items(): #mean_score = np.min(score_list) mean_score = np.mean(score_list) if mean_score > best_mean_cross_val_score: best_param = parameter_config best_mean_cross_val_score = mean_score best_score_list = copy.deepcopy(score_list) test_score = None if Config.get_default('score.test', 'False') == 'True': # refit to entire training and test on test set clf = classifier(**best_param) clf.fit(training_all, train_y_all_target) y_pred = clf.predict(one_test_set_transformed) test_score = score._sign * score._score_func(test_target, y_pred, ** score._kwargs) #np.save('/tmp/true_predictions', self.test_target) return best_mean_cross_val_score, test_score, best_param, y_pred, best_score_list, clf
def grid(grid_search_parameters, preprocessed_folds, train_data, current_target, test_data, test_target, pipeline, classifier, score): hyperparam_to_score_list = {} my_keys = list(grid_search_parameters.keys()) for parameter_combination in itertools.product( *[grid_search_parameters[k] for k in my_keys]): parameter_set = hashabledict(zip(my_keys, parameter_combination)) hyperparam_to_score_list[parameter_set] = [] for train_id, test_id in preprocessed_folds: clf = classifier(**parameter_set) my_train = pipeline.fit_transform(train_data[train_id], current_target[train_id]) clf.fit(my_train, current_target[train_id]) y_pred = clf.predict(pipeline.transform(train_data[test_id])) hyperparam_to_score_list[parameter_set].append( score._sign * score._score_func(current_target[test_id], y_pred, **score._kwargs)) best_param = None best_mean_cross_val_score = -float("inf") for parameter_config, score_list in hyperparam_to_score_list.items(): mean_score = np.mean(score_list) if mean_score > best_mean_cross_val_score: best_param = parameter_config best_mean_cross_val_score = mean_score test_score = None if Config.get_default('score.test', 'False') == 'True': # refit to entire training and test on test set clf = classifier(**best_param) my_train = pipeline.fit_transform(train_data, current_target) clf.fit(my_train, current_target) y_pred = clf.predict(pipeline.transform(test_data)) test_score = score._sign * score._score_func(test_target, y_pred, ** score._kwargs) # np.save('/tmp/true_predictions', self.test_target) return best_mean_cross_val_score, test_score, best_param, y_pred
def __init__(self, dataset_config, classifier=LogisticRegression, grid_search_parameters={'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000], 'solver': ['lbfgs'], 'class_weight': ['balanced'], 'max_iter': [10000], 'multi_class':['auto'] }, transformation_producer=get_transformation_for_division, epsilon=0.0, c_max=2, folds=10, score=make_scorer(f1_score, average='micro'), max_seconds=None, save_logs=False, reader=None, upload2openml=False, remove_parents=True, n_jobs=None, max_feature_depth=np.inf ): super(ComplexityDrivenFeatureConstruction, self).__init__(dataset_config, classifier, grid_search_parameters, transformation_producer) self.epsilon = epsilon self.c_max = c_max self.folds = folds self.score = score self.save_logs = save_logs self.reader = reader self.upload2openml = upload2openml self.remove_parents = remove_parents self.max_timestamp = None if type(max_seconds) != type(None): self.max_timestamp = time.time() + max_seconds if type(n_jobs) == type(None): self.n_jobs = int(Config.get_default("parallelism", mp.cpu_count())) else: self.n_jobs = n_jobs self.max_feature_depth = max_feature_depth
def evaluate(candidate_id: int): #process = psutil.Process(os.getpid()) #print(str(process.memory_info().rss) + " " + str(sys.getsizeof(my_globale_module.candidate_list_global))) candidate: CandidateFeature = my_globale_module.candidate_list_global[ candidate_id] if type(my_globale_module.max_timestamp_global) != type( None) and time.time() >= my_globale_module.max_timestamp_global: raise RuntimeError('Out of time!') train_transformed = [None] * len( my_globale_module.preprocessed_folds_global) test_transformed = [None] * len( my_globale_module.preprocessed_folds_global) #test training_all = None one_test_set_transformed = None if isinstance(candidate, RawFeature): if 'training_all' in candidate.runtime_properties: training_all = candidate.runtime_properties['training_all'] one_test_set_transformed = candidate.runtime_properties[ 'one_test_set_transformed'] train_transformed = candidate.runtime_properties['train_transformed'] test_transformed = candidate.runtime_properties['test_transformed'] else: #print(self.name_to_train_transformed.keys()) #merge columns from parents test_unique_values = 0 for fold in range(len(my_globale_module.preprocessed_folds_global)): train_transformed_input = np.hstack([ p.runtime_properties['train_transformed'][fold] for p in candidate.parents ]) test_transformed_input = np.hstack([ p.runtime_properties['test_transformed'][fold] for p in candidate.parents ]) candidate.transformation.fit( train_transformed_input, my_globale_module.target_train_folds_global[fold]) train_transformed[fold] = candidate.transformation.transform( train_transformed_input) test_transformed[fold] = candidate.transformation.transform( test_transformed_input) test_unique_values += len(np.unique(test_transformed[fold])) if not isinstance(candidate.transformation, IdentityTransformation): #check whether feature is constant if test_unique_values == len( my_globale_module.preprocessed_folds_global): return None ## check if we computed an equivalent feature before materialized_all = [] for fold_ii in range( len(my_globale_module.preprocessed_folds_global)): materialized_all.extend(test_transformed[fold_ii].flatten()) materialized = tuple(materialized_all) if materialized in my_globale_module.materialized_set: return None if 'training_all' in list(candidate.parents)[0].runtime_properties: training_all_input = np.hstack([ p.runtime_properties['training_all'] for p in candidate.parents ]) one_test_set_transformed_input = np.hstack([ p.runtime_properties['one_test_set_transformed'] for p in candidate.parents ]) candidate.transformation.fit( training_all_input, my_globale_module.train_y_all_target_global) training_all = candidate.transformation.transform( training_all_input) one_test_set_transformed = candidate.transformation.transform( one_test_set_transformed_input) evaluated = True if ( ( isinstance(candidate.transformation, MinusTransformation) or (isinstance(candidate.transformation, HigherOrderCommutativeTransformation) and candidate.transformation.method == np.nansum) or (isinstance(candidate.transformation, NonCommutativeBinaryTransformation) and candidate.transformation.method == np.subtract) ) \ and \ ( my_globale_module.classifier_global == LogisticRegression or my_globale_module.classifier_global == LinearRegression ) ): candidate.runtime_properties['score'] = np.max( [p.runtime_properties['score'] for p in candidate.parents]) candidate.runtime_properties['test_score'] = -1.0 candidate.runtime_properties['hyperparameters'] = None y_pred = None evaluated = False candidate.runtime_properties['passed'] = True else: candidate.runtime_properties['passed'] = False candidate.runtime_properties['score'], candidate.runtime_properties[ 'test_score'], candidate.runtime_properties[ 'hyperparameters'], test_fold_predictions, candidate.runtime_properties[ 'fold_scores'], my_clf, candidate.runtime_properties[ 'mean_scores'], candidate.runtime_properties[ 'additional_metrics'] = grid_search( candidate, train_transformed, test_transformed, training_all, one_test_set_transformed, my_globale_module. grid_search_parameters_global, my_globale_module.score_global, my_globale_module.classifier_global, my_globale_module.target_train_folds_global, my_globale_module.target_test_folds_global, my_globale_module.train_y_all_target_global, my_globale_module.test_target_global) ''' candidate.runtime_properties['score'], candidate.runtime_properties['test_score'], candidate.runtime_properties[ 'hyperparameters'], test_fold_predictions, candidate.runtime_properties['fold_scores'], my_clf, \ candidate.runtime_properties['mean_scores'], candidate.runtime_properties['additional_metrics'] = hyperopt_search( train_transformed, test_transformed, training_all, one_test_set_transformed, my_globale_module.grid_search_parameters_global, my_globale_module.score_global, my_globale_module.classifier_global, my_globale_module.target_train_folds_global, my_globale_module.target_test_folds_global, my_globale_module.train_y_all_target_global, my_globale_module.test_target_global) ''' #if True: # candidate.runtime_properties['coef_'] = my_clf.coef_ if Config.get_default('store.predictions', 'False') == 'True': candidate.runtime_properties[ 'test_fold_predictions'] = test_fold_predictions ''' ## check whether prediction already exists materialized_all = [] for fold_ii in range(len(my_globale_module.preprocessed_folds_global)): materialized_all.extend(candidate.runtime_properties['test_fold_predictions'][fold_ii].flatten()) materialized = tuple(materialized_all) if materialized in my_globale_module.predictions_set: return None ''' if isinstance( candidate.transformation, OneHotTransformation ) or isinstance(candidate, RawFeature) or not evaluated or ( (candidate.runtime_properties['score'] - np.max([p.runtime_properties['score'] for p in candidate.parents])) / my_globale_module.complexity_delta_global ) * my_globale_module.score_global._sign > my_globale_module.epsilon_global: candidate.runtime_properties['passed'] = True if not isinstance(candidate, RawFeature): candidate.runtime_properties[ 'train_transformed'] = train_transformed candidate.runtime_properties['test_transformed'] = test_transformed if Config.get_default( 'score.test', 'False') == 'True' and type(training_all) != type(None): candidate.runtime_properties['training_all'] = training_all candidate.runtime_properties[ 'one_test_set_transformed'] = one_test_set_transformed # derive properties if not isinstance(candidate, RawFeature): candidate.derive_properties( candidate.runtime_properties['train_transformed'][0]) #avoid nan for specific ml models if candidate.properties[ 'missing_values'] and my_globale_module.classifier_global == LogisticRegression: return None # remove parents' materialization candidate.get_name() candidate.get_complexity() candidate.get_sympy_representation() if my_globale_module.remove_parents: candidate.parents = None return candidate return None
def run(self): self.global_starting_time = time.time() # generate all candidates self.generate() #starting_feature_matrix = self.create_starting_features() self.generate_target() unary_transformations, binary_transformations = self.transformation_producer(self.train_X_all, self.raw_features) cost_2_raw_features: Dict[int, List[CandidateFeature]] = {} cost_2_unary_transformed: Dict[int, List[CandidateFeature]] = {} cost_2_binary_transformed: Dict[int, List[CandidateFeature]] = {} cost_2_combination: Dict[int, List[CandidateFeature]] = {} if self.save_logs: cost_2_dropped_evaluated_candidates: Dict[int, List[CandidateFeature]] = {} self.complexity_delta = 1.0 unique_raw_combinations = False baseline_score = 0.0#self.evaluate_candidates([CandidateFeature(DummyOneTransformation(None), [self.raw_features[0]])])[0]['score'] #print("baseline: " + str(baseline_score)) max_feature = CandidateFeature(IdentityTransformation(None), [self.raw_features[0]]) max_feature.runtime_properties['score'] = -float("inf") max_feature_per_complexity: Dict[int, CandidateFeature] = {} all_evaluated_features = set() my_globale_module.global_starting_time_global = copy.deepcopy(self.global_starting_time) my_globale_module.grid_search_parameters_global = copy.deepcopy(self.grid_search_parameters) my_globale_module.score_global = copy.deepcopy(self.score) my_globale_module.classifier_global = copy.deepcopy(self.classifier) my_globale_module.target_train_folds_global = copy.deepcopy(self.target_train_folds) my_globale_module.target_test_folds_global = copy.deepcopy(self.target_test_folds) my_globale_module.train_y_all_target_global = copy.deepcopy(self.train_y_all_target) my_globale_module.test_target_global = copy.deepcopy(self.test_target) my_globale_module.max_timestamp_global = copy.deepcopy(self.max_timestamp) my_globale_module.preprocessed_folds_global = copy.deepcopy(self.preprocessed_folds) my_globale_module.epsilon_global = copy.deepcopy(self.epsilon) my_globale_module.complexity_delta_global = copy.deepcopy(self.complexity_delta) my_globale_module.remove_parents = copy.deepcopy(self.remove_parents) my_globale_module.materialized_set = set() my_globale_module.predictions_set = set() number_of_multiple_cvs = 10 nested_my_globale_module.splitting_seeds = np.random.randint(low=0, high=10000, size=number_of_multiple_cvs) nested_my_globale_module.model_seeds = np.random.randint(low=0, high=10000, size=number_of_multiple_cvs) #pickle.dump(my_globale_module.target_test_folds_global, open('/tmp/test_groundtruth.p', 'wb+')) c = 1 while(True): current_layer: List[CandidateFeature] = [] if c <= self.max_feature_depth: #0th if c == 1: cost_2_raw_features[c]: List[CandidateFeature] = [] #print(self.raw_features) for raw_f in self.raw_features: sympy_representation = sympy.Symbol('X' + str(raw_f.column_id)) raw_f.sympy_representation = sympy_representation all_evaluated_features.add(sympy_representation) if raw_f.is_numeric(): if raw_f.properties['missing_values']: raw_f.runtime_properties['score'] = 0.0 cost_2_raw_features[c].append(raw_f) else: current_layer.append(raw_f) #print("numeric: " + str(raw_f)) else: raw_f.runtime_properties['score'] = 0.0 cost_2_raw_features[c].append(raw_f) #print("nonnumeric: " + str(raw_f)) self.materialize_raw_features(raw_f) #raw_f.derive_properties(raw_f.runtime_properties['train_transformed'][0]) # first unary # we apply all unary transformation to all c-1 in the repo (except combinations and other unary?) unary_candidates_to_be_applied: List[CandidateFeature] = [] if (c - 1) in cost_2_raw_features: unary_candidates_to_be_applied.extend(cost_2_raw_features[c - 1]) if (c - 1) in cost_2_unary_transformed: unary_candidates_to_be_applied.extend(cost_2_unary_transformed[c - 1]) if (c - 1) in cost_2_binary_transformed: unary_candidates_to_be_applied.extend(cost_2_binary_transformed[c - 1]) all_unary_features = self.generate_features(unary_transformations, unary_candidates_to_be_applied, all_evaluated_features) current_layer.extend(all_unary_features) #second binary #get length 2 partitions for current cost partition = self.get_length_2_partition(c-1) #print("bin: c: " + str(c) + " partition" + str(partition)) #apply cross product from partitions binary_candidates_to_be_applied: List[CandidateFeature] = [] for p in partition: lists_for_each_element: List[List[CandidateFeature]] = [[], []] for element in range(2): if p[element] in cost_2_raw_features: lists_for_each_element[element].extend(cost_2_raw_features[p[element]]) if p[element] in cost_2_unary_transformed: lists_for_each_element[element].extend(cost_2_unary_transformed[p[element]]) if p[element] in cost_2_binary_transformed: lists_for_each_element[element].extend(cost_2_binary_transformed[p[element]]) for bt in binary_transformations: list_of_combinations = self.generate_merge(lists_for_each_element[0], lists_for_each_element[1], bt.parent_feature_order_matters, bt.parent_feature_repetition_is_allowed) #print(list_of_combinations) for combo in list_of_combinations: if bt.is_applicable(combo): sympy_representation = bt.get_sympy_representation( [p.get_sympy_representation() for p in combo]) try: if len(sympy_representation.free_symbols) > 0: # if expression is not constant if not sympy_representation in all_evaluated_features: bin_candidate = CandidateFeature(copy.deepcopy(bt), combo) bin_candidate.sympy_representation = copy.deepcopy(sympy_representation) all_evaluated_features.add(sympy_representation) binary_candidates_to_be_applied.append(bin_candidate) else: #print(str(bin_candidate) + " skipped: " + str(sympy_representation)) pass else: #print(str(bin_candidate) + " skipped: " + str(sympy_representation)) pass except: pass current_layer.extend(binary_candidates_to_be_applied) #third: feature combinations #first variant: treat combination as a transformation #therefore, we can use the same partition as for binary data partition = self.get_length_2_partition(c) #print("combo c: " + str(c) + " partition" + str(partition)) def filter_minus(features: List[CandidateFeature]): filtered_features: List[CandidateFeature] = [] if my_globale_module.classifier_global == LogisticRegression: for check_f in features: if not isinstance(check_f.transformation, MinusTransformation): filtered_features.append(check_f) return filtered_features ''' combinations_to_be_applied: List[CandidateFeature] = [] for p in partition: lists_for_each_element: List[List[CandidateFeature]] = [[], []] for element in range(2): if p[element] in cost_2_raw_features: lists_for_each_element[element].extend(cost_2_raw_features[p[element]]) if p[element] in cost_2_unary_transformed: lists_for_each_element[element].extend(filter_minus(cost_2_unary_transformed[p[element]])) if p[element] in cost_2_binary_transformed: lists_for_each_element[element].extend(filter_minus(cost_2_binary_transformed[p[element]])) if p[element] in cost_2_combination: lists_for_each_element[element].extend(cost_2_combination[p[element]]) combinations_to_be_applied = self.generate_merge_for_combination(all_evaluated_features, lists_for_each_element[0], lists_for_each_element[1]) current_layer.extend(combinations_to_be_applied) ''' if unique_raw_combinations: length = len(current_layer) current_layer = self.filter_non_unique_combinations(current_layer) print("From " + str(length) + " combinations, we filter " + str(length - len(current_layer)) + " nonunique raw feature combinations.") #now evaluate all from this layer #print(current_layer) print("----------- Evaluation of " + str(len(current_layer)) + " representations -----------") results = evaluate_candidates_parallel(current_layer, self.n_jobs) print("----------- Evaluation Finished -----------") ##nested cv ''' new_results_with_nested = [] for r_result in results: if type(r_result) != type(None): new_results_with_nested.append(r_result) #results = nested_cv_score_parallel(new_results_with_nested, self.reader.splitted_values['train'], self.reader.splitted_target['train']) results = multiple_cv_score_parallel(new_results_with_nested, self.reader.splitted_values['train'], self.reader.splitted_target['train']) for r_result in results: #print(str(r_result) + ' cv: ' + str(r_result.runtime_properties['score']) + ' test: ' + str(r_result.runtime_properties['test_score']) + ' nested: ' + str(r_result.runtime_properties['nested_cv_score'])) print(str(r_result) + ' cv: ' + str(r_result.runtime_properties['score']) + ' test: ' + str( r_result.runtime_properties['test_score']) + ' nested: ' + str( r_result.runtime_properties['multiple_cv_score'])) ''' #print(results) layer_end_time = time.time() - self.global_starting_time #calculate whether we drop the evaluated candidate for candidate in results: ## check if we computed an equivalent feature before if type(candidate) != type(None) and not isinstance(candidate.transformation, IdentityTransformation): materialized_all = [] for fold_ii in range(len(my_globale_module.preprocessed_folds_global)): materialized_all.extend(candidate.runtime_properties['test_transformed'][fold_ii].flatten()) materialized = tuple(materialized_all) if materialized in my_globale_module.materialized_set: candidate = None else: my_globale_module.materialized_set.add(materialized) ''' ## check if predictions exist already if type(candidate) != type(None) and 'test_fold_predictions' in candidate.runtime_properties: materialized_all = [] for fold_ii in range(len(my_globale_module.preprocessed_folds_global)): materialized_all.extend(candidate.runtime_properties['test_fold_predictions'][fold_ii].flatten()) materialized = tuple(materialized_all) if materialized in my_globale_module.predictions_set: candidate = None else: my_globale_module.predictions_set.add(materialized) ''' if type(candidate) != type(None): candidate.runtime_properties['layer_end_time'] = layer_end_time #print(str(candidate) + " -> " + str(candidate.runtime_properties['score'])) if candidate.runtime_properties['score'] > max_feature.runtime_properties['score']: max_feature = candidate if candidate.runtime_properties['passed']: if isinstance(candidate, RawFeature): if not c in cost_2_raw_features: cost_2_raw_features[c]: List[CandidateFeature] = [] cost_2_raw_features[c].append(candidate) elif isinstance(candidate.transformation, UnaryTransformation): if not c in cost_2_unary_transformed: cost_2_unary_transformed[c]: List[CandidateFeature] = [] cost_2_unary_transformed[c].append(candidate) elif isinstance(candidate.transformation, IdentityTransformation): if not c in cost_2_combination: cost_2_combination[c]: List[CandidateFeature] = [] cost_2_combination[c].append(candidate) else: if not c in cost_2_binary_transformed: cost_2_binary_transformed[c]: List[CandidateFeature] = [] cost_2_binary_transformed[c].append(candidate) else: if self.save_logs: if not c in cost_2_dropped_evaluated_candidates: cost_2_dropped_evaluated_candidates[c]: List[CandidateFeature] = [] cost_2_dropped_evaluated_candidates[c].append(candidate) satisfied_count = 0 if c in cost_2_raw_features: satisfied_count += len(cost_2_raw_features[c]) if c in cost_2_unary_transformed: satisfied_count += len(cost_2_unary_transformed[c]) if c in cost_2_binary_transformed: satisfied_count += len(cost_2_binary_transformed[c]) if c in cost_2_combination: satisfied_count += len(cost_2_combination[c]) all_count = len(current_layer) if c == 1: all_count = len(cost_2_raw_features[c]) print("Of " + str(all_count) + " candidate representations, " + str(satisfied_count) + " did satisfy the epsilon threshold.") if len(current_layer) > 0: if 'test_score' in max_feature.runtime_properties: print("\nBest representation found for complexity = " + str(c) + ": " + str(max_feature) + "\nmean cross-validation score: " + "{0:.2f}".format(max_feature.runtime_properties['score']) + ", score on test: " + "{0:.2f}".format(max_feature.runtime_properties['test_score']) + "\n") else: print("\nBest representation found for complexity = " + str(c) + ": " + str( max_feature) + "\nmean cross-validation score: " + "{0:.2f}".format( max_feature.runtime_properties['score']) + "\n") #print("hyper: " + str(max_feature.runtime_properties['hyperparameters'])) #print(max_feature.runtime_properties['fold_scores']) # upload best feature to OpenML if self.upload2openml: candidate2openml(max_feature, my_globale_module.classifier_global, self.reader.task, 'ComplexityDriven') if self.save_logs: try: pickle.dump(cost_2_raw_features, open(Config.get_default("tmp.folder", "/tmp") + "/data_raw" + str(self.reader.rotate_test) + ".p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_unary_transformed, open(Config.get_default("tmp.folder", "/tmp") + "/data_unary" + str(self.reader.rotate_test) + ".p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_binary_transformed, open(Config.get_default("tmp.folder", "/tmp") + "/data_binary" + str(self.reader.rotate_test) + ".p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_combination, open(Config.get_default("tmp.folder", "/tmp") + "/data_combination" + str(self.reader.rotate_test) + ".p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_dropped_evaluated_candidates, open(Config.get_default("tmp.folder", "/tmp") + "/data_dropped" + str(self.reader.rotate_test) + ".p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) except: pickle.dump(cost_2_raw_features, open( Config.get_default("tmp.folder", "/tmp") + "/data_raw.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_unary_transformed, open( Config.get_default("tmp.folder", "/tmp") + "/data_unary.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_binary_transformed, open( Config.get_default("tmp.folder", "/tmp") + "/data_binary.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_combination, open( Config.get_default("tmp.folder", "/tmp") + "/data_combination.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_dropped_evaluated_candidates, open( Config.get_default("tmp.folder", "/tmp") + "/data_dropped.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) max_feature_per_complexity[c] = max_feature if type(self.c_max) == type(None) and c > 2: # calculate harmonic mean harmonic_means = [0.0]*3 for h_i in range(len(harmonic_means)): simplicity_cum_score = self.getSimplicityScore(max_feature_per_complexity[c-h_i].get_complexity(), c, cost_2_raw_features, cost_2_unary_transformed, cost_2_binary_transformed, cost_2_combination) accuracy_cum_score = self.getAccuracyScore(max_feature_per_complexity[c-h_i].runtime_properties['score'], c, cost_2_raw_features, cost_2_unary_transformed, cost_2_binary_transformed, cost_2_combination) harmonic_means[h_i] = self.harmonic_mean(simplicity_cum_score, accuracy_cum_score) #print(str(max_feature_per_complexity[c-h_i]) + ": " + str(harmonic_means[h_i]) + " h: " + str(h_i)) if harmonic_means[2] >= harmonic_means[1] and harmonic_means[2] >= harmonic_means[0]: print("Best Harmonic Mean: " + str(max_feature_per_complexity[c-2])) break if type(self.max_timestamp) != type(None) and time.time() >= self.max_timestamp: break c += 1 if type(self.c_max) != type(None) and self.c_max < c: break def extend_all(all_representations: List[CandidateFeature], new_llist): for mylist in new_llist: all_representations.extend(mylist) #get all representation all_representations: List[CandidateFeature] = [] extend_all(all_representations, cost_2_raw_features.values()) extend_all(all_representations, cost_2_unary_transformed.values()) extend_all(all_representations, cost_2_binary_transformed.values()) extend_all(all_representations, cost_2_combination.values()) self.all_representations = all_representations ''' #find top k based on cv score scores = [c.runtime_properties['score'] for c in all_representations] sorted_cv_score_ids = np.argsort(np.array(scores)*-1) checking_k = 50 top_k_representations = [all_representations[sorted_id] for sorted_id in sorted_cv_score_ids[0:checking_k]] #from top k - select best based on nested cv score top_k_representations = multiple_cv_score_parallel(top_k_representations, self.reader.splitted_values['train'], self.reader.splitted_target['train']) scores = [c.runtime_properties['multiple_cv_score'] for c in top_k_representations] max_nested_cv_score = -1 max_nested_rep = None for eval_candidate in top_k_representations: if eval_candidate.runtime_properties['multiple_cv_score'] > max_nested_cv_score: max_nested_cv_score = eval_candidate.runtime_properties['multiple_cv_score'] max_nested_rep = eval_candidate print(max_nested_rep) max_feature = max_nested_rep ''' ''' all_features = list(max_feature_per_complexity.values()) all_features = multiple_cv_score_parallel(all_features, self.reader.splitted_values['train'], self.reader.splitted_target['train']) best_multiple_cv_score = -np.inf best_multiple_cv_candidate = None for all_f in all_features: if all_f.runtime_properties['multiple_cv_score'] > best_multiple_cv_score: best_multiple_cv_score = all_f.runtime_properties['multiple_cv_score'] best_multiple_cv_candidate = all_f #find the most simple representation that is within the best representation's std complexities = [all_f.get_complexity() for all_f in all_features] ids_complex = np.argsort(complexities) for all_f_i in range(len(all_features)): print(str(all_features[ids_complex[all_f_i]]) + ' mcv: ' + str(all_features[ids_complex[all_f_i]].runtime_properties['multiple_cv_score']) + ' mcv_std: ' + str( all_features[ids_complex[all_f_i]].runtime_properties['multiple_cv_score_std'])) if all_features[ids_complex[all_f_i]].runtime_properties['multiple_cv_score'] > best_multiple_cv_candidate.runtime_properties['multiple_cv_score'] - best_multiple_cv_candidate.runtime_properties['multiple_cv_score_std']: max_feature = all_features[ids_complex[all_f_i]] break print(max_feature) ''' #min AICc selection min_aicc = np.inf min_aicc_feature = None all_aiccs = [] for rep in list(max_feature_per_complexity.values()): all_aiccs.append(np.mean(rep.runtime_properties['additional_metrics']['AICc_complexity'])) def calculate_AIC_for_classification_paper(rss, n, k): AIC = 2 * k + float(n) * np.log(rss / float(n)) return AIC def calculate_AICc_for_classification_paper(rss, n, k): AIC = calculate_AIC_for_classification_paper(rss, n, k) AICc = AIC + ((2 * k * (k + 1)) / (n - k - 1)) return AICc def calc_global_aicc(rep): return calculate_AICc_for_classification_paper(np.sum(rep.runtime_properties['additional_metrics']['rss']), np.sum(rep.runtime_properties['additional_metrics']['n']), rep.get_complexity()) def is_better(old_aics, new_aics): print(np.sum(np.array(new_aics) < np.array(old_aics))) return np.sum(np.array(new_aics) < np.array(old_aics)) > len(new_aics) / 2.0 for rep in list(max_feature_per_complexity.values()): curr = np.mean(rep.runtime_properties['additional_metrics']['AICc_complexity']) #print(str(rep) + ': ' + str(curr) + ' AICc min: ' + str(np.min(rep.runtime_properties['additional_metrics']['AICc_complexity'])) + ' AICc std: ' + str(np.std(rep.runtime_properties['additional_metrics']['AICc_complexity'])) + ' P: ' + str(np.exp((min(all_aiccs) - curr)/2)) + ' CV AUC: ' + str(rep.runtime_properties['score'])) print(str(rep) + ':' + str(rep.runtime_properties['additional_metrics']['AICc_complexity'])) print(str(rep) + ':' + str(rep.runtime_properties['additional_metrics']['rss'])) print(str(rep) + ':' + str(rep.runtime_properties['additional_metrics']['n'])) print(str(rep) + 'global_aicc: ' + str(calc_global_aicc(rep))) #if type(min_aicc_feature) == type(None) or is_better(min_aicc_feature.runtime_properties['additional_metrics']['AICc_complexity'], rep.runtime_properties['additional_metrics']['AICc_complexity']): if type(min_aicc_feature) == type(None) or calc_global_aicc(rep) < calc_global_aicc(min_aicc_feature): #min_aicc = np.min(rep.runtime_properties['additional_metrics']['AICc_complexity']) min_aicc_feature = rep max_feature = min_aicc_feature print(max_feature) return max_feature
def evaluate(candidate: CandidateFeature, classifier, grid_search_parameters, preprocessed_folds, score, train_data, current_target, train_X_all, train_y_all_target, test_data, test_target, cv_jobs=1): pipeline = Pipeline([('features', FeatureUnion([(candidate.get_name(), candidate.pipeline)])), ('classifier', classifier())]) refit = False if Config.get_default('score.test', 'False') == 'True' and not Config.get_default( 'instance.selection', 'False') == 'True': refit = True print(grid_search_parameters) clf = GridSearchCV(pipeline, grid_search_parameters, cv=preprocessed_folds, scoring=score, iid=False, error_score='raise', refit=refit, n_jobs=cv_jobs) clf.fit(train_data, current_target) #dataset.splitted_values['train'] candidate.runtime_properties['score'] = clf.best_score_ candidate.runtime_properties['hyperparameters'] = clf.best_params_ #for test_fold_predictions = [] for fold in range(len(preprocessed_folds)): test_fold_predictions.append( clf.predict(train_data[preprocessed_folds[fold][1]]) == current_target[preprocessed_folds[fold][1]]) candidate.runtime_properties[ 'test_fold_predictions'] = test_fold_predictions if Config.get_default('score.test', 'False') == 'True' and len(test_data) > 0: if Config.get_default('instance.selection', 'False') == 'True': clf = GridSearchCV(pipeline, grid_search_parameters, cv=preprocessed_folds, scoring=score, iid=False, error_score='raise', refit=True) clf.fit(train_X_all, train_y_all_target) candidate.runtime_properties['test_score'] = clf.score( test_data, test_target) #self.dataset.splitted_values['test'] else: candidate.runtime_properties['test_score'] = 0.0 return candidate
def run(self): self.global_starting_time = time.time() # generate all candidates self.generate() #starting_feature_matrix = self.create_starting_features() self.generate_target() unary_transformations, binary_transformations = self.transformation_producer() cost_2_raw_features: Dict[int, List[CandidateFeature]] = {} cost_2_unary_transformed: Dict[int, List[CandidateFeature]] = {} cost_2_binary_transformed: Dict[int, List[CandidateFeature]] = {} cost_2_combination: Dict[int, List[CandidateFeature]] = {} cost_2_dropped_evaluated_candidates: Dict[int, List[CandidateFeature]] = {} complexity_delta = 1.0 epsilon = self.epsilon limit_runs = self.c_max + 1 # 5 unique_raw_combinations = False baseline_score = 0.0#self.evaluate_candidates([CandidateFeature(DummyOneTransformation(None), [self.raw_features[0]])])[0]['score'] #print("baseline: " + str(baseline_score)) max_feature = CandidateFeature(IdentityTransformation(None), [self.raw_features[0]]) max_feature.runtime_properties['score'] = -2 self.name_to_transfomed = {} for c in range(1, limit_runs): current_layer: List[CandidateFeature] = [] #0th if c == 1: cost_2_raw_features[c]: List[CandidateFeature] = [] for raw_f in self.raw_features: if raw_f.is_numeric(): current_layer.append(raw_f) else: raw_f.runtime_properties['score'] = 0.0 cost_2_raw_features[c].append(raw_f) # first unary # we apply all unary transformation to all c-1 in the repo (except combinations and other unary?) unary_candidates_to_be_applied: List[CandidateFeature] = [] if (c - 1) in cost_2_raw_features: unary_candidates_to_be_applied.extend(cost_2_raw_features[c - 1]) if (c - 1) in cost_2_unary_transformed: unary_candidates_to_be_applied.extend(cost_2_unary_transformed[c - 1]) if (c - 1) in cost_2_binary_transformed: unary_candidates_to_be_applied.extend(cost_2_binary_transformed[c - 1]) current_layer.extend(self.generate_features(unary_transformations, unary_candidates_to_be_applied)) #second binary #get length 2 partitions for current cost partition = self.get_length_2_partition(c-1) #print("bin: c: " + str(c) + " partition" + str(partition)) #apply cross product from partitions binary_candidates_to_be_applied: List[CandidateFeature] = [] for p in partition: lists_for_each_element: List[List[CandidateFeature]] = [[], []] for element in range(2): if p[element] in cost_2_raw_features: lists_for_each_element[element].extend(cost_2_raw_features[p[element]]) if p[element] in cost_2_unary_transformed: lists_for_each_element[element].extend(cost_2_unary_transformed[p[element]]) if p[element] in cost_2_binary_transformed: lists_for_each_element[element].extend(cost_2_binary_transformed[p[element]]) for bt in binary_transformations: list_of_combinations = self.generate_merge(lists_for_each_element[0], lists_for_each_element[1], bt.parent_feature_order_matters, bt.parent_feature_repetition_is_allowed) for combo in list_of_combinations: if bt.is_applicable(combo): binary_candidates_to_be_applied.append(CandidateFeature(copy.deepcopy(bt), combo)) current_layer.extend(binary_candidates_to_be_applied) #third: feature combinations #first variant: treat combination as a transformation #therefore, we can use the same partition as for binary data partition = self.get_length_2_partition(c) #print("combo c: " + str(c) + " partition" + str(partition)) combinations_to_be_applied: List[CandidateFeature] = [] for p in partition: lists_for_each_element: List[List[CandidateFeature]] = [[], []] for element in range(2): if p[element] in cost_2_raw_features: lists_for_each_element[element].extend(cost_2_raw_features[p[element]]) if p[element] in cost_2_unary_transformed: lists_for_each_element[element].extend(cost_2_unary_transformed[p[element]]) if p[element] in cost_2_binary_transformed: lists_for_each_element[element].extend(cost_2_binary_transformed[p[element]]) if p[element] in cost_2_combination: lists_for_each_element[element].extend(cost_2_combination[p[element]]) list_of_combinations = self.generate_merge_for_combination(lists_for_each_element[0], lists_for_each_element[1]) for combo in list_of_combinations: if IdentityTransformation(None).is_applicable(list(combo)): combinations_to_be_applied.append(CandidateFeature(IdentityTransformation(None), list(combo))) current_layer.extend(combinations_to_be_applied) if unique_raw_combinations: length = len(current_layer) current_layer = self.filter_non_unique_combinations(current_layer) print("From " + str(length) + " combinations, we filter " + str(length - len(current_layer)) + " nonunique raw feature combinations.") #now evaluate all from this layer #print(current_layer) print("----------- Evaluation of " + str(len(current_layer)) + " representations -----------") results = self.evaluate_candidates(current_layer) print("----------- Evaluation Finished -----------") layer_end_time = time.time() - self.global_starting_time #calculate whether we drop the evaluated candidate for result in results: candidate: CandidateFeature = result['candidate'] candidate.runtime_properties['score'] = result['score'] candidate.runtime_properties['test_score'] = result['test_score'] candidate.runtime_properties['execution_time'] = result['execution_time'] candidate.runtime_properties['global_time'] = result['global_time'] candidate.runtime_properties['hyperparameters'] = result['hyperparameters'] candidate.runtime_properties['layer_end_time'] = layer_end_time #print(str(candidate) + " -> " + str(candidate.score)) if candidate.runtime_properties['score'] > max_feature.runtime_properties['score']: max_feature = candidate #calculate original score original_score = baseline_score #or zero?? if not isinstance(candidate, RawFeature): original_score = max([p.runtime_properties['score'] for p in candidate.parents]) accuracy_delta = result['score'] - original_score if accuracy_delta / complexity_delta > epsilon: if isinstance(candidate, RawFeature): if not c in cost_2_raw_features: cost_2_raw_features[c]: List[CandidateFeature] = [] cost_2_raw_features[c].append(candidate) elif isinstance(candidate.transformation, UnaryTransformation): if not c in cost_2_unary_transformed: cost_2_unary_transformed[c]: List[CandidateFeature] = [] cost_2_unary_transformed[c].append(candidate) elif isinstance(candidate.transformation, IdentityTransformation): if not c in cost_2_combination: cost_2_combination[c]: List[CandidateFeature] = [] cost_2_combination[c].append(candidate) else: if not c in cost_2_binary_transformed: cost_2_binary_transformed[c]: List[CandidateFeature] = [] cost_2_binary_transformed[c].append(candidate) else: if not c in cost_2_dropped_evaluated_candidates: cost_2_dropped_evaluated_candidates[c]: List[CandidateFeature] = [] cost_2_dropped_evaluated_candidates[c].append(candidate) if c in cost_2_dropped_evaluated_candidates: print("Of " + str(len(current_layer)) + " candidate representations, " + str(len(cost_2_dropped_evaluated_candidates[c])) + " did not satisfy the epsilon threshold.") else: print("Of " + str(len(current_layer)) + " candidate representations, all satisfied the epsilon threshold.") print("Best representation found for complexity = " + str(c) + ": " + str(max_feature) + "\n") if self.save_logs: pickle.dump(cost_2_raw_features, open(Config.get_default("tmp.folder", "/tmp") + "/data_raw.p", "wb")) pickle.dump(cost_2_unary_transformed, open(Config.get_default("tmp.folder", "/tmp") + "/data_unary.p", "wb")) pickle.dump(cost_2_binary_transformed, open(Config.get_default("tmp.folder", "/tmp") + "/data_binary.p", "wb")) pickle.dump(cost_2_combination, open(Config.get_default("tmp.folder", "/tmp") + "/data_combination.p", "wb")) pickle.dump(cost_2_dropped_evaluated_candidates, open(Config.get_default("tmp.folder", "/tmp") + "/data_dropped.p", "wb"))
def nested_grid_search(self, train_transformed, validation_transformed, train_and_validation_transformed, test_transformed, training_all, one_test_set_transformed): q = deque(range(self.folds)) nested_test_score = [] for outer_i in range(self.folds): test_fold = q.pop() hyperparam_to_score_list = {} my_keys = list(self.grid_search_parameters.keys()) for parameter_combination in itertools.product(*[self.grid_search_parameters[k] for k in my_keys]): parameter_set = hashabledict(zip(my_keys, parameter_combination)) hyperparam_to_score_list[parameter_set] = [] for inner_i in range(self.folds - 1): q.rotate(-1) validation_fold = q[0] training_folds = [q[training_i] for training_i in range(1, self.folds - 1)] ################################################################ clf = self.classifier(**parameter_set) clf.fit(train_transformed[frozenset(training_folds)], self.store_cv_sets_train_target[frozenset(training_folds)]) y_val_pred = clf.predict(validation_transformed[frozenset(training_folds)]) #validation hyperparam_to_score_list[parameter_set].append( f1_score(self.store_cv_sets_validation_target[frozenset(training_folds)], y_val_pred, average='micro')) ################################################################ #find best parameter best_param = None best_mean_cross_val_score = -1 for parameter_config, score_list in hyperparam_to_score_list.items(): mean_score = np.mean(score_list) if mean_score > best_mean_cross_val_score: best_param = parameter_config best_mean_cross_val_score = mean_score ###################### clf = self.classifier(**best_param) clf.fit(train_and_validation_transformed[test_fold], self.store_cv_sets_train_and_validation_target[test_fold]) y_test_pred = clf.predict(test_transformed[test_fold]) nested_test_score.append(f1_score(self.store_cv_sets_test_target[test_fold], y_test_pred, average='micro')) ###################### q.appendleft(test_fold) nested_cross_val_score = np.mean(nested_test_score) test_score = None if Config.get_default('score.test', 'False') == 'True': # refit to entire training and test on test set clf = self.classifier(**best_param) clf.fit(training_all, self.train_y_all_target) y_pred = clf.predict(one_test_set_transformed) test_score = f1_score(self.test_target, y_pred, average='micro') #np.save('/tmp/true_predictions', self.test_target) return nested_cross_val_score, test_score, best_param, y_pred
def run(self): self.global_starting_time = time.time() # generate all candidates self.generate() #starting_feature_matrix = self.create_starting_features() self.generate_target() unary_transformations, binary_transformations = self.transformation_producer(self.train_X_all, self.raw_features) cost_2_raw_features: Dict[int, List[CandidateFeature]] = {} cost_2_unary_transformed: Dict[int, List[CandidateFeature]] = {} cost_2_binary_transformed: Dict[int, List[CandidateFeature]] = {} cost_2_combination: Dict[int, List[CandidateFeature]] = {} if self.save_logs: cost_2_dropped_evaluated_candidates: Dict[int, List[CandidateFeature]] = {} self.complexity_delta = 1.0 unique_raw_combinations = False baseline_score = 0.0#self.evaluate_candidates([CandidateFeature(DummyOneTransformation(None), [self.raw_features[0]])])[0]['score'] #print("baseline: " + str(baseline_score)) max_feature = CandidateFeature(IdentityTransformation(None), [self.raw_features[0]]) max_feature.runtime_properties['score'] = -float("inf") max_feature_per_complexity: Dict[int, CandidateFeature] = {} all_evaluated_features = set() my_globale_module.global_starting_time_global = copy.deepcopy(self.global_starting_time) my_globale_module.grid_search_parameters_global = copy.deepcopy(self.grid_search_parameters) my_globale_module.score_global = copy.deepcopy(self.score) my_globale_module.classifier_global = copy.deepcopy(self.classifier) my_globale_module.target_train_folds_global = copy.deepcopy(self.target_train_folds) my_globale_module.target_test_folds_global = copy.deepcopy(self.target_test_folds) my_globale_module.train_y_all_target_global = copy.deepcopy(self.train_y_all_target) my_globale_module.test_target_global = copy.deepcopy(self.test_target) my_globale_module.max_timestamp_global = copy.deepcopy(self.max_timestamp) my_globale_module.preprocessed_folds_global = copy.deepcopy(self.preprocessed_folds) my_globale_module.epsilon_global = copy.deepcopy(self.epsilon) my_globale_module.complexity_delta_global = copy.deepcopy(self.complexity_delta) my_globale_module.remove_parents = copy.deepcopy(self.remove_parents) c = 1 while(True): current_layer: List[CandidateFeature] = [] #0th if c == 1: cost_2_raw_features[c]: List[CandidateFeature] = [] #print(self.raw_features) for raw_f in self.raw_features: sympy_representation = sympy.Symbol('X' + str(raw_f.column_id)) raw_f.sympy_representation = sympy_representation all_evaluated_features.add(sympy_representation) if raw_f.is_numeric(): if raw_f.properties['missing_values']: raw_f.runtime_properties['score'] = 0.0 cost_2_raw_features[c].append(raw_f) else: current_layer.append(raw_f) #print("numeric: " + str(raw_f)) else: raw_f.runtime_properties['score'] = 0.0 cost_2_raw_features[c].append(raw_f) #print("nonnumeric: " + str(raw_f)) self.materialize_raw_features(raw_f) #raw_f.derive_properties(raw_f.runtime_properties['train_transformed'][0]) # first unary # we apply all unary transformation to all c-1 in the repo (except combinations and other unary?) unary_candidates_to_be_applied: List[CandidateFeature] = [] if (c - 1) in cost_2_raw_features: unary_candidates_to_be_applied.extend(cost_2_raw_features[c - 1]) if (c - 1) in cost_2_unary_transformed: unary_candidates_to_be_applied.extend(cost_2_unary_transformed[c - 1]) if (c - 1) in cost_2_binary_transformed: unary_candidates_to_be_applied.extend(cost_2_binary_transformed[c - 1]) all_unary_features = self.generate_features(unary_transformations, unary_candidates_to_be_applied, all_evaluated_features) current_layer.extend(all_unary_features) #second binary #get length 2 partitions for current cost partition = self.get_length_2_partition(c-1) #print("bin: c: " + str(c) + " partition" + str(partition)) #apply cross product from partitions binary_candidates_to_be_applied: List[CandidateFeature] = [] for p in partition: lists_for_each_element: List[List[CandidateFeature]] = [[], []] for element in range(2): if p[element] in cost_2_raw_features: lists_for_each_element[element].extend(cost_2_raw_features[p[element]]) if p[element] in cost_2_unary_transformed: lists_for_each_element[element].extend(cost_2_unary_transformed[p[element]]) if p[element] in cost_2_binary_transformed: lists_for_each_element[element].extend(cost_2_binary_transformed[p[element]]) for bt in binary_transformations: list_of_combinations = self.generate_merge(lists_for_each_element[0], lists_for_each_element[1], bt.parent_feature_order_matters, bt.parent_feature_repetition_is_allowed) #print(list_of_combinations) for combo in list_of_combinations: if bt.is_applicable(combo): sympy_representation = bt.get_sympy_representation( [p.get_sympy_representation() for p in combo]) try: if len(sympy_representation.free_symbols) > 0: # if expression is not constant if not sympy_representation in all_evaluated_features: bin_candidate = CandidateFeature(copy.deepcopy(bt), combo) bin_candidate.sympy_representation = copy.deepcopy(sympy_representation) all_evaluated_features.add(sympy_representation) binary_candidates_to_be_applied.append(bin_candidate) else: #print(str(bin_candidate) + " skipped: " + str(sympy_representation)) pass else: #print(str(bin_candidate) + " skipped: " + str(sympy_representation)) pass except: pass current_layer.extend(binary_candidates_to_be_applied) #third: feature combinations #first variant: treat combination as a transformation #therefore, we can use the same partition as for binary data partition = self.get_length_2_partition(c) #print("combo c: " + str(c) + " partition" + str(partition)) combinations_to_be_applied: List[CandidateFeature] = [] for p in partition: lists_for_each_element: List[List[CandidateFeature]] = [[], []] for element in range(2): if p[element] in cost_2_raw_features: lists_for_each_element[element].extend(cost_2_raw_features[p[element]]) if p[element] in cost_2_unary_transformed: lists_for_each_element[element].extend(cost_2_unary_transformed[p[element]]) if p[element] in cost_2_binary_transformed: lists_for_each_element[element].extend(cost_2_binary_transformed[p[element]]) if p[element] in cost_2_combination: lists_for_each_element[element].extend(cost_2_combination[p[element]]) combinations_to_be_applied = self.generate_merge_for_combination(all_evaluated_features, lists_for_each_element[0], lists_for_each_element[1]) current_layer.extend(combinations_to_be_applied) if unique_raw_combinations: length = len(current_layer) current_layer = self.filter_non_unique_combinations(current_layer) print("From " + str(length) + " combinations, we filter " + str(length - len(current_layer)) + " nonunique raw feature combinations.") #now evaluate all from this layer #print(current_layer) print("----------- Evaluation of " + str(len(current_layer)) + " representations -----------") results = evaluate_candidates(current_layer) print("----------- Evaluation Finished -----------") #print(results) layer_end_time = time.time() - self.global_starting_time #calculate whether we drop the evaluated candidate for candidate in results: if type(candidate) != type(None): candidate.runtime_properties['layer_end_time'] = layer_end_time #print(str(candidate) + " -> " + str(candidate.runtime_properties['score'])) if candidate.runtime_properties['score'] > max_feature.runtime_properties['score']: max_feature = candidate if candidate.runtime_properties['passed']: if isinstance(candidate, RawFeature): if not c in cost_2_raw_features: cost_2_raw_features[c]: List[CandidateFeature] = [] cost_2_raw_features[c].append(candidate) elif isinstance(candidate.transformation, UnaryTransformation): if not c in cost_2_unary_transformed: cost_2_unary_transformed[c]: List[CandidateFeature] = [] cost_2_unary_transformed[c].append(candidate) elif isinstance(candidate.transformation, IdentityTransformation): if not c in cost_2_combination: cost_2_combination[c]: List[CandidateFeature] = [] cost_2_combination[c].append(candidate) else: if not c in cost_2_binary_transformed: cost_2_binary_transformed[c]: List[CandidateFeature] = [] cost_2_binary_transformed[c].append(candidate) else: if self.save_logs: if not c in cost_2_dropped_evaluated_candidates: cost_2_dropped_evaluated_candidates[c]: List[CandidateFeature] = [] cost_2_dropped_evaluated_candidates[c].append(candidate) satisfied_count = 0 if c in cost_2_raw_features: satisfied_count += len(cost_2_raw_features[c]) if c in cost_2_unary_transformed: satisfied_count += len(cost_2_unary_transformed[c]) if c in cost_2_binary_transformed: satisfied_count += len(cost_2_binary_transformed[c]) if c in cost_2_combination: satisfied_count += len(cost_2_combination[c]) all_count = len(current_layer) if c == 1: all_count = len(cost_2_raw_features[c]) print("Of " + str(all_count) + " candidate representations, " + str(satisfied_count) + " did satisfy the epsilon threshold.") if len(current_layer) > 0: if Config.get_default('score.test', 'False') == 'True': print("\nBest representation found for complexity = " + str(c) + ": " + str(max_feature) + "\nmean cross-validation score: " + "{0:.2f}".format(max_feature.runtime_properties['score']) + ", score on test: " + "{0:.2f}".format(max_feature.runtime_properties['test_score']) + "\n") else: print("\nBest representation found for complexity = " + str(c) + ": " + str( max_feature) + "\nmean cross-validation score: " + "{0:.2f}".format( max_feature.runtime_properties['score']) + "\n") #print("hyper: " + str(max_feature.runtime_properties['hyperparameters'])) #print(max_feature.runtime_properties['fold_scores']) # upload best feature to OpenML if self.upload2openml: candidate2openml(max_feature, my_globale_module.classifier_global, self.reader.task, 'ComplexityDriven') if self.save_logs: pickle.dump(cost_2_raw_features, open(Config.get_default("tmp.folder", "/tmp") + "/data_raw.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_unary_transformed, open(Config.get_default("tmp.folder", "/tmp") + "/data_unary.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_binary_transformed, open(Config.get_default("tmp.folder", "/tmp") + "/data_binary.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_combination, open(Config.get_default("tmp.folder", "/tmp") + "/data_combination.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) pickle.dump(cost_2_dropped_evaluated_candidates, open(Config.get_default("tmp.folder", "/tmp") + "/data_dropped.p", "wb"), protocol=pickle.HIGHEST_PROTOCOL) max_feature_per_complexity[c] = max_feature if type(self.c_max) == type(None) and c > 2: # calculate harmonic mean harmonic_means = [0.0]*3 for h_i in range(len(harmonic_means)): simplicity_cum_score = self.getSimplicityScore(max_feature_per_complexity[c-h_i].get_complexity(), c, cost_2_raw_features, cost_2_unary_transformed, cost_2_binary_transformed, cost_2_combination) accuracy_cum_score = self.getAccuracyScore(max_feature_per_complexity[c-h_i].runtime_properties['score'], c, cost_2_raw_features, cost_2_unary_transformed, cost_2_binary_transformed, cost_2_combination) harmonic_means[h_i] = self.harmonic_mean(simplicity_cum_score, accuracy_cum_score) #print(str(max_feature_per_complexity[c-h_i]) + ": " + str(harmonic_means[h_i]) + " h: " + str(h_i)) if harmonic_means[2] >= harmonic_means[1] and harmonic_means[2] >= harmonic_means[0]: print("Best Harmonic Mean: " + str(max_feature_per_complexity[c-2])) break if type(self.max_timestamp) != type(None) and time.time() >= self.max_timestamp: break c += 1 if type(self.c_max) != type(None) and self.c_max < c: break
def evaluate(self, candidate: CandidateFeature, score=make_scorer(f1_score, average='micro')): if type(self.max_timestamp) != type(None) and time.time() >= self.max_timestamp: raise RuntimeError('Out of time!') result = {} train_transformed = {} validation_transformed = {} train_and_validation_transformed = [None] * self.folds test_transformed = [None] * self.folds #test training_all = None one_test_set_transformed = None result['train_transformed'] = None result['validation_transformed'] = None result['train_and_validation_transformed'] = None result['test_transformed'] = None result['one_test_set_transformed'] = None if isinstance(candidate, RawFeature): if Config.get_default('score.test', 'False') == 'True': result['training_all'] = training_all = self.name_to_training_all[str(candidate)] result['one_test_set_transformed'] = one_test_set_transformed = self.name_to_one_test_set_transformed[str(candidate)] result['train_transformed'] = train_transformed = self.name_to_train_transformed[str(candidate)] result['validation_transformed'] = validation_transformed = self.name_to_validation_transformed[str(candidate)] result['train_and_validation_transformed'] = train_and_validation_transformed =self.name_to_train_and_validation_transformed[str(candidate)] result['test_transformed'] = test_transformed = self.name_to_test_transformed[str(candidate)] else: #print(self.name_to_train_transformed.keys()) #merge columns from parents for key, value in self.name_to_train_transformed[str(list(candidate.parents)[0])].items(): train_transformed_input = np.hstack([self.name_to_train_transformed[str(p)][key] for p in candidate.parents]) validation_transformed_input = np.hstack([self.name_to_validation_transformed[str(p)][key] for p in candidate.parents]) candidate.transformation.fit(train_transformed_input) train_transformed[key] = candidate.transformation.transform(train_transformed_input) validation_transformed[key] = candidate.transformation.transform(validation_transformed_input) for fold_i in range(self.folds): train_and_validation_transformed_input = np.hstack([self.name_to_train_and_validation_transformed[str(p)][fold_i] for p in candidate.parents]) test_transformed_input = np.hstack([self.name_to_test_transformed[str(p)][fold_i] for p in candidate.parents]) candidate.transformation.fit(train_and_validation_transformed_input) train_and_validation_transformed[fold_i] = candidate.transformation.transform(train_and_validation_transformed_input) test_transformed[fold_i] = candidate.transformation.transform(test_transformed_input) if Config.get_default('score.test', 'False') == 'True': training_all_input = np.hstack( [self.name_to_training_all[str(p)] for p in candidate.parents]) one_test_set_transformed_input = np.hstack( [self.name_to_one_test_set_transformed[str(p)] for p in candidate.parents]) candidate.transformation.fit(training_all_input) training_all = candidate.transformation.transform(training_all_input) one_test_set_transformed = candidate.transformation.transform(one_test_set_transformed_input) candidate.runtime_properties['score'], candidate.runtime_properties['test_score'], candidate.runtime_properties['hyperparameters'], y_pred = self.nested_grid_search(train_transformed, validation_transformed, train_and_validation_transformed, test_transformed, training_all, one_test_set_transformed) if Config.get_default('store.predictions', 'False') == 'True': candidate.runtime_properties['predictions'] = y_pred if not isinstance(candidate, RawFeature): #only save the transformed data if we need it in the future max_parent = np.max([p.runtime_properties['score'] for p in candidate.parents]) accuracy_delta = candidate.runtime_properties['score'] - max_parent if accuracy_delta / self.complexity_delta > self.epsilon: result['train_transformed'] = train_transformed result['validation_transformed'] = validation_transformed result['train_and_validation_transformed'] = train_and_validation_transformed result['test_transformed'] = test_transformed result['training_all'] = training_all result['one_test_set_transformed'] = one_test_set_transformed # derive properties if not isinstance(candidate, RawFeature): candidate.derive_properties(result['train_and_validation_transformed'][0]) return result