'''newtrain = train_df.applymap(encode_units) print(newtrain) newtrain=newtrain.dropna() predictors1 = train_df.columns.drop(['Product_ID','User_ID','Marital_Status','Stay_In_Current_City_Years']) frequent_itemsets = apriori(newtrain[predictors1], min_support=0.07, use_colnames=True) rules = association_rules(frequent_itemsets, metric="lift", min_threshold=1) print(rules) print(rules[ (rules['lift'] > 1.0) & (rules['confidence'] > 0.73)]) ''' X=train_df[predictors].loc[:2000,] y=train_df[target].loc[:2000,] clf = tree.DecisionTreeRegressor() clf = clf.fit(X,y) print(clf) pd.DataFrame(clf.decision_path(X).toarray()).head(5) pd.concat([X.reset_index(drop=True),pd.DataFrame(clf.decision_path(X).toarray())],1).head(5) print("\n\n",find_node(tree_ = clf.tree_, current_node = 0, search_node = 13, features = X.columns.tolist()),"\n\n") print(dataset[(dataset['Purchase'] >= 10000)]) dTree3 = DecisionTreeRegressor(max_depth = 6) commonfit(dTree3, train_df, test_df, predictors, target, IDcol, 'DT-new.csv') Xrules = pd.concat([X.reset_index(drop=True),pd.DataFrame(dTree3.decision_path(X).toarray()).iloc[:,1:]],1) tree_model = DecisionTreeRegressor() tree_model.fit(Xrules, y)\ commonfit(tree_model, train_df, test_df, predictors, target, IDcol, 'DT-new.csv')
max_leaf_nodes=70, min_samples_split=3, splitter='best', criterion='mse') tree.fit(X_train, y_train ) #Build a decision tree classifier from the training set (X, y). print('min samples leaf = ', i) predictions = tree.predict(X_test) err1 = np.sum(abs(y_test - predictions)**2) / len(y_test) error.append(err1) r = r2_score(y_test, predictions) r2.append(r) print('Minimum numbers of leafs: ', err1) print('R SQUARED: ', r) print(tree.decision_path(X_test)) df = pd.DataFrame({'Actual': y_test, 'Predicted': np.round(predictions)}) print(df) err = np.sum(abs(y_test - predictions)) / len(y_test) print('Mean absolute error for test =', err) err1 = np.sum(abs(y_test - predictions)**2) / len(y_test) print('Mean square error for test =', err1) mse = np.sum(abs(y_test - predictions)**2) print('MSE =', mse) rmse = np.sqrt(mse / len(y_test)) print('RMSE Root mean squared error =', rmse)
def print_tree(estimator: DecisionTreeRegressor, X_test: DataFrame, scaler, feature_labels): n_nodes = estimator.tree_.node_count children_left = estimator.tree_.children_left children_right = estimator.tree_.children_right feature = estimator.tree_.feature threshold = estimator.tree_.threshold # The tree structure can be traversed to compute various properties such # as the depth of each node and whether or not it is a leaf. node_depth = np.zeros(shape=n_nodes, dtype=np.int64) is_leaves = np.zeros(shape=n_nodes, dtype=bool) stack = [(0, -1)] # seed is the root node id and its parent depth while len(stack) > 0: node_id, parent_depth = stack.pop() node_depth[node_id] = parent_depth + 1 # If we have a test node if (children_left[node_id] != children_right[node_id]): stack.append((children_left[node_id], parent_depth + 1)) stack.append((children_right[node_id], parent_depth + 1)) else: is_leaves[node_id] = True # print("The binary tree structure has %s nodes and has " # "the following tree structure (top 3):" # % n_nodes) # for i in range(max(3, n_nodes)): # if is_leaves[i]: # print("%snode=%s leaf node." % (node_depth[i] * "\t", i)) # else: # print("%snode=%s test node: go to node %s if X[:, %s] <= %s else to " # "node %s." # % (node_depth[i] * "\t", # i, # children_left[i], # feature[i], # threshold[i], # children_right[i], # )) # print() # First let's retrieve the decision path of each sample. The decision_path # method allows to retrieve the node indicator functions. A non zero element of # indicator matrix at the position (i, j) indicates that the sample i goes # through the node j. vals = scaler.transform(X_test) node_indicator = estimator.decision_path(vals) feature_labels = X_test.columns.tolist() # Similarly, we can also have the leaves ids reached by each sample. # leave_id = estimator.apply(X_test) # Now, it's possible to get the tests that were used to predict a sample or # a group of samples. First, let's make it for the sample. sample_id = 0 node_index = node_indicator.indices[ node_indicator.indptr[sample_id]:node_indicator.indptr[sample_id + 1]] print('Rules used to predict sample %s: ' % sample_id) for node_id in node_index: # if leave_id[sample_id] != node_id: # continue if (vals[sample_id, feature[node_id]] <= threshold[node_id]): threshold_sign = "<=" else: threshold_sign = ">" print("decision id node %s : (%s: %s %s %s), original val: %s" % (node_id, feature_labels[feature[node_id]], vals[sample_id, feature[node_id]], threshold_sign, threshold[node_id], X_test.iloc[sample_id, feature[node_id]])) # For a group of samples, we have the following common node. sample_ids = [0, 1] common_nodes = (node_indicator.toarray()[sample_ids].sum( axis=0) == len(sample_ids)) common_node_id = np.arange(n_nodes)[common_nodes] print("\nThe following samples %s share the node %s in the tree" % (sample_ids, common_node_id)) print("It is %s %% of all nodes." % (100 * len(common_node_id) / n_nodes, ))
class GroupPCADecisionTreeRegressor(BaseEstimator, RegressorMixin): """ PCA on random group of features followed by a Decision Tree See : GroupPCA and DecisionTreeRegressor """ def __init__( self, criterion="mse", splitter="best", max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=None, random_state=None, max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, presort=False, pca_bootstrap=False, pca_max_nb_groups=0.25, pca_max_group_size=0.05, ): self.criterion = criterion self.splitter = splitter self.max_depth = max_depth self.min_samples_split = min_samples_split self.min_samples_leaf = min_samples_leaf self.min_weight_fraction_leaf = min_weight_fraction_leaf self.max_features = max_features self.random_state = random_state self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease self.min_impurity_split = min_impurity_split self.presort = presort self.pca_bootstrap = pca_bootstrap self.pca_max_nb_groups = pca_max_nb_groups self.pca_max_group_size = pca_max_group_size self._tree = None self._group_pca = None def fit(self, X, y, sample_weight=None, check_input=True, X_idx_sorted=None): self.n_features_ = X.shape[1] # 1) create GroupPCA self._group_pca = GroupPCA( random_state=self.random_state, bootstrap=self.pca_bootstrap, max_nb_groups=self.pca_max_nb_groups, max_group_size=self.pca_max_group_size, ) # 2) Create Tree self._tree = DecisionTreeRegressor( criterion=self.criterion, splitter=self.splitter, max_depth=self.max_depth, min_samples_split=self.min_samples_split, min_samples_leaf=self.min_samples_leaf, min_weight_fraction_leaf=self.min_weight_fraction_leaf, max_features=self.max_features, max_leaf_nodes=self.max_leaf_nodes, random_state=self.random_state, min_impurity_decrease=self.min_impurity_decrease, min_impurity_split=self.min_impurity_split, presort=self.presort, ) # 3) Apply group PCA Xpca = self._group_pca.fit_transform(X, y) # 4) fit Tree self._tree.fit(Xpca, y, sample_weight=sample_weight, check_input=check_input, X_idx_sorted=None) return self def predict(self, X, check_input=True): if self._tree is None: raise NotFittedError("You should fit the model first") Xpca = self._group_pca.transform(X) return self._tree.predict(Xpca, check_input=check_input) def apply(self, X, check_input=True): if self._tree is None: raise NotFittedError("You should fit the model first") Xpca = self._group_pca.transform(X) return self._tree.apply(Xpca, check_input=check_input) def decision_path(self, X, check_input=True): Xpca = self._group_pca.transform(X) return self._tree.decision_path(Xpca, check_input=check_input) @property def tree_(self): return self._tree.tree_ def _validate_X_predict(self, X, check_input): """Validate X whenever one tries to predict, apply, predict_proba""" if check_input: X = check_array(X, dtype=DTYPE, accept_sparse="csr") if issparse(X) and (X.indices.dtype != np.intc or X.indptr.dtype != np.intc): raise ValueError("No support for np.int64 index based " "sparse matrices") n_features = X.shape[1] if self.n_features_ != n_features: raise ValueError( "Number of features of the model must " "match the input. Model n_features is %s and " "input n_features is %s " % (self.n_features_, n_features) ) return X
class SingleTreeCateInterpreter(_SingleTreeInterpreter): """ An interpreter for the effect estimated by a CATE estimator Parameters ---------- include_model_uncertainty : bool, optional, default False Whether to include confidence interval information when building a simplified model of the cate model. If set to True, then cate estimator needs to support the `const_marginal_ate_inference` method. uncertainty_level : double, optional, default .1 The uncertainty level for the confidence intervals to be constructed and used in the simplified model creation. If value=alpha then a multitask decision tree will be built such that all samples in a leaf have similar target prediction but also similar alpha confidence intervals. uncertainty_only_on_leaves : bool, optional, default True Whether uncertainty information should be displayed only on leaf nodes. If False, then interpretation can be slightly slower, especially for cate models that have a computationally expensive inference method. splitter : string, optional, default "best" The strategy used to choose the split at each node. Supported strategies are "best" to choose the best split and "random" to choose the best random split. max_depth : int or None, optional, default None The maximum depth of the tree. If None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples. min_samples_split : int, float, optional, default 2 The minimum number of samples required to split an internal node: - If int, then consider `min_samples_split` as the minimum number. - If float, then `min_samples_split` is a fraction and `ceil(min_samples_split * n_samples)` are the minimum number of samples for each split. min_samples_leaf : int, float, optional, default 1 The minimum number of samples required to be at a leaf node. A split point at any depth will only be considered if it leaves at least ``min_samples_leaf`` training samples in each of the left and right branches. This may have the effect of smoothing the model, especially in regression. - If int, then consider `min_samples_leaf` as the minimum number. - If float, then `min_samples_leaf` is a fraction and `ceil(min_samples_leaf * n_samples)` are the minimum number of samples for each node. min_weight_fraction_leaf : float, optional, default 0. The minimum weighted fraction of the sum total of weights (of all the input samples) required to be at a leaf node. Samples have equal weight when sample_weight is not provided. max_features : int, float or {"auto", "sqrt", "log2"}, default=None The number of features to consider when looking for the best split: - If int, then consider `max_features` features at each split. - If float, then `max_features` is a fraction and `int(max_features * n_features)` features are considered at each split. - If "auto", then `max_features=n_features`. - If "sqrt", then `max_features=sqrt(n_features)`. - If "log2", then `max_features=log2(n_features)`. - If None, then `max_features=n_features`. Note: the search for a split does not stop until at least one valid partition of the node samples is found, even if it requires to effectively inspect more than ``max_features`` features. random_state : int, RandomState instance or None, optional, default None If int, random_state is the seed used by the random number generator; If RandomState instance, random_state is the random number generator; If None, the random number generator is the RandomState instance used by `np.random`. max_leaf_nodes : int or None, optional, default None Grow a tree with ``max_leaf_nodes`` in best-first fashion. Best nodes are defined as relative reduction in impurity. If None then unlimited number of leaf nodes. min_impurity_decrease : float, optional, default 0. A node will be split if this split induces a decrease of the impurity greater than or equal to this value. The weighted impurity decrease equation is the following:: N_t / N * (impurity - N_t_R / N_t * right_impurity - N_t_L / N_t * left_impurity) where ``N`` is the total number of samples, ``N_t`` is the number of samples at the current node, ``N_t_L`` is the number of samples in the left child, and ``N_t_R`` is the number of samples in the right child. ``N``, ``N_t``, ``N_t_R`` and ``N_t_L`` all refer to the weighted sum, if ``sample_weight`` is passed. """ def __init__(self, *, include_model_uncertainty=False, uncertainty_level=.1, uncertainty_only_on_leaves=True, splitter="best", max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0., max_features=None, random_state=None, max_leaf_nodes=None, min_impurity_decrease=0.): self.include_uncertainty = include_model_uncertainty self.uncertainty_level = uncertainty_level self.uncertainty_only_on_leaves = uncertainty_only_on_leaves self.criterion = "mse" self.splitter = splitter self.max_depth = max_depth self.min_samples_split = min_samples_split self.min_samples_leaf = min_samples_leaf self.min_weight_fraction_leaf = min_weight_fraction_leaf self.max_features = max_features self.random_state = random_state self.max_leaf_nodes = max_leaf_nodes self.min_impurity_decrease = min_impurity_decrease def interpret(self, cate_estimator, X): """ Interpret the heterogeneity of a CATE estimator when applied to a set of features Parameters ---------- cate_estimator : :class:`.LinearCateEstimator` The fitted estimator to interpret X : array-like The features against which to interpret the estimator; must be compatible shape-wise with the features used to fit the estimator Returns ------- self: object instance """ self.tree_model_ = DecisionTreeRegressor(criterion=self.criterion, splitter=self.splitter, max_depth=self.max_depth, min_samples_split=self.min_samples_split, min_samples_leaf=self.min_samples_leaf, min_weight_fraction_leaf=self.min_weight_fraction_leaf, max_features=self.max_features, random_state=self.random_state, max_leaf_nodes=self.max_leaf_nodes, min_impurity_decrease=self.min_impurity_decrease) y_pred = cate_estimator.const_marginal_effect(X) self.tree_model_.fit(X, y_pred.reshape((y_pred.shape[0], -1))) paths = self.tree_model_.decision_path(X) node_dict = {} for node_id in range(paths.shape[1]): mask = paths.getcol(node_id).toarray().flatten().astype(bool) Xsub = X[mask] if (self.include_uncertainty and ((not self.uncertainty_only_on_leaves) or (self.tree_model_.tree_.children_left[node_id] < 0))): res = cate_estimator.const_marginal_ate_inference(Xsub) node_dict[node_id] = {'mean': res.mean_point, 'std': res.std_point, 'ci': res.conf_int_mean(alpha=self.uncertainty_level)} else: cate_node = y_pred[mask] node_dict[node_id] = {'mean': np.mean(cate_node, axis=0), 'std': np.std(cate_node, axis=0)} self.node_dict_ = node_dict return self def _make_dot_exporter(self, *, out_file, feature_names, treatment_names, max_depth, filled, leaves_parallel, rotate, rounded, special_characters, precision): return _CateTreeDOTExporter(self.include_uncertainty, self.uncertainty_level, out_file=out_file, feature_names=feature_names, treatment_names=treatment_names, max_depth=max_depth, filled=filled, leaves_parallel=leaves_parallel, rotate=rotate, rounded=rounded, special_characters=special_characters, precision=precision) def _make_mpl_exporter(self, *, title, feature_names, treatment_names, max_depth, filled, rounded, precision, fontsize): return _CateTreeMPLExporter(self.include_uncertainty, self.uncertainty_level, title=title, feature_names=feature_names, treatment_names=treatment_names, max_depth=max_depth, filled=filled, rounded=rounded, precision=precision, fontsize=fontsize)
meanSquare = (N_left*meanSquareLeft + N_right*meanSqaureRight) / N linearReg = linearReg[0] meanSquare = np.sqrt(meanSquare[0]) if(linearReg < linearRegBest): linearRegBest = linearReg meanSquareBest = meanSquare print(leftModel) model = [leftModel, rightModel] print(model) return linearReg, meanSquareBest modelTree, meanSquare = ModelTree(); print(modelTree," ",meanSquare) print ("Time taken to build the model: ",datetime.now() - startTime) node_indicator = regressionTree.decision_path(X_test) leave_id = regressionTree.apply(X_test) sample_id = 0 node_index = node_indicator.indices[node_indicator.indptr[sample_id]: node_indicator.indptr[sample_id + 1]] for i in range(n_nodes): if is_leaves[i]: print("%snode=%s leaf node." % (node_depth[i] * "\t", i)) else: print("%snode=%s test node: go to node %s if X[:, %s] <= %s else to " "node %s." % (node_depth[i] * "\t", i,