def train(self, features: List[List[float]], labels: List[int]): X = np.array(features) N = X.shape[0] pi = np.full((N), 1 / 2) y = np.array(labels) f = np.zeros((N)) hx = np.array((N)) for t in range(self.T): num = ((y + 1) / 2) - pi den = np.multiply(pi, 1 - pi) z = num / den w = np.multiply(pi, 1 - pi) #step5 min = 9223372036854775807 for clf in self.clfs: decstump = DecisionStump(clf.s, clf.b, clf.d) hxpred = np.array(decstump.predict(features)) check = np.sum( np.multiply(w, np.multiply(z - hxpred, z - hxpred))) if check < min: min_clf = clf hx = hxpred min = check self.clfs_picked.append(min_clf) self.betas.append(0.5) f = f + (1 / 2) * hx den = 1 + np.exp(-2 * f) pi = 1 / den
def predict(self, features: List[List[float]]) -> List[int]: x = np.array(features) f = np.zeros(x.shape[0]) for t in range(self.T): decstump = DecisionStump(self.clfs_picked[t].s, self.clfs_picked[t].b, self.clfs_picked[t].d) f = f + (self.betas[t] * np.array(decstump.predict(features))) predictions = np.ones(f.shape, np.int) predictions[np.where(f < 0)[0]] = -1 return predictions.tolist()
def q14(): T = 1 clf = AdaBoost(T, lambda u: DecisionStump(u), lambda dd: dd.err_) X, y = load_ada_boost_train() clf.fit(X, y) print clf.u print np.sum(clf.u) print 'AND SEE q13'
def build_tree(self, X, Y, w, depth, curr_depth): # See if we can do any splitting at all tree = Node() yw = Y*w if len(X)<2 or len(unique(Y)) < 2 or curr_depth >= depth: tree.stump = 1.0 if abs(sum(yw[yw>=0]))>abs(sum(yw[yw<0])) else -1.0 return tree # TODO: check for inconsistent data # Learn the decision stump stump = DecisionStump().fit(X,Y,w) side1 = stump.predict(X)>=0 side2 = stump.predict(X)<0 tree.stump = stump tree.left = self.build_tree(X[side1], Y[side1], w[side1], depth, curr_depth+1) tree.right = self.build_tree(X[side2], Y[side2], w[side2], depth, curr_depth+1) return tree
def train(self, features: List[List[float]], labels: List[int]): X = np.array(features) N = X.shape[0] w = np.full((N), 1 / N) labels = np.array(labels) hx = np.array((N)) for t in range(self.T): #step3 min = 9223372036854775807 for clf in self.clfs: decstump = DecisionStump(clf.s, clf.b, clf.d) hxpred = np.array(decstump.predict(features)) indicator = np.zeros((N)) indicator[np.where(labels != hxpred)[0]] = 1 check = np.sum(np.multiply(w, indicator)) if check < min: min_clf = clf hx = hxpred min = check self.clfs_picked.append(min_clf) error = 0 for i in range(N): if labels[i] != hx[i]: error = error + w[i] beta = (1 / 2) * np.log((1 - error) / error) self.betas.append(beta) for i in range(N): if labels[i] == hx[i]: w[i] = w[i] * np.exp((-1) * self.betas[t]) else: w[i] = w[i] * np.exp(self.betas[t]) w_sum = np.sum(w) w = w / w_sum
def q13(): T = 300 clf = AdaBoost(T, lambda u: DecisionStump(u), lambda dd: dd.err_) X, y = load_ada_boost_train() clf.fit(X, y) print clf.e print 'Ein:', clf.score(X, y) print '%f <= U_T <= %f' % (np.min(clf.U), np.max(clf.U)) print 'e_t >= %f' % np.min(clf.e) X, y = load_ada_boost_test() print 'Eout:', clf.score(X, y) print 'U(2):', clf.U[1]
def q12(): clf = DecisionStump() X, y = load_ada_boost_train() clf.fit(X, y) print 'Ein:', clf.score(X, y) X, y = load_ada_boost_test() print 'Eout:', clf.score(X, y)
def train(self, features: List[List[float]], labels: List[int]): ############################################################ # TODO: implement "train" ############################################################ N = len(features) w = [[0 for n in range(N)] for m in range(self.T +1)] ###row weight of feature, col iteration w[0] = [1/N for n in range(N)] ###initial weight of each feature is 1/N for t in range(self.T): ### find T classifiers(T iterations) min_w_sum = 2147483647 best_d = 0 best_b = 0.0 best_s = 0 for clf in self.clfs:### find the best classifier w_sum = 0 ### sum of weight of n features whose label is not equal to classifer estimate for j in range(len(features)): ### compute the error rate of each classifier a = 0 if features[j][clf.d] > clf.b: a = clf.s else: a = -clf.s if a != labels[j]: w_sum += w[t][j] if w_sum < min_w_sum: min_w_sum = w_sum best_d = clf.d best_b = clf.b best_s = clf.s self.clfs_picked.append(DecisionStump(best_s,best_b,best_d)) error = min_w_sum beta = 0.5 * np.log((1-error)/error) self.betas.append(beta) for i in range(len(features)): #upadate weight of N features b =0 if features[i][best_d] > best_b: b = best_s else: b = -best_s if b == labels[i]: w[t+1][i] = w[t][i] * np.exp(-beta) else: w[t+1][i] = w[t][i] * np.exp(beta) sum = 0 #normalize weight for i in range(len(w[0])): sum += w[t+1][i] for i in range(len(w[0])): w[t+1][i] /= sum
def build_tree(self, X, Y, w, depth, curr_depth): # See if we can do any splitting at all tree = Node() yw = Y * w if len(X) < 2 or len(unique(Y)) < 2 or curr_depth >= depth: tree.stump = 1.0 if abs(sum(yw[yw >= 0])) > abs(sum( yw[yw < 0])) else -1.0 return tree # TODO: check for inconsistent data # Learn the decision stump stump = DecisionStump().fit(X, Y, w) side1 = stump.predict(X) >= 0 side2 = stump.predict(X) < 0 tree.stump = stump tree.left = self.build_tree(X[side1], Y[side1], w[side1], depth, curr_depth + 1) tree.right = self.build_tree(X[side2], Y[side2], w[side2], depth, curr_depth + 1) return tree
def boostRound(self): weakLearners = [] self.weights /= self.weights.sum() for feature in self.featuretbl: rec1 = feature[:4] rec2 = feature[2:] stump = DecisionStump(rec1, rec2) stump.fit(self.iimages, self.labels, self.weights) weakLearners.append(stump) errors = np.array([learner.error for learner in weakLearners]) bestLearner = weakLearners[errors.argmin()] error = bestLearner.error beta = error/(1-error) alpha = np.log(1/beta) predictions = bestLearner.predict(self.iimages) self.featuretbl = np.delete(self.featuretbl, np.argmin(errors), 0) self.weights *= np.power(beta, 1 - np.equal(predictions, self.labels)) return alpha, bestLearner
def train(self, features: List[List[float]], labels: List[int]): ''' Inputs: - features: the features of all examples - labels: the label of all examples Require: - store what you learn in self.clfs_picked and self.betas ''' ############################################################ # TODO: implement "train" ############################################################ features = np.asarray(features) #initization if np.ndim(features) == 1: features = np.expand_dims(features, axis=0) labels = np.asarray(labels) h = list(self.clfs) h_list = [] for n in range(len(h)): h_list.append(DecisionStump(h[n].s, h[n].b, h[n].d)) # import classifiers N = features.shape[0] D = np.zeros(N) #means D_t, D_t+1 h_t = [] e_t = b_t = np.zeros(self.T) D[:] = 1 / N for t in range(0, self.T): h_t1 = np.array([ np.multiply( D, (h_list[n].predict(features) != labels).astype(int)).sum() for n in range(len(h)) ]) h_t.append(h_list[np.argmin(h_t1)]) e_t[t] = np.multiply( D, (h_t[t].predict(features) != labels).astype(int)).sum() b_t[t] = 0.5 * np.log((1 - e_t[t]) / e_t[t]) sgn = (h_t[t].predict(features) != labels).astype(int) sgn[sgn == 0] = -1 D = D * np.exp(b_t[t] * sgn) D = D / np.sum(D) self.clfs_picked = h_t self.betas = b_t.tolist() return
def train(self, features: List[List[float]], labels: List[int]): ############################################################ # TODO: implement "train" ############################################################ N = len(features) pi = [[0 for i in range(N)] for j in range(self.T + 1)]### pi_0 is 1/2, N*(T +1) matrix pi[0] = [0.5 for i in range(N)] z = [[0 for i in range(N)] for j in range(self.T)] ### z is N * T w = [[0 for i in range(N)] for j in range(self.T)] ### w is N * T, w_i[0] = 1/N, i =1,2,...N w[0] = [0 for i in range(N)] f = [[0 for i in range(N)] for j in range(self.T + 1)] ### F(x) = 0 for t in range(self.T):### T iterations for n in range(len(features)): ### update w and z z[t][n]=((labels[n]+1)/2 - pi[t][n])/(pi[t][n]*(1 - pi[t][n])) w[t][n] = pi[t][n] * (1 - pi[t][n]) min_w_sum = 2147483647 best_d = 0 best_b = 0.0 best_s = 0 for clf in self.clfs: ### find the best classifier w_sum = 0 ### compute the sum of weight of each classifier for n in range(len(features)): h_xn = 0 if features[n][clf.d] > clf.b: h_xn = clf.s else: h_xn = -clf.s w_sum += w[t][n] * np.square(z[t][n] - h_xn) if w_sum < min_w_sum: ### find the best classifer h_t min_w_sum = w_sum best_d = clf.d best_b = clf.b best_s = clf.s self.clfs_picked.append(DecisionStump(best_s,best_b,best_d)) ### add the best classifier self.betas.append(1) ### should we use 0.5 or 1 as beta? for n in range(len(features)): h_t_x = 0 if features[n][best_d] > best_b: h_t_x = best_s else: h_t_x = -best_s f[t+1][n] = f[t][n] + 0.5 * h_t_x for n in range(len(features)): ### the last step use f to update pi pi[t+1][n] = 1/(1+ np.exp(-2 * f[t+1][n]))
# PLOT RESULT utils.plotClassifier(model, X, y) fname = os.path.join("..", "figs", "q2_decisionBoundary.pdf") plt.savefig(fname) print("\nFigure saved as '%s'" % fname) elif question == "2.2": # 1. Load citiesSmall dataset dataset = utils.load_dataset("citiesSmall") X = dataset["X"] y = dataset["y"] # 3. Evaluate decision stump model = DecisionStump() model.fit(X, y) y_pred = model.predict(X) error = np.mean(y_pred != y) print("Decision Stump with inequality rule error: %.3f" % error) # PLOT RESULT utils.plotClassifier(model, X, y) fname = os.path.join("..", "figs", "q2.2_decisionBoundary.pdf") plt.savefig(fname) print("\nFigure saved as '%s'" % fname) elif question == "2.3": # 1. Load citiesSmall dataset
'--module', required=True, choices=["1.1", "1.2", "1.3", "1.4", "1.5"]) io_args = parser.parse_args() module = io_args.module # Decision Stump using inequalities/threshold if module == "1.1": # 1. Load citiesSmall dataset dataset = load_dataset("citiesSmall.pkl") X = dataset["X"] y = dataset["y"] # 2. Evaluate decision stump model = DecisionStump() model.fit(X, y) y_pred = model.predict(X) error = np.mean(y_pred != y) print("Decision Stump with inequality rule error: %.3f" % error) # PLOT RESULT utils.plotClassifier(model, X, y) fname = os.path.join("..", "figs", "decision_stump_boundary.pdf") plt.savefig(fname) print("\nFigure saved as '%s'" % fname) # Simple decision tree using decision stumps elif module == "1.2":
self.E.append(self.score(X, y)) def predict(self, x): return sign(np.sum([a * g.predict(x) for a, g in zip(self.alpha, self.g)])) def score(self, X, y): y_ = np.apply_along_axis(lambda x: self.predict(x), axis=1, arr=X) return np.sum(y != y_) * 1.0 / len(y) if __name__ == '__main__': X = np.array([[ 1. , 2.1], [ 2. , 1.1], [ 1.3, 1. ], [ 1. , 1. ], [ 2. , 1. ]]) y = np.array([1.0, 1.0, -1.0, -1.0, 1.0]) T = 5 from decision_stump import DecisionStump clf = AdaBoost(T, lambda u: DecisionStump(u), lambda dd: dd.err_) clf.fit(X, y) print clf.score(X, y) print clf.E for a, g in zip(clf.alpha, clf.g): print a, g for i in range(len(X)): print clf.predict(X[i])
# -*- coding: utf-8 -*- from decision_stump import DecisionStump if __name__ == '__main__': q7 = DecisionStump(20, 1000) q7.run_and_render_histogram() q8 = DecisionStump(2000, 1000) q8.run_and_render_histogram()