def main():
print('--- Adaboost ---')
data = datasets.load_digits()
X, y = data.data, data.target
digit1 = 1
digit2 = 8
idx = np.append(np.where(y == digit1)[0], np.where(y == digit2)[0])
y = data.target[idx]
y[y == digit1] = 1
y[y == digit2] = -1
X = data.data[idx]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5)
clf = Adaboost(n_estimators=5)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
acc = accuracy_score(y_pred, y_test)
clf_tree = ClassificationTree()
clf_tree.fit(X_train, y_train)
y_pred_tree = clf_tree.predict(X_test)
acc_tree = accuracy_score(y_pred, y_test)
print("Adaboost_Accuracy:", acc)
print("Tree_Accuracy:", acc_tree)
def fit(self, X, y):
n_samples, n_features = X.shape[0], X.shape[1]
# 初始化各样本的权重
w = np.full(n_samples, (1 / n_samples))
# 储存每个分类器
self.clfs = []
for i in tqdm(range(self.n_clfs)):
# 实例化一个分类器
clf = ClassificationTree()
# 训练
clf.fit(X, y)
# 得到训练结果
y_pred = clf.predict(X)
# 计算训练误差
print(accuracy_score(y, y_pred))
error = sum(w[y != y_pred])
# 对于错误率大于0.5的分类器，因为是二分类问题(Adaboost只能解决二分类问题)，我们可以翻转
# 分类器的预测结果来使得其错误率为1-error>0.5
print(error)
if error > 0.5:
self.polarity[i] = -1
y_pred *= -1
error = 1 - error
self.alphas[i] = 0.5 * np.log((1.0 - error) / (error + 1e-10))
predictions = np.array(self.polarity[i] * y_pred)
w *= np.exp(-self.alphas[i] * y * predictions)
w /= sum(w)
self.clfs.append(clf)
def evolve(self, X, y, n_iter=7, min_size_prune=1):
self.X = X
self.y = y
for i in range(n_iter):
#print("TAO iter ", i, " di ", n_iter)
for depth in reversed(range(self.classification_tree.depth + 1)):
#print("Ottimizzo depth", depth, "....")
T = self.classification_tree
nodes = ClassificationTree.get_nodes_at_depth(depth, T)
#print ([node.id for node in nodes])
for node in nodes:
self.optimize_nodes(node)
#pool = Pool(4)
#pool.map(self.optimize_nodes, nodes)
#pool.close()
#pool.join()
#for node in nodes:
#self.optimize_nodes(node)
#Rimetto apposto i punti associati ad ogni nodo
self.classification_tree.build_idxs_of_subtree(
X, range(len(X)), T.tree[0], oblique=T.oblique)
#Effettua il pruning finale per togliere dead branches e pure subtrees
#self.prune(min_size = min_size_prune)
ClassificationTree.restore_tree(self.classification_tree)
def optimize_node_parallel(self, node, X, y, alfa, complexity):
rho = 0
if node.is_leaf:
rho = 1
error_best = ClassificationTree.misclassification_loss(
node, X, y, node.data_idxs,
self.classification_tree.oblique) + alfa * complexity
#print("prima parallel")
#Provo lo split
if self.classification_tree.oblique:
node_para, error_para = self.best_svm_split(
node, X, y, alfa, complexity)
else:
node_para, error_para = self.best_parallel_split(
node, X, y, alfa, complexity)
#print(node.id, " fatto parallel plit")
if error_para < error_best:
#print("error para migliore")
error_best = error_para
ClassificationTree.replace_node(node, node_para,
self.classification_tree)
#Metto questo if perchè nel caso di svm se il nodo è una foglia pura allora
#non riesco a fare SVM perchè giustamente si aspetta label differenti.
#Questo non perde di generalità poichè se il nodo fosse una foglia pura allora
#creando due nuovi figli e ottimizzando otterremmo un figlio foglia puro e l'altro
#senza punti. Che avrebbe la stessa loss iniziale del padre. Quindi alla fine la foglia iniziale
#non verrebbe branchata.
if node_para.left_node != None and node_para.right_node != None:
#Errore sul figlio sinistro, se è migliore allora sostituisco
error_lower = ClassificationTree.misclassification_loss(
node_para.left_node, X, y, node.data_idxs,
self.classification_tree.oblique) + alfa * (complexity + rho)
if error_lower < error_best:
#print("error lower migliore")
ClassificationTree.replace_node(node, node_para.left_node,
self.classification_tree)
error_best = error_lower
#Errore sul figlio sinistro, se è migliore allora sostituisco
error_upper = ClassificationTree.misclassification_loss(
node_para.right_node, X, y, node.data_idxs,
self.classification_tree.oblique) + alfa * (complexity + rho)
if error_upper < error_best:
#print("error upper migliore")
ClassificationTree.replace_node(node, node_para.right_node,
self.classification_tree)
error_best = error_upper
return node
def zero_one_loss(node, data, targets):
loss = 0
#Per ogni punto del nodo verifico dove questo lo inoltrerebbe
#se il punto viene inoltrato su un figlio che porta a misclassificazione
#è errore
predictions = ClassificationTree.predict_label(data, node, False)
n_misclassified = np.count_nonzero(targets - predictions)
return n_misclassified
def evolve(self, X, y, alfa=0, max_iteration=1):
ClassificationTree.restore_tree(self.classification_tree)
complexity = self.classification_tree.n_leaves - 1
T = self.classification_tree.tree
self.X = X
self.y = y
i = 0
while (i < max_iteration):
optimized = []
error_prev = ClassificationTree.misclassification_loss(
T[0], X, y, T[0].data_idxs,
self.classification_tree.oblique) + alfa * complexity
values = list(self.classification_tree.tree.keys())
random.shuffle(values)
#print(values)
#print("values: ", values)
while (len(values) > 0):
#print(complexity)
node_id = values.pop()
optimized.append(node_id)
#print("optimizing node:", node_id)
self.optimize_node_parallel(T[node_id], X, y, alfa, complexity)
#print("nodo ottimizzato: ", node_id)
ids = ClassificationTree.restore_tree(self.classification_tree)
complexity = self.classification_tree.n_leaves - 1
#print("complexity: ", complexity)
#print("ids: ", ids)
values = list(set(ids) - set(optimized))
#print("values dopo restore: ", values)
self.classification_tree.build_idxs_of_subtree(
X, range(len(X)), T[0], self.classification_tree.oblique)
error_curr = ClassificationTree.misclassification_loss(
T[0], X, y, T[0].data_idxs,
self.classification_tree.oblique) + alfa * complexity
#print(self.max_id)
#print("i-->", i, "node: ", node_id)
#for node_id in to_delete:
#self.delete_node(node_id)
i += 1
#print("Ottimizzato nodi algoritmo ls: ", i, " volte")
if np.abs(error_curr - error_prev) < 1e-01:
break
def best_svm_split(self, node, X, y, alfa, complexity):
T = self.classification_tree
#Creo un nuovo sottoalbero fittizio con root nel nodo
new_node = TreeNode.copy_node(node)
#if new_node.is_leaf:
#complexity += 1
rho = 0
data = X[new_node.data_idxs]
label = y[new_node.data_idxs]
if not all(i == label[0] for i in label) and len(data) > 0:
#Questo fa SVM multiclasse 1 vs rest
clf = LinearSVC(tol=1e-6,
C=10,
max_iter=10000,
loss='squared_hinge',
penalty='l1',
dual=False)
clf.fit(data, label)
#for n_class in range(n_classes):
#n_misclassified = np.count_nonzero(label-np.sign(np.dot(data, clf.coef_[n_class].T)+clf.intercept_[n_class]))
#Devo capire come ottenere i coefficienti migliori tra il numero di iperpiani addestrati
weights = clf.coef_.reshape((len(X[0])), )
intercept = clf.intercept_
if new_node.is_leaf:
ClassificationTree.create_new_children(node,
X,
y,
self.max_id,
None,
None,
oblique=T.oblique,
weights=weights,
intercept=intercept)
rho = 1
else:
new_node.weights = weights
new_node.intercept = intercept
return new_node, ClassificationTree.misclassification_loss(
new_node, X, y, new_node.data_idxs,
T.oblique) + alfa * (complexity + rho)
return new_node, np.inf
def genetic_tree_optimization(n_trees, n_iter, depth, X, labels, oblique, X_valid, y_valid, CR=0.95, l = 1):
clf = RandomForestClassifier(n_estimators = n_trees, max_depth=depth, random_state=0, min_samples_leaf= 4)
clf.fit(X, labels)
random_trees = clf.estimators_
trees = []
best_loss = np.inf
best_tree = None
for tree in random_trees:
T = ClassificationTree(oblique = oblique)
T.initialize_from_CART(X, labels, tree)
tao_opt(T, X, labels)
trees.append(T)
ClassificationTree.build_idxs_of_subtree(X, range(len(labels)), T.tree[0], oblique)
#ClassificationTree.restore_tree(T)
#multi_optimize_tao(trees, X, labels)
best_loss = np.inf
best_tree = None
for i in range(n_iter):
print("Iter: ", i)
#multi_optimize_evolution(trees, X, labels, CR)
for tree in trees:
#optimize_evolution(tree, trees, X, labels, X_valid, y_valid, CR)
trial = mutation(tree, trees, CR, X, labels, depth)
tao_opt(trial, X, labels)
trial_loss = regularized_loss(trial.tree[0], X, labels, X_valid, y_valid, range(len(labels)), oblique, l=l)
loss = regularized_loss(tree.tree[0], X, labels, X_valid, y_valid, range(len(labels)), oblique, l=l)
if trial_loss < loss:
tree = trial
#print("migliore")
if loss < best_loss:
best_loss = loss
best_tree = tree
print ("best loss: ", best_loss)
print("loss train best: ", 1-ClassificationTree.misclassification_loss(best_tree.tree[0], X, labels, range(len(labels)), oblique))
print("loss valid: ", 1-ClassificationTree.misclassification_loss(best_tree.tree[0], X_valid, y_valid, range(len(y_valid)), oblique))
print("ritorno loss train best: ", 1-ClassificationTree.misclassification_loss(best_tree.tree[0], X, labels, range(len(labels)), oblique))
print("ritono loss valid: ", 1-ClassificationTree.misclassification_loss(best_tree.tree[0], X_valid, y_valid, range(len(y_valid)), oblique))
return best_tree, best_loss
def crossover(tree, trees, CR, X):
partners = random.sample(trees, 3)
partners = [ClassificationTree.copy_tree(partners[0]), ClassificationTree.copy_tree(partners[1]), ClassificationTree.copy_tree(partners[2])]
tree_nodes = list(tree.tree.values())
d = random.choice(range(1, depth))
#for d in reversed(range(tree.depth-1)):
nodes_at_depth = ClassificationTree.get_nodes_at_depth(d, tree)
other_nodes_at_depth = ClassificationTree.get_nodes_at_depth(d, partners[0])
#other_nodes_at_depth.extend(ClassificationTree.get_nodes_at_depth(d, partners[1]))
#other_nodes_at_depth.extend(ClassificationTree.get_nodes_at_depth(d, partners[2]))
#for node in nodes_at_depth:
node = random.choice(nodes_at_depth)
p = np.random.uniform()
if p < CR:
choice = random.choice(other_nodes_at_depth)
if choice.left_node != None and not choice.left_node.is_leaf and choice.right_node != None and not choice.right_node.is_leaf:
p2 = np.random.uniform()
if p2 < 0.5:
node.feature = choice.left_node.feature
node.threshold = choice.left_node.threshold
#ClassificationTree.replace_node(node, choice.left_node, tree)
else:
node.feature = choice.right_node.feature
node.threshold = choice.right_node.threshold
#ClassificationTree.replace_node(node, choice.right_node, tree)
else:
#ClassificationTree.replace_node(node, choice, tree)
node.feature = choice.feature
node.threshold = choice.threshold
'''
if choice.parent_id != -1:
node.feature = tree.tree[choice.parent_id].feature
node.threshold = tree.tree[choice.parent_id].threshold
'''
else:
node.feature = random.choice(range(len(X[0])))
node.threshold = 0
def best_split(node, X, labels):
error_best = np.inf
j = node.feature
vals = {}
best_node = None
for point_idx in node.data_idxs:
vals[point_idx] = X[point_idx, j]
values = sorted(vals.items(), key=lambda x: x[1])
sorted_indexes = [tuple[0] for tuple in values]
#print ("len sort indx: ", len(sorted_indexes))
thresh = 0.5 * X[sorted_indexes[0], j]
node.threshold = thresh
ClassificationTree.create_new_children(node, X, labels, node.id, j, thresh,
False)
actual_loss = zero_one_loss(node, X, labels)
#Ciclo su ogni valore di quella componente e provo tutti gli split
#possibili
i = 0
while i < len(sorted_indexes):
if i < len(sorted_indexes) - 1:
thresh = 0.5 * (X[sorted_indexes[i], j] +
X[sorted_indexes[i + 1], j])
else:
thresh = 1.5 * X[sorted_indexes[i], j]
node.threshold = thresh
ClassificationTree.create_new_children(node, X, labels, node.id, j,
thresh, False)
actual_loss = zero_one_loss(node, X, labels)
if actual_loss < error_best:
error_best = actual_loss
best_left = node.left_node
best_right = node.right_node
best_t = thresh
i += 1
return best_t, best_left, best_right
def optimize_evolution(tree, trees, X, labels, X_valid, y_valid, CR):
trial = ClassificationTree.copy_tree(tree)
'''
n_nodes = len(X[0])/(2**tree.depth - 1)
if n_nodes < 1:
n_nodes = 1
p = np.random.uniform()
for node in nodes:
if p < CR and not node.is_leaf:
node.feature = np.random.randint(0, len(X[0]))
node.threshold = np.random.uniform(np.min(X[node.feature]),np.max(X[node.feature]))
'''
crossover(trial, trees, CR, X)
ClassificationTree.build_idxs_of_subtree(X, range(len(labels)), trial.tree[0], oblique)
#partners = random.sample(trees, 2)
#optimize_crossover(trial, ClassificationTree.copy_tree(partners[0]), ClassificationTree.copy_tree(partners[1]), depth, X, labels, oblique)
tao_opt(trial, X, labels)
loss = regularized_loss(trial.tree[0], X, labels, X_valid, y_valid, range(len(labels)), oblique)
#Se il nuovo individuo è migliore del padre li scambio
if loss < regularized_loss(tree.tree[0], X, labels, X_valid, y_valid, range(len(labels)), oblique):
#if loss < ClassificationTree.misclassification_loss(tree.tree[0], X, labels, range(len(labels)), oblique):
tree = trial
def __init__(self, n_estimators=10, feature_num=None, min_samples_split=2, min_impurity=1e-7,
max_depth=10):
self.n_estimators = n_estimators # 决策树的数量
self.min_samples_split = min_samples_split # 决策树生长所需的最小样本数量
self.min_impurity = min_impurity # 最小不纯度--即最大纯度
self.max_depth = max_depth # 树的最大深度
self.feature_num = feature_num # 每棵树所需的特征数量
self.trees = []
# 实例化每棵决策树
for _ in range(self.n_estimators):
tree = ClassificationTree(min_impurity=self.min_impurity,
min_samples_split=self.min_samples_split,
max_depth=self.max_depth)
self.trees.append(tree)
self.feature_index = []
def optimize_crossover(tree_a, tree_b, tree_c, depth, X, labels, oblique):
d = random.choice(range(depth))
#for d in reversed(range(depth)):
nodes_a = ClassificationTree.get_nodes_at_depth(d, tree_a)
nodes_a = [node for node in nodes_a if not node.is_leaf]
all_nodes = ClassificationTree.get_nodes_at_depth(d, tree_a)
all_nodes.extend(ClassificationTree.get_nodes_at_depth(d, tree_b))
all_nodes.extend(ClassificationTree.get_nodes_at_depth(d, tree_c))
all_nodes = [node for node in all_nodes if not node.is_leaf]
#print (node.id for node in nodes_a)
#worst = nodes_a[np.argmax([ClassificationTree.misclassification_loss(node, X, labels, node.data_idxs, oblique) for node in nodes_a])]
for node in nodes_a:
if len(node.data_idxs) > 0:
best = find_best_branch(all_nodes, node.data_idxs, X, labels, oblique)
best.data_idxs = node.data_idxs
ClassificationTree.replace_node(node, best, tree_a)
def fit(self, X, y):
"""
Grows a forest of decision trees based off the num_trees
attribute
Parameters
----------
X : N x D matrix of real or ordinal values
y : size N vector consisting of either real values or labels for corresponding
index in X
"""
data = np.column_stack((X, y))
self.forest = np.empty(shape=self.num_trees, dtype='object')
sample_size = int(X.shape[0] * self.sample_percentage)
for i in range(self.num_trees):
sample = data[np.random.choice(data.shape[0], sample_size, replace=True)]
sampled_X = data[:, :data.shape[1] - 1]
sampled_y = data[:, data.shape[1] - 1]
if isinstance(self, RegressionForest):
tree = RegressionTree(
max_depth=self.max_depth,
min_size=self.min_size,
in_forest=True)
else:
tree = ClassificationTree(
cost_func=self.cost_func,
max_depth=self.max_depth,
min_size=self.min_size,
in_forest=True)
tree.fit(sampled_X, sampled_y)
self.forest[i] = tree
def prune(self, min_size=1):
#Prima controllo se ci sono sottoalberi puri.
#Visito l'albero e verifico se esistono nodi branch ai quali arrivano punti associati a solo una label
#Poi vedo se il nodo attuale è morto
T = self.classification_tree
stack = [T.tree[0]]
while (stack):
actual = stack.pop()
if len(actual.data_idxs) > 0:
#Se il nodo è puro
if not actual.is_leaf and all(
i == self.y[actual.data_idxs[0]]
for i in self.y[actual.data_idxs]):
#Devo far diventare una foglia questo nodo con valore pari alla label
actual.is_leaf = True
actual.left_node = None
actual.right_node = None
actual.left_node_id = -1
actual.right_node_id = -1
actual.value = self.y[actual.data_idxs[0]]
#Se il nodo ha un figlio morto devo sostituire il padre con l'altro figlio
elif not actual.is_leaf and len(
actual.left_node.data_idxs) < min_size:
stack.append(actual.right_node)
ClassificationTree.replace_node(actual, actual.right_node,
T)
elif not actual.is_leaf and len(
actual.right_node.data_idxs) < min_size:
stack.append(actual.left_node)
ClassificationTree.replace_node(actual, actual.left_node,
T)
elif not actual.is_leaf:
stack.append(actual.right_node)
stack.append(actual.left_node)
ClassificationTree.restore_tree(T)
y_train = label[0:valid_id]
X_valid = data[valid_id:]
y_valid = label[valid_id:]
'''
X_train, X_valid, y_train, y_valid = train_test_split(data,
label,
stratify=label,
test_size=0.2)
clf = DecisionTreeClassifier(random_state=0,
max_depth=3,
min_samples_leaf=10)
clf.fit(X_train, y_train)
T = ClassificationTree(oblique=False)
T.initialize_from_CART(X_train, y_train, clf)
T.compute_prob(X_train, y_train)
cart_auc_train += T.auc(X_train, y_train)
cart_auc_valid += T.auc(X_valid, y_valid)
#tao_train_score+=1-T.misclassification_loss(X_train, y_train, T.tree[0])
#print ("score before: ", tao_train_score)
#x = data[8]
#print (T.predict_label(x.reshape((1, -1)), 0))
#print (clf.predict(x.reshape((1, -1))))
#print ("x--->", x)
#print(T.get_path_to(x, 0))
#T.print_tree_structure()
#print ("T acc -> ", 1-T.misclassification_loss(data, label, T.tree[0]))
#print ("clf acc -> ", clf.score(data, label))
def best_parallel_split(self, node, X, y, alfa, complexity):
T = self.classification_tree
#Creo un nuovo sottoalbero fittizio con root nel nodo
new_node = TreeNode.copy_node(node)
error_best = np.inf
#if new_node.is_leaf:
#complexity += 1
was_leaf = False
improve = False
rho = 0
if new_node.is_leaf:
was_leaf = True
rho = 1
if new_node.data_idxs:
for j in range(len(X[0])):
#Prendo tutte le j-esime componenti del dataset e le ordino
#in modo crescente
vals = {}
for point_idx in new_node.data_idxs:
vals[point_idx] = X[point_idx, j]
values = sorted(vals.items(), key=lambda x: x[1])
sorted_indexes = [tuple[0] for tuple in values]
#plt.scatter(X[sorted_indexes, j], range(len(values)), s=0.4, c=list(correct_classification_tuples.values()))
#plt.show()
new_node.feature = j
#if j==2:
#base = actual_loss
#actual_loss = self.binary_loss(node_id, X, y, sorted_indexes[i], correct_classification_tuples[sorted_indexes[i]], actual_loss, thresh)
#print ("loss: ", actual_loss, "n punti: ", len(care_points_idxs))
#print("vecchia: ", self.vecchia_loss(node_id, X, y, care_points_idxs, correct_classification_tuples, thresh))
#Ciclo su ogni valore di quella componente e provo tutti gli split
#possibili
'''
new_node.threshold = 0.5*X[sorted_indexes[0], j]
'''
i = -1
actual_loss = ClassificationTree.misclassification_loss(
new_node, X, y, sorted_indexes, self.classification_tree.
oblique) + alfa * (complexity + rho)
while i < len(sorted_indexes):
pre_thresh = new_node.threshold
#print("Ottimizzo best parallel: ", i*100/len(sorted_indexes))
if i < 0:
thresh = 0.5 * X[sorted_indexes[0], j]
if i < len(sorted_indexes) - 1:
thresh = 0.5 * (X[sorted_indexes[i], j] +
X[sorted_indexes[i + 1], j])
else:
thresh = 1.5 * X[sorted_indexes[i], j]
new_node.threshold = thresh
'''
#Se il nodo da ottimizzare era una foglia allora dobbiamo creare le foglie figlie
#queste vengono ottimizzate subito per maggioranza
if was_leaf:
self.create_new_children(new_node, j, thresh)
inc, k = self.binary_loss(X, new_node, sorted_indexes, i, y, pre_thresh)
actual_loss += inc
else:
inc, k = self.binary_loss(X, new_node, sorted_indexes, i, y, pre_thresh)
actual_loss += inc
'''
#Se il nodo da ottimizzare era una foglia allora dobbiamo creare le foglie figlie
#queste vengono ottimizzate subito per maggioranza
if was_leaf:
ClassificationTree.create_new_children(
new_node,
X,
y,
self.max_id,
j,
thresh,
oblique=T.oblique)
actual_loss = ClassificationTree.misclassification_loss(
new_node, X, y, sorted_indexes,
self.classification_tree.oblique) + alfa * (
complexity + rho)
if actual_loss < error_best:
improve = True
error_best = actual_loss
best_t = thresh
best_feature = j
i += 1
#print ("error best: ", error_best)
new_node.threshold = best_t
new_node.feature = best_feature
if was_leaf and improve:
self.max_id += 2
return new_node, error_best
def regularized_loss(node, X, labels, X_valid, y_valid, idxs, oblique, n_classes = 2, l = 0.9):
#print(len(node.data_idxs))
return ClassificationTree.misclassification_loss(node, X, labels, idxs, oblique) + l*ClassificationTree.misclassification_loss(node, X_valid, y_valid, range(len(y_valid)), oblique)#+ l*gini(node, X, labels, idxs, oblique, n_classes)
def test(n_runs, ls_train, ls_test, svm_train, svm_test, random_train, random_test, cart_train, tao_train, global_train, cart_test, tao_test, global_test):
for run in range(n_runs):
depth = 3
oblique = False
n_trees = 200
n_iter = 5
data = np.load('cancer_train.npy')
y = np.load('cancer_label.npy')
print ("Run -> ", run)
idx = np.random.permutation(len(data))
data = data[idx]
y = y[idx]
train_split = 0.50
valid_split = 0.75
#data = dataset.data[idx]
#label = dataset.target[idx]
train_id = int(len(data)*train_split)
valid_id = int(len(data)*valid_split)
X = data[0:train_id]
labels = y[0:train_id]
X_valid = data[train_id:valid_id]
y_valid = y[train_id:valid_id]
X_test = data[valid_id:]
y_test = y[valid_id:]
#CART
clf = DecisionTreeClassifier(random_state=0, max_depth=depth, min_samples_leaf=4)
clf.fit(X, labels)
#TAO
T = ClassificationTree(oblique = oblique)
T.initialize_from_CART(X, labels, clf)
tao = TAO(T)
tao.evolve(X, labels)
T.print_tree_structure()
#LS
'''
L = ClassificationTree(oblique = oblique)
L.initialize_from_CART(X, labels, clf)
ls = LocalSearch(L)
ls.evolve(X, labels, alfa=1000000, max_iteration=10)
'''
#SVM
svm = LinearSVC(tol=1e-6, max_iter=10000, dual=False)
svm.fit(X, labels)
#RandomForest
random_for = RandomForestClassifier(n_estimators = n_trees, max_depth=depth, random_state=0, min_samples_leaf= 4)
random_for.fit(X, labels)
#Genetic
best_t, best_loss = genetic_tree_optimization(n_trees, n_iter, depth, X, labels, oblique, X_valid, y_valid, CR = 0, l = 0)
#best_t.print_tree_structure()
best_t.print_tree_structure()
#Train Score
cart_train.append(clf.score(X, labels))
#ls_train.append(1-ClassificationTree.misclassification_loss(L.tree[0], X, labels, range(len(labels)), oblique))
tao_train.append(1-ClassificationTree.misclassification_loss(T.tree[0], X, labels, range(len(labels)), oblique))
global_train.append(1-ClassificationTree.misclassification_loss(best_t.tree[0], X, labels, range(len(labels)), oblique))
svm_train.append(svm.score(X, labels))
random_train.append(random_for.score(X, labels))
#Test Score
cart_test.append(clf.score(X_test, y_test))
#ls_test.append(1-ClassificationTree.misclassification_loss(L.tree[0], X_test, y_test, range(len(y_test)), oblique))
tao_test.append(1-ClassificationTree.misclassification_loss(T.tree[0], X_test, y_test, range(len(y_test)), oblique))
global_test.append(1-ClassificationTree.misclassification_loss(best_t.tree[0], X_test, y_test, range(len(y_test)), oblique))
svm_test.append(svm.score(X_test, y_test))
random_test.append(random_for.score(X_test, y_test))
def TAO_best_parallel_split(self, node_id, X, y, care_points_idxs,
correct_classification_tuples, T):
#print ("node id:", node_id)
error_best = np.inf
best_t = T.tree[node_id].threshold
best_feature = T.tree[node_id].feature
if self.train_on_all_features:
features = range(len(X[0]))
else:
features = ClassificationTree.get_features(T)
for j in features:
#print(j)
#Prendo tutte le j-esime componenti del dataset e le ordino
#in modo crescente
vals = {}
for point_idx in care_points_idxs:
vals[point_idx] = X[point_idx, j]
values = sorted(vals.items(), key=lambda x: x[1])
sorted_indexes = [tuple[0] for tuple in values]
#plt.scatter(X[sorted_indexes, j], range(len(values)), s=0.4, c=list(correct_classification_tuples.values()))
#plt.show()
T.tree[node_id].feature = j
thresh = 0.5 * (X[sorted_indexes[0], j] + X[sorted_indexes[1], j])
actual_loss, start = self.zero_one_loss(
node_id, X, y, sorted_indexes, correct_classification_tuples,
thresh, sorted_indexes)
#if j==2:
#base = actual_loss
#actual_loss = self.binary_loss(node_id, X, y, sorted_indexes[i], correct_classification_tuples[sorted_indexes[i]], actual_loss, thresh)
#print ("loss: ", actual_loss, "n punti: ", len(care_points_idxs))
#print("vecchia: ", self.vecchia_loss(node_id, X, y, care_points_idxs, correct_classification_tuples, thresh))
#Ciclo su ogni valore di quella componente e provo tutti gli split
#possibili
if actual_loss < error_best:
error_best = actual_loss
best_t = thresh
best_feature = j
i = start
while i < len(sorted_indexes) - 1:
thresh = 0.5 * (X[sorted_indexes[i], j] +
X[sorted_indexes[i + 1], j])
#sum, k = self.binary_loss(X, j, sorted_indexes, i, correct_classification_tuples)
#actual_loss += sum
#Qualche print per debug
'''
if j==2 and node_id==3:
ones = [correct_classification_tuples[k] for k in sorted_indexes]
print("base: ", base, " n punti:", len(sorted_indexes), " loss:", actual_loss, " thresh:", thresh, " x:", X[sorted_indexes[i], j], " prediction: ", correct_classification_tuples[sorted_indexes[i]])
print(X[sorted_indexes, j])
print()
'''
#actual_loss = self.zero_one_loss(node_id, X, y, care_points_idxs, correct_classification_tuples, thresh)
s, k = self.binary_loss(X, j, sorted_indexes, i,
correct_classification_tuples)
#print ("dopo binary")
actual_loss += s
if actual_loss < error_best:
error_best = actual_loss
best_t = thresh
best_feature = j
i += k
'''
#print(error_best)
#errors = [self.binary_loss(node_id, X, y, care_points_idxs, correct_classification_tuples, threshes[i]) for i in range(len(values)-1)]
#min_index = np.argmin(errors)
'''
#Ancora print per debug
#if node_id==1:
#print("finale: ", error_best)
#print ("ones:", ones, " len: ", len(ones), " care_p_len:", len(care_points_idxs))
#print("best_feature:", best_feature, " best_thresh:", best_t)
return best_t, best_feature
"node %s." % (
actual_node.depth * "\t",
actual_node.id,
actual_node.parent_id,
actual_node.left_node_id,
actual_node.feature,
actual_node.threshold,
actual_node.right_node_id,
))
stack.append(actual_node.left_node)
stack.append(actual_node.right_node)
spear_train += 1 - zero_one_loss(node, X, labels) / len(labels)
spear_valid += 1 - zero_one_loss(node, X_valid, y_valid) / len(y_valid)
clf = DecisionTreeClassifier(random_state=0,
max_depth=3,
min_samples_leaf=4)
clf.fit(X, labels)
clf_train += clf.score(X, labels)
clf_valid += clf.score(X_valid, y_valid)
L = ClassificationTree(oblique=False)
L.initialize_from_CART(X, labels, clf)
L.print_tree_structure()
print("clf train: ", clf_train / 30)
print("spearman train: ", spear_train / 30)
print("clf valid: ", clf_valid / 30)
print("spearman valid: ", spear_valid / 30)