def AdaBoost_print(S_train, S_test, Attributes, T):
trees = []
alphas = []
f = open("adaboost_out2.txt", "w")
for _ in range(0, T):
tree = id3_with_weight(S_train, Attributes, 1, 0)
trees.append(tree)
norm = 0.0
epsilon = id3_weighted_err(tree, S_train)
f.write(str(_) + "\t" + str(epsilon) + "\t" + str(id3_weighted_err(tree, S_test)) + "\n")
alpha = .5 * math.log((1 - epsilon)/epsilon)
alphas.append(alpha)
for s in S_train:
label = get_label(s, tree)
if label != s["Label"]:
newWeight = s["Weight"] * math.exp(alpha)
s["Weight"] = newWeight
else:
newWeight = s["Weight"] * math.exp(-alpha)
s["Weight"] = newWeight
norm += newWeight
# normalize values
for s in S_train:
s["Weight"] /= norm
return (trees, alphas)
def AdaBoost(S, Attributes, T):
trees = []
alphas = []
for _ in range(0, T):
tree = id3_with_weight(S, Attributes, 1, 0)
trees.append(tree)
norm = 0.0
epsilon = id3_weighted_err(tree, S)
alpha = .5 * math.log((1 - epsilon)/epsilon)
alphas.append(alpha)
for s in S:
label = get_label(s, tree)
if label != s["Label"]:
newWeight = s["Weight"] * math.exp(alpha)
s["Weight"] = newWeight
else:
newWeight = s["Weight"] * math.exp(-alpha)
s["Weight"] = newWeight
norm += newWeight
# normalize values
for s in S:
s["Weight"] /= norm
return (trees, alphas)
def Bagging_Train(S, Attributes, T):
M = len(S) / 2
predictions = []
weights = []
for _ in range(0, T):
new_S = [random.choice(S) for __ in range(0, M)]
tree = id3_with_weight(new_S, Attributes, None, 0)
predictions.append(tree)
return predictions
# random forest
f.write("Iter\tTrain\tTest\n")
for T in range(1, 1050, 50):
hypothesis = Random_Forest_Train(S_train, Attributes, T, 4)
err_train = Random_Forest_Test(hypothesis, S_train)
err_test = Random_Forest_Test(hypothesis, S_test)
print "T = " + str(T - 1) + ": " + str(err_train) + ", " + str(err_test)
f.write(str(T - 1) + "\t" + str(err_train) + "\t" + str(err_test) + "\n")
f.write("\n\n")
# adaboost
f.write("Iter\tTrain\tTest\n")
for T in range(1, 1050, 50):
hypothesis = AdaBoost(S_train, Attributes, T)
err_train = AdaBoost_Test(hypothesis, S_train)
err_test = AdaBoost_Test(hypothesis, S_test)
print str(T - 1) + "\t" + str(err_train) + "\t" + str(err_test)
f.write(str(T - 1) + "\t" + str(err_train) + "\t" + str(err_test) + "\n")
reset_weights(S_train)
f.write("\n\n")
# fully expanded tree
reset_weights(S_train)
reset_weights(S_test)
tree = id3_with_weight(S_train, Attributes, None, 0)
err_train = id3_weighted_err(tree, S_train)
err_test = id3_weighted_err(tree, S_test)
print "T = " + str(T - 1) + "\t" + str(err_train) + "\t" + str(err_test)
f.write(str(T - 1) + "\t" + str(err_train) + "\t" + str(err_test) + "\n")
from id3 import id3_with_weight, render
### MAIN ###
attrList = ["1", "2", "3", "4", "Label"]
S = []
with open("logic_in.txt") as f:
Attributes = {}
line = f.readline()
while line != "\n":
splitLine = line.split(':')
attr = splitLine[0]
Attributes[attr] = "".join(splitLine[1].split()).split(',')
line = f.readline()
for line in f:
i = 0
example = {}
for attr in line.strip().split(' '):
example[attrList[i]] = attr
i += 1
example["Weight"] = 1
S.append(example)
tree = id3_with_weight(S, Attributes, None, 0)
render(tree, "4e_tree")