def learn_rule(self, domain, data, pos_uncovered_indices, neg_indices):
# Create a new model
m = milp.Model("kDNF")
print("Data")
for row, l in data:
print(*([
"{}: {:.2f}".format(v, Learner._convert(row[v]))
for v in domain.variables
] + [l]))
print()
# --- Computed constants
n_r = len(domain.real_vars)
n_b = len(domain.bool_vars)
n_h = self.max_hyperplanes_per_rule
n_e = len(data)
k = self.max_terms_per_rule
mu = 0.005
x_er = [[Learner._convert(row[v]) for v in domain.real_vars]
for row, _ in data]
x_eb = [[Learner._convert(row[v]) for v in domain.bool_vars]
for row, _ in data]
# --- Variables
# Inequality h is selected for the rule
s_h = [
m.addVar(vtype=milp.GRB.BINARY, name="s_h({h})".format(h=h))
for h in range(n_h)
]
# Example e covered by inequality h
c_eh = [[
m.addVar(vtype=milp.GRB.BINARY,
name="c_eh({e}, {h})".format(e=e, h=h))
for h in range(n_h)
] for e in range(n_e)]
# Example e covered by rule
c_e = [
m.addVar(vtype=milp.GRB.BINARY, name="c_e({e})".format(e=e))
for e in range(n_e)
]
# Boolean feature (or negation) selected
s_b = [
m.addVar(vtype=milp.GRB.BINARY, name="s_b({b})".format(b=b))
for b in range(2 * n_b)
]
# Coefficient of the r-th variable of inequality h
a_hr = [[
m.addVar(vtype=milp.GRB.CONTINUOUS,
lb=-1,
ub=1,
name="a_hr({h}, {r})".format(h=h, r=r))
for r in range(n_r)
] for h in range(n_h)]
# Offset of inequality h
b_h = [
m.addVar(vtype=milp.GRB.CONTINUOUS,
lb=-1,
ub=1,
name="b_h({h})".format(h=h)) for h in range(n_h)
]
b_abs_h = [
m.addVar(vtype=milp.GRB.CONTINUOUS,
lb=-1,
ub=1,
name="b_abs_h({h})".format(h=h)) for h in range(n_h)
]
# Auxiliary variable (c_eh AND s_h)
and_eh = [[
m.addVar(vtype=milp.GRB.BINARY,
name="and_eh({e}, {h})".format(e=e, h=h))
for h in range(n_h)
] for e in range(n_e)]
# Auxiliary variable (x_eb AND s_b)
and_eb = [[
m.addVar(vtype=milp.GRB.BINARY,
name="and_eb({e}, {b})".format(e=e, b=b))
for b in range(n_b)
] for e in range(n_e)]
print()
# Set objective
m.setObjective(sum(c_e[e] for e in range(n_e) if data[e][1]),
milp.GRB.MAXIMIZE)
# Add constraint: and_eh <= c_eh
for e in range(n_e):
if not data[e][1]:
for h in range(n_h):
name = "and_eh({e}, {h}) <= c_eh({e}, {h})".format(e=e,
h=h)
m.addConstr(and_eh[e][h] <= c_eh[e][h], name)
print(name)
print()
# Add constraint: and_eh <= s_h
for e in range(n_e):
if not data[e][1]:
for h in range(n_h):
name = "and_eh({e}, {h}) <= s_h({h})".format(e=e, h=h)
m.addConstr(and_eh[e][h] <= s_h[h], name)
print(name)
print()
# Add constraint: and_eh >= c_eh + s_h - 1
for e in range(n_e):
if not data[e][1]:
for h in range(n_h):
name = "and_eh({e}, {h}) >= c_eh({e}, {h}) + s_h({h}) - 1".format(
e=e, h=h)
m.addConstr(and_eh[e][h] >= c_eh[e][h] + s_h[h] - 1, name)
print(name)
print()
# Add constraint: and_eb <= x_eb
for e in range(n_e):
if not data[e][1]:
for b in range(2 * n_b):
name = "and_eb({e}, {b}) <= x_eb({e}, {b})".format(e=e,
b=b)
m.addConstr(
and_eb[e][b] <=
(x_eb[e][b] if b < n_b else 1 - x_eb[e][b]), name)
print(name)
print()
# Add constraint: and_eb <= s_h
for e in range(n_e):
if not data[e][1]:
for b in range(2 * n_b):
name = "and_eb({e}, {b}) <= s_b({b})".format(e=e, b=b)
m.addConstr(and_eb[e][b] <= s_b[b], name)
print(name)
print()
# Add constraint: and_eb >= x_eb + s_b - 1
for e in range(n_e):
if not data[e][1]:
for b in range(2 * n_b):
name = "and_eb({e}, {b}) >= x_eb({e}, {b}) + s_b({b}) - 1".format(
e=e, b=b)
m.addConstr(
and_eb[e][b] >=
(x_eb[e][b] if b < n_b else 1 - x_eb[e][b]) + s_b[b] -
1, name)
print(name)
print()
# Add constraint: SUM_(h = 1..n_h) and_eh - s_h + SUM_(b = 1..2 * n_b) and_eb - s_b <= -1
for e in range(n_e):
if not data[e][1]:
name = "SUM_(h = 1..n_h) and_eh({e}, h) - s_h(h) + SUM_(b = 1..2*n_b) and_eb({e}, b) - s_b(b) <= -1"\
.format(e=e)
m.addConstr(
sum(and_eh[e][h] - s_h[h]
for h in range(n_h)) + sum(and_eb[e][b] - s_b[b]
for b in range(2 * n_b)) <=
-1, name)
print(name)
# Add constraint: c_e <= c_eh + (1 - s_h)
for e in range(n_e):
if data[e][1]:
for h in range(n_h):
name = "c_e({e}) <= c_eh({e}, {h}) + (1 - s_h({h}))".format(
e=e, h=h)
m.addConstr(c_e[e] <= c_eh[e][h] + (1 - s_h[h]), name)
print(name)
# Add constraint: c_e <= x_eb + (1 - s_b)
for e in range(n_e):
if data[e][1]:
for b in range(2 * n_b):
name = "c_e({e}) <= x_eb({e}, {b}) + (1 - s_b({b}))".format(
e=e, b=b)
m.addConstr(c_e[e] <= x_eb[e][b] + (1 - s_b[b]), name)
print(name)
# Add constraint: SUM_(r = 1..n_r) a_hr * x_er <= b_h + 2 * (1 - c_eh)
for e in range(n_e):
for h in range(n_h):
name = "SUM_(r = 1..n_r) a_hr({h}, r) * x_er({e}, r) <= b_h({h}) + 2 * (1 - c_eh({e}, {h}))"\
.format(e=e, h=h)
m.addConstr(
sum(a_hr[h][r] * x_er[e][r]
for r in range(n_r)) <= b_h[h] + 2 * (1 - c_eh[e][h]),
name)
print(name)
# Add constraint: SUM_(r = 1..n_r) a_hr * x_er >= b_h - mu - 2 * c_eh
for e in range(n_e):
for h in range(n_h):
name = "SUM_(r = 1..n_r) a_hr({h}, r) * x_er({e}, r) >= b_h({h}) - mu - 2 * c_eh({e}, {h})"\
.format(e=e, h=h)
m.addConstr(
sum(a_hr[h][r] * x_er[e][r]
for r in range(n_r)) >= b_h[h] + mu - 2 * c_eh[e][h],
name)
print(name)
# Add constraint: SUM_(h = 1..n_h) s_h + SUM_(b = 1..n_b) s_b <= k
name = "SUM_(h = 1..n_h) s_h(h) + SUM_(b = 1..n_b) s_b(b) <= k"
m.addConstr(
sum(s_h[h]
for h in range(n_h)) + sum(s_b[b]
for b in range(2 * n_b)) <= k, name)
print(name)
# Hack example
# m.addConstr(a[0][0] == 1)
# m.addConstr(a[0][1] == 0)
# m.addConstr(b[0] == 0.5)
#
# m.addConstr(a[1][0] == 0)
# m.addConstr(a[1][1] == 1)
# m.addConstr(b[1] == 0.5)
#
# m.addConstr(a[2][0] == -1)
# m.addConstr(a[2][1] == 0)
# m.addConstr(b[2] == -0.5)
#
# m.addConstr(a[3][0] == 0)
# m.addConstr(a[3][1] == -1)
# m.addConstr(b[3] == -0.5)
#
# m.addConstr(s_h[0] == 1)
# m.addConstr(s_h[1] == 1)
# m.addConstr(s_h[2] == 1)
# m.addConstr(s_h[3] == 1)
# m.addConstr(z_h[0][0] == 1)
# m.addConstr(z_h[1][0] == 1)
# m.addConstr(z_h[2][0] == 0)
# m.addConstr(z_h[3][0] == 0)
# m.addConstr(z_h[0][1] == 0)
# m.addConstr(z_h[1][1] == 0)
# m.addConstr(z_h[2][1] == 1)
# m.addConstr(z_h[3][1] == 1)
# m.addConstr(z_ih[0][0] == 1)
# m.addConstr(z_ih[1][0] == 0)
# m.addConstr(z_ih[2][0] == 1)
# m.addConstr(z_ih[3][0] == 0)
# m.addConstr(z_ih[0][1] == 1)
# m.addConstr(z_ih[1][1] == 0)
# m.addConstr(z_ih[2][1] == 0)
# m.addConstr(z_ih[3][1] == 1)
# m.addConstr(z_ih[0][2] == 0)
# m.addConstr(z_ih[1][2] == 1)
# m.addConstr(z_ih[2][2] == 0)
# m.addConstr(z_ih[3][2] == 1)
# m.addConstr(z_ih[0][3] == 0)
# m.addConstr(z_ih[1][3] == 1)
# m.addConstr(z_ih[2][3] == 1)
# m.addConstr(z_ih[3][3] == 0)
# m.addConstr(z_d[0][0] == 1)
# m.addConstr(z_d[1][0] == 0)
# m.addConstr(z_d[2][0] == 0)
# m.addConstr(z_d[3][0] == 0)
# m.addConstr(z_d[0][1] == 0)
# m.addConstr(z_d[1][1] == 0)
# m.addConstr(z_d[2][1] == 1)
# m.addConstr(z_d[3][1] == 0)
# Wrong solution
# m.addConstr(a[0][0] == 1)
# m.addConstr(a[0][1] == 0)
# m.addConstr(b[0] == 0.5)
#
# m.addConstr(a[1][0] == 0)
# m.addConstr(a[1][1] == 1)
# m.addConstr(b[1] == 0.5)
#
# m.addConstr(a[2][0] == -1)
# m.addConstr(a[2][1] == 0)
# m.addConstr(b[2] == -0.6)
#
# m.addConstr(a[3][0] == 0)
# m.addConstr(a[3][1] == -1)
# m.addConstr(b[3] == -0.6)
#
# m.addConstr(s_h[0] == 1)
# m.addConstr(s_h[1] == 1)
# m.addConstr(s_h[2] == 1)
# m.addConstr(s_h[3] == 1)
# m.addConstr(s_h[0] == 1)
# m.addConstr(s_h[1] == 1)
# m.addConstr(s_h[2] == 0)
# m.addConstr(s_h[3] == 0)
#
# m.addConstr(a_hr[0][0] == -1)
# m.addConstr(a_hr[0][1] == -0.5)
# m.addConstr(b_h[0] == -0.2)
#
# m.addConstr(a_hr[1][0] == 1)
# m.addConstr(a_hr[1][1] == 0)
# m.addConstr(b_h[1] == 0.75)
m.optimize()
for v in m.getVars():
print(v.varName, v.x)
print('Obj:', m.objVal)
print()
# Lets extract the rule formulation
rule = []
from pysmt.shortcuts import Real, LE, Plus, Times, Not
for h in range(n_h):
if int(s_h[h].x) == 1:
coefficients = [Real(float(a_hr[h][r].x)) for r in range(n_r)]
constant = Real(float(b_h[h].x))
linear_sum = Plus([
Times(c, domain.get_symbol(v))
for c, v in zip(coefficients, domain.real_vars)
])
rule.append(LE(linear_sum, constant))
for b in range(2 * n_b):
if int(s_b[b].x) == 1:
var = domain.get_symbol(domain.bool_vars[b if b < n_b else b -
n_b])
rule.append(var if b < n_b else Not(var))
print("Features", x_er[4][0], x_er[4][1])
print(sum(a_hr[1][r].x * x_er[4][r] for r in range(n_r)), "<=",
b_h[1].x + 2 * (1 - c_eh[4][1].x))
print(sum(a_hr[1][r].x * x_er[4][r] for r in range(n_r)), ">=",
b_h[1].x + mu - 2 * c_eh[4][1].x)
return rule
def learn(self, domain, data):
# Create a new model
m = Model("OCT")
print("Data")
for row, l in data:
print(*(["{}: {:.2f}".format(v, Learner._convert(row[v])) for v in domain.variables] + [l]))
print()
# Computed constants
L_hat = OCTLearner._get_misclassification(data)
print("Majority label misclassification cost", L_hat)
p = len(domain.variables)
K = 2
n = len(data)
leaf_count = 2 ** self.D
inline_count = leaf_count - 1
T = list(range(leaf_count + inline_count))
# T_B = [t for t in T if t < inline_count]
# T_L = [t for t in T if t >= inline_count]
print("Nodes: {} internal, {} leaves".format(inline_count, leaf_count))
Y_ik = [
[1 if data[i][1] else -1, -1 if data[i][1] else 1]
for i in range(n)
]
A_L = [[] for _ in range(inline_count + leaf_count)] # TODO
A_R = [[] for _ in range(inline_count + leaf_count)] # TODO
def populate_ancestors(index):
left = (index + 1) * 2 - 1
right = (index + 1) * 2
if right < inline_count + leaf_count:
A_L[left] = A_L[index] + [index]
A_R[left] = A_R[index]
A_L[right] = A_L[index]
A_R[right] = A_R[index] + [index]
populate_ancestors(left)
populate_ancestors(right)
populate_ancestors(0)
print("A_L: {}, A_R: {}".format(A_L, A_R))
x_ij = [[Learner._convert(row[v]) for v in domain.variables] for row, _ in data]
p_t = [None if i == 0 else int((i + 1) / 2) - 1 for i in range(inline_count)]
# Variables
# x = m.addVar(vtype=GRB.BINARY, name="x")
L_t = [m.addVar(vtype=GRB.INTEGER, name="L_{}".format(i)) for i in range(leaf_count)]
s_jt = [
[m.addVar(vtype=GRB.BINARY, name="s{}_{}".format(j, i)) for i in range(inline_count)]
for j in range(p)
]
N_t = [m.addVar(vtype=GRB.INTEGER, name="N_{}".format(i)) for i in range(leaf_count)]
N_kt = [
[m.addVar(vtype=GRB.INTEGER, name="N_{}_{}".format(k, i)) for i in range(leaf_count)]
for k in range(K)
]
c_kt = [
[m.addVar(vtype=GRB.BINARY, name="c_{}_{}".format(k, i)) for i in range(leaf_count)]
for k in range(K)
]
z_it = [
[m.addVar(vtype=GRB.BINARY, name="z_{}_{}".format(i, t)) for t in range(leaf_count)]
for i in range(n)
]
l_t = [m.addVar(vtype=GRB.BINARY, name="l_{}".format(t)) for t in range(leaf_count)]
a_jt = [
[m.addVar(vtype=GRB.CONTINUOUS, lb=-1, ub=1, name="a_{}_{}".format(j, t)) for t in range(inline_count)]
for j in range(p)
]
a_hat_jt = [
[m.addVar(vtype=GRB.CONTINUOUS, name="a_hat_{}_{}".format(j, t)) for t in range(inline_count)]
for j in range(p)
]
b_t = [m.addVar(vtype=GRB.CONTINUOUS, name="b_{}".format(t)) for t in range(inline_count)]
d_t = [m.addVar(vtype=GRB.BINARY, name="d_{}".format(t)) for t in range(inline_count)]
print()
# Set objective
m.setObjective(1.0 / L_hat * sum(L_t) + self.alpha * sum(sum(entry) for entry in s_jt), GRB.MINIMIZE)
# Add constraint: L_t >= N_t - N_kt - n(1 - c_kt) forall k = 1..K, t in T_L
for k in range(K):
for t in range(leaf_count):
name = "L_{t} >= N_{t} - N_{k}_{t} - n(1 - c_{k}_{t})".format(k=k, t=t)
m.addConstr(L_t[t] >= N_t[t] - N_kt[k][t] - n * (1 - c_kt[k][t]), name)
print(name)
print()
# Add constraint: L_t <= N_t - N_kt + n * c_kt forall k = 1..K, t in T_L
for k in range(K):
for t in range(leaf_count):
name = "L_{t} <= N_{t} - N_{k}_{t} + n * c_{k}_{t}".format(k=k, t=t)
m.addConstr(L_t[t] <= N_t[t] - N_kt[k][t] + n * c_kt[k][t], name)
print(name)
print()
# Add constraint: L_t >= 0 forall t in T_l
for t in range(leaf_count):
name = "L_{t} >= 0".format(t=t)
m.addConstr(L_t[t] >= 0, name)
print(name)
print()
# Add constraint: N_kt = 1/2 * Sum_i = 1..n (1 + Y_ik) * z_it forall k = 1..K, t in T_L
for k in range(K):
for t in range(leaf_count):
name = "N_{k}_{t} = 1/2 * SUM_(i = 0..{n}) (1 + Y_i_{k}) * z_i_{t}".format(k=k, t=t, n=n-1)
m.addConstr(N_kt[k][t] == sum((1 + Y_ik[i][k]) * z_it[i][t] for i in range(n)) / 2, name)
print(name)
print()
# Add constraint: N_t = Sum_i = 1..n z_it forall t in T_L
for t in range(leaf_count):
name = "N_{t} = SUM_(i = 0..{n}) z_i_{t}".format(t=t, n=n-1)
m.addConstr(N_t[t] == sum(z_it[i][t] for i in range(n)), name)
print(name)
print()
# Add constraint: Sum_k = 1..K c_kt = l_t forall t in T_L
for t in range(leaf_count):
# print(sum([c_kt[k][t] for k in range(K)]))
name = "SUM_(k = 0..{K}) c_k_{t} = l_{t}".format(t=t, K=K-1)
m.addConstr(sum(c_kt[k][t] for k in range(K)) == l_t[t], name)
print(name)
print()
# Add constraint: Sum_j = 1..p (a_jm^T * x_ji) + mu <= b_m + (2 + mu) * (1 - z_it)
# forall i in 1..n, t in T_B, m in A_L(t)
for i in range(n):
for t in range(leaf_count):
for _m in A_L[inline_count + t]:
vector_product = sum(a_jt[j][_m] * x_ij[i][j] for j in range(p))
name = "SUM_(j = 0..{p}) (a_j_{m}^T * x_j_{i}) + mu <= b_{m} + (2 + mu) * (1 - z_{i}_{t})"\
.format(i=i, t=t, m=_m, p=p-1)
m.addConstr(vector_product + self.mu <= b_t[_m] + (2 + self.mu) * (1 - z_it[i][t]), name)
print(name)
print()
# TODO, both times 1 - z_it??
# Add constraint: Sum_j = 1..p a_jm^T * x_ji >= b_m - 2 * (1 - z_it) forall i in 1..n, t in T_B, m in A_R(t)
for i in range(n):
for t in range(leaf_count):
for _m in A_R[inline_count + t]:
name = "SUM_(j = 0..{p}) a_j_{m}^T * x_j_{i} >= b_{m} - 2 * (1 - z_{i}_{t})"\
.format(i=i, t=t, m=_m, p=p-1)
vector_product = sum(a_jt[j][_m] * x_ij[i][j] for j in range(p))
m.addConstr(vector_product >= b_t[_m] - 2 * (1 - z_it[i][t]), name)
print(name)
print()
# Add constraint: Sum_t in T_L z_it = 1 forall i = 1..n
for i in range(n):
name = "SUM_(t in {T_L}) z_{i}_t = 1".format(i=i, T_L=list(range(leaf_count)))
m.addConstr(sum(z_it[i][t] for t in range(leaf_count)) == 1, name)
print(name)
print()
# Add constraint: z_it <= l_t forall i = 1..n, t in T_L
for i in range(n):
for t in range(leaf_count):
name = "z_{i}_{t} <= l_{t}".format(i=i, t=t)
m.addConstr(z_it[i][t] <= l_t[t], name)
print(name)
print()
# Add constraint: Sum_i = 1..n z_it >= N_min * l_t forall t in T_L
for t in range(leaf_count):
name = "SUM_(i = 0..{n}) z_i_{t} >= N_min * l_{t}".format(t=t, n=n-1)
m.addConstr(sum(z_it[i][t] for i in range(n)) >= self.N_min * l_t[t], name)
print(name)
print()
# Add constraint: Sum_j = 1..p a_hat_jt <= d_t forall t in T_B
for t in range(inline_count):
name = "SUM_(j = 0..{p}) a_hat_j_{t} <= d_{t}".format(t=t, p=p-1)
m.addConstr(sum(a_hat_jt[j][t] for j in range(p)) <= d_t[t], name)
print(name)
print()
# Add constraint: a_hat_jt >= a_jt forall j = 1..p, t in T_B
for j in range(p):
for t in range(inline_count):
name = "a_hat_{j}_{t} >= a_{j}_{t}".format(j=j, t=t)
m.addConstr(a_hat_jt[j][t] >= a_jt[j][t], name)
print(name)
print()
# Add constraint: a_hat_jt >= -a_jt forall j = 1..p, t in T_B
for j in range(p):
for t in range(inline_count):
name = "a_hat_{j}_{t} >= -a_{j}_{t}".format(j=j, t=t)
m.addConstr(a_hat_jt[j][t] >= -a_jt[j][t], name)
print(name)
print()
# Add constraint: -s_jt <= a_jt <= s_jt forall j = 1..p, t in T_B
for j in range(p):
for t in range(inline_count):
name1 = "-s_{j}_{t} <= a_{j}_{t}".format(j=j, t=t)
m.addConstr(-s_jt[j][t] <= a_jt[j][t], name1)
name2 = "s_{j}_{t} >= a_{j}_{t}".format(j=j, t=t)
m.addConstr(s_jt[j][t] >= a_jt[j][t], name2)
print(name1)
print(name2)
print()
# Add constraint: s_jt <= d_t forall j = 1..p, t in T_B
for j in range(p):
for t in range(inline_count):
name = "s_{j}_{t} <= d_{t}".format(j=j, t=t)
m.addConstr(s_jt[j][t] <= d_t[t], name)
print(name)
print()
# Add constraint: Sum_j = 1..p s_jt >= d_t forall t in T_B
for t in range(inline_count):
name = "SUM_(j = 0..{p}) s_j_{t} >= d_{t}".format(t=t, p=p-1)
m.addConstr(sum(s_jt[j][t] for j in range(p)) >= d_t[t], name)
print(name)
print()
# Add constraint: -d_t <= b_t <= d_t forall t in T_B
for t in range(inline_count):
name1 = "-d_{t} <= b_{t}".format(t=t)
m.addConstr(-d_t[t] <= b_t[t], name1)
name2 = "d_{t} >= b_{t}".format(t=t)
m.addConstr(d_t[t] >= b_t[t], name2)
print(name1)
print(name2)
print()
# Add constraint: d_t <= d_p(t) forall t in T_B \ {1}
for t in range(1, inline_count):
name = "d_{t} <= d_{p_t}".format(t=t, p_t=p_t[t])
m.addConstr(d_t[t] <= d_t[p_t[t]], name)
print(name)
print()
# Hack example
# m.addConstr(N_t[0] == 2)
# m.addConstr(N_t[1] == 1)
# m.addConstr(z_it[0][0] == 1)
# m.addConstr(z_it[1][0] == 1)
# m.addConstr(z_it[2][0] == 0)
# m.addConstr(z_it[0][1] == 0)
# m.addConstr(z_it[1][1] == 0)
# m.addConstr(z_it[2][1] == 1)
# m.addConstr(c_kt[0][0] == 1)
# m.addConstr(c_kt[1][0] == 0)
# m.addConstr(c_kt[0][1] == 0)
# m.addConstr(c_kt[1][1] == 1)
# m.addConstr(a_jt[0][0] == 1)
# m.addConstr(b_t[0] == 0.6)
m.optimize()
for v in m.getVars():
print(v.varName, v.x)
print('Obj:', m.objVal)
paths = []
def populate_paths(index, prefix=None):
if prefix is None:
prefix = []
left = (index + 1) * 2 - 1
right = (index + 1) * 2
if right < inline_count and d_t[index].x > 0.5:
coefficients = [Real(float(a_jt[j][index].x)) for j in range(p)]
linear_sum = Plus([Times(c, domain.get_symbol(v)) for c, v in zip(coefficients, domain.variables)])
constant = Real(float(b_t[index].x))
node = prefix + [LE(linear_sum, constant)]
populate_paths(left, node)
populate_paths(right, node)
else:
paths.append(prefix)
populate_paths(0)
return paths
def learn(self, domain, data):
# Create a new model
m = Model("kDNF")
print("Data")
for row, l in data:
print(*(["{}: {:.2f}".format(v, Learner._convert(row[v])) for v in domain.variables] + [l]))
print()
# Computed constants
n_f = len(domain.variables)
n_h = (2 * self.max_hyperplanes) if self.negated_hyperplanes else self.max_hyperplanes
n_d = len(data)
k = self.k
naive_misclassification = float(Learner._get_misclassification(data))
x = [[Learner._convert(row[v]) for v in domain.variables] for row, _ in data]
# Variables
s_h = [m.addVar(vtype=GRB.BINARY, name="s_H({h})".format(h=h)) for h in range(n_h)]
s_d = [m.addVar(vtype=GRB.BINARY, name="s_D({i})".format(i=i)) for i in range(n_d)]
z_h = [
[m.addVar(vtype=GRB.BINARY, name="z_H({h}, {c})".format(h=h, c=c)) for c in range(k)]
for h in range(n_h)
]
z_d = [
[m.addVar(vtype=GRB.BINARY, name="z_D({i}, {c})".format(i=i, c=c)) for c in range(k)]
for i in range(n_d)
]
z_ih = [
[m.addVar(vtype=GRB.BINARY, name="z_IH({i}, {h})".format(i=i, h=h)) for h in range(n_h)]
for i in range(n_d)
]
z_ihc = [
[
[m.addVar(vtype=GRB.BINARY, name="z_IHC({i}, {h}, {c})".format(i=i, h=h, c=c)) for c in range(k)]
for h in range(n_h)
]
for i in range(n_d)
]
a = [
[m.addVar(vtype=GRB.CONTINUOUS, lb=-1, ub=1, name="a({h}, {j})".format(h=h, j=j)) for j in range(n_f)]
for h in range(n_h / 2 if self.negated_hyperplanes else n_h)
]
b = [m.addVar(vtype=GRB.CONTINUOUS, lb=-1, ub=1, name="b({h})".format(h=h))
for h in range(n_h / 2 if self.negated_hyperplanes else n_h)]
print()
# Set objective
misclassification = sum(1 - s_d[i] if data[i][1] else s_d[i] for i in range(n_d))
m.setObjective(misclassification / naive_misclassification + self.alpha * sum(s_h), GRB.MINIMIZE)
# Add constraint: SUM_(j = 1..n_h) a(h, j) * x(i, j) <= b(h) + M(1 - z_ih(i, h)) for all h, i
for h in range(n_h / 2 if self.negated_hyperplanes else n_h):
for i in range(n_d):
name = "SUM_(j = 1..n_H) a({h}, j) * x({i}, j) <= b({h}) + M(1 - z_IH({i}, {h}))".format(h=h, i=i)
m.addConstr(sum(a[h][j] * x[i][j] for j in range(n_f)) <= b[h] + (n_f + 1) * (1 - z_ih[i][h]), name)
print(name)
print()
for h in range(n_h / 2 if self.negated_hyperplanes else n_h):
for i in range(n_d):
name = "SUM_(j = 1..n_H) a({h}, j) * x({i}, j) >= b({h}) - mu - M * z_IH({i}, {h})".format(h=h, i=i)
m.addConstr(sum(a[h][j] * x[i][j] for j in range(n_f)) >= b[h] - self.mu - (n_f + 1) * z_ih[i][h], name)
print(name)
print()
if self.negated_hyperplanes:
for h in range(n_h / 2):
for i in range(n_d):
name = "z_IH({i}, {h}) = 1 - z_IH({i}, {hi})".format(i=i, h=h, hi=(h + n_h / 2))
m.addConstr(z_ih[i][n_h / 2 + h] == 1 - z_ih[i][h], name)
print(name)
print()
# If a hyperplane is assigned to any conjunction, it is selected (used)
for h in range(n_h):
for c in range(k):
name = "s_H({h}) >= z_H({h}, {c})".format(h=h, c=c)
m.addConstr(s_h[h] >= z_h[h][c], name)
print(name)
print()
# If a hyperplane is not assigned to any conjunction, it cannot be selected (used)
for h in range(n_h):
name = "s_H({h}) <= SUM_(c = 1..k) z_H({h}, c)".format(h=h)
m.addConstr(s_h[h] <= sum(z_h[h][c] for c in range(k)), name)
print(name)
print()
# If an example i is not assigned to c, then it cannot be assigned to every h that is assigned to c
# for c in range(k):
# for i in range(n_d):
# # TODO name
# name = "z_IH({i}, {h}) + z_H({h}, {c}) >= 2 - 2 * (1 - z_D({i}, {c}))".format(h=h, c=c, i=i)
# m.addConstr(sum(z_h[h][c] for h in range(n_h)) - sum(z_ihc[i][h][c] for h in range(n_h)) <= n_h * (1 - z_d[i][c]))
# m.addConstr(sum(z_h[h][c] for h in range(n_h)) - sum(z_ihc[i][h][c] for h in range(n_h)) >= n_h * (1 - z_d[i][c]))
# print(name)
# print()
# If (h, c) then (i, c) can only be selected if every (i, h, c) is selected
for c in range(k):
for h in range(n_h):
for i in range(n_d):
name = "z_D({i}, {c}) <= z_IHC({i}, {h}, {c}) + (1 - z_H({h}, {c}))".format(c=c, h=h, i=i)
m.addConstr(z_d[i][c] <= z_ihc[i][h][c] + (1 - z_h[h][c]), name)
print(name)
print()
# If (h, c) then (i, c) can only not be selected if some (i, h, c) is not selected
# for c in range(k):
# for h in range(n_h):
# for i in range(n_d):
# name = "z_D({i}, {c}) <= z_IHC({i}, {h}, {c}) + (1 - z_H({h}, {c}))".format(c=c, h=h, i=i)
# m.addConstr(z_d[i][c] <= z_ihc[i][h][c] + (1 - z_h[h][c]), name)
# print(name)
# print()
# If (i, c) is not selected, then there must exist an h such that: (h, c) and not (i, h, c)
for c in range(k):
for i in range(n_d):
name = "..." # TODO name
m.addConstr(sum(z_h[h][c] for h in range(n_h)) - sum(z_ihc[i][h][c] for h in range(n_h)) >= 1 - z_d[i][c], name)
print(name)
print()
# If (i, c) is selected, then there must be at least one selected tuple (h, c)
for c in range(k):
for i in range(n_d):
name = "SUM_(h = 1..n_H) z_H(h, {c}) >= 1 - (1 - z_D({i}, {c}))".format(c=c, i=i)
m.addConstr(sum(z_h[h][c] for h in range(n_h)) >= 1 - (1 - z_d[i][c]))
print(name)
print()
# If the triple (i, h, c) is selected, then both (i, h) and (h, c) must be true
for h in range(n_h):
for c in range(k):
for i in range(n_d):
name = "z_IH({i}, {h}) + z_H({h}, {c}) >= 2 - 2 * (1 - z_IHC({i}, {h}, {c}))".format(h=h, c=c, i=i)
m.addConstr(z_ih[i][h] + z_h[h][c] >= 2 - 2 * (1 - z_ihc[i][h][c]), name)
print(name)
print()
# If the triple (i, h, c) is not selected, then either (i, h) or (h, c) must be false
for h in range(n_h):
for c in range(k):
for i in range(n_d):
name = "z_IH({i}, {h}) + z_H({h}, {c}) <= 2 * z_IHC({i}, {h}, {c}) + 1".format(h=h, c=c, i=i)
m.addConstr(z_ih[i][h] + z_h[h][c] <= 2 * z_ihc[i][h][c] + 1, name)
print(name)
print()
# If an example is assigned to any conjunction, it is selected (covered)
for i in range(n_d):
for c in range(k):
name = "s_D({i}) >= z_D({i}, {c})".format(i=i, c=c)
m.addConstr(s_d[i] >= z_d[i][c], name)
print(name)
print()
# If an example is not assigned to any conjunction, it cannot be selected (covered)
for i in range(n_d):
name = "s_D({i}) <= SUM_(c = 1..k) z_D({i}, c)".format(i=i)
m.addConstr(s_d[i] <= sum(z_d[i][c] for c in range(k)), name)
print(name)
print()
# Examples can only be assigned to active hyperplanes
# z_ih <= s_h for all i, h
# for h in range(n_h):
# for i in range(n_d):
# name = "z_ih({i}, {h}) <= s_h({h})".format(i=i, h=h)
# m.addConstr(z_ih[i][h] <= s_h[h], name)
# print(name)
# print()
# Hack example
# m.addConstr(a[0][0] == 1)
# m.addConstr(a[0][1] == 0)
# m.addConstr(b[0] == 0.5)
#
# m.addConstr(a[1][0] == 0)
# m.addConstr(a[1][1] == 1)
# m.addConstr(b[1] == 0.5)
#
# m.addConstr(a[2][0] == -1)
# m.addConstr(a[2][1] == 0)
# m.addConstr(b[2] == -0.5)
#
# m.addConstr(a[3][0] == 0)
# m.addConstr(a[3][1] == -1)
# m.addConstr(b[3] == -0.5)
#
# m.addConstr(s_h[0] == 1)
# m.addConstr(s_h[1] == 1)
# m.addConstr(s_h[2] == 1)
# m.addConstr(s_h[3] == 1)
# m.addConstr(z_h[0][0] == 1)
# m.addConstr(z_h[1][0] == 1)
# m.addConstr(z_h[2][0] == 0)
# m.addConstr(z_h[3][0] == 0)
# m.addConstr(z_h[0][1] == 0)
# m.addConstr(z_h[1][1] == 0)
# m.addConstr(z_h[2][1] == 1)
# m.addConstr(z_h[3][1] == 1)
# m.addConstr(z_ih[0][0] == 1)
# m.addConstr(z_ih[1][0] == 0)
# m.addConstr(z_ih[2][0] == 1)
# m.addConstr(z_ih[3][0] == 0)
# m.addConstr(z_ih[0][1] == 1)
# m.addConstr(z_ih[1][1] == 0)
# m.addConstr(z_ih[2][1] == 0)
# m.addConstr(z_ih[3][1] == 1)
# m.addConstr(z_ih[0][2] == 0)
# m.addConstr(z_ih[1][2] == 1)
# m.addConstr(z_ih[2][2] == 0)
# m.addConstr(z_ih[3][2] == 1)
# m.addConstr(z_ih[0][3] == 0)
# m.addConstr(z_ih[1][3] == 1)
# m.addConstr(z_ih[2][3] == 1)
# m.addConstr(z_ih[3][3] == 0)
# m.addConstr(z_d[0][0] == 1)
# m.addConstr(z_d[1][0] == 0)
# m.addConstr(z_d[2][0] == 0)
# m.addConstr(z_d[3][0] == 0)
# m.addConstr(z_d[0][1] == 0)
# m.addConstr(z_d[1][1] == 0)
# m.addConstr(z_d[2][1] == 1)
# m.addConstr(z_d[3][1] == 0)
# Wrong solution
# m.addConstr(a[0][0] == 1)
# m.addConstr(a[0][1] == 0)
# m.addConstr(b[0] == 0.5)
#
# m.addConstr(a[1][0] == 0)
# m.addConstr(a[1][1] == 1)
# m.addConstr(b[1] == 0.5)
#
# m.addConstr(a[2][0] == -1)
# m.addConstr(a[2][1] == 0)
# m.addConstr(b[2] == -0.6)
#
# m.addConstr(a[3][0] == 0)
# m.addConstr(a[3][1] == -1)
# m.addConstr(b[3] == -0.6)
#
# m.addConstr(s_h[0] == 1)
# m.addConstr(s_h[1] == 1)
# m.addConstr(s_h[2] == 1)
# m.addConstr(s_h[3] == 1)
m.optimize()
for v in m.getVars():
print(v.varName, v.x)
print('Obj:', m.objVal)
print()
# Lets extract the kDNF formulation
print("Hyperplanes")
conjunctions = []
from pysmt.shortcuts import And, Or, Real, LE, Plus, Times, Symbol
for conj in range(k):
conjunction = []
for h in range(n_h):
if int(z_h[h][conj].x) == 1:
if h < n_h / 2 or not self.negated_hyperplanes:
coefficients = [Real(float(a[h][j].x)) for j in range(n_f)]
constant = Real(float(b[h].x))
else:
coefficients = [Real(-float(a[h - n_h / 2][j].x)) for j in range(n_f)]
constant = Real(-float(b[h - n_h / 2].x))
linear_sum = Plus([Times(c, domain.get_symbol(v)) for c, v in zip(coefficients, domain.variables)])
conjunction.append(LE(linear_sum, constant))
conjunctions.append(conjunction)
return conjunctions
