def test_auc_node_with_no_parents(self):
"""Should be possible to compute auc for state with no parent nodes"""
train = pd.DataFrame(
[[a, b, 0] for a in range(0, 2) for b in range(0, 2)
for _ in range(1)] + [[a, b, 1] for a in range(0, 2)
for b in range(0, 2)
for _ in range(a * 10 + b * 10 + 1000)] +
[[a, b, 0] for a in range(2, 4) for b in range(2, 4)
for _ in range(a * 10 + b * 10 + 1000)] + [[a, b, 1]
for a in range(2, 4)
for b in range(2, 4)
for _ in range(1)],
columns=["a", "b", "c"],
)
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
_, auc = roc_auc(bn, train, "a")
assert math.isclose(auc, 0.5, abs_tol=0.01)
def test_auc_with_missing_state_in_test(self):
"""AUC should still be calculated correctly with states missing in test set"""
# equal class (c) weighting to guarantee high ROC expected
train = pd.DataFrame(
[[a, b, 0] for a in range(0, 2) for b in range(0, 2)
for _ in range(1)] + [[a, b, 1] for a in range(0, 2)
for b in range(0, 2)
for _ in range(a * 10 + b * 10 + 1000)] +
[[a, b, 0] for a in range(2, 4) for b in range(2, 4)
for _ in range(a * 10 + b * 10 + 1000)] + [[a, b, 1]
for a in range(2, 4)
for b in range(2, 4)
for _ in range(1)],
columns=["a", "b", "c"],
)
test = train[train["c"] == 1]
assert len(test["c"].unique()) == 1
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
_, auc = roc_auc(bn, test, "c")
assert math.isclose(auc, 1, abs_tol=0.01)
def test_roc_of_incorrect_has_fpr_lt_tpr(self):
"""The ROC of incorrect predictions should have FPR < TPR"""
# regardless of a or b, c=1 is always more likely to varying amounts (to create multiple threshold
# points in roc curve)
train = pd.DataFrame(
[[a, b, 0] for a in range(3) for b in range(3)
for _ in range(1)] + [[a, b, 1] for a in range(3)
for b in range(3)
for _ in range(a * 1000 + b * 1000 + 1000)],
columns=["a", "b", "c"],
)
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
assert np.allclose(bn.cpds["c"].loc[1].values, 1, atol=0.02)
# in test, c=0 is always more likely (opposite of train)
test = pd.DataFrame(
[[a, b, 0] for a in range(3) for b in range(3)
for _ in range(1000)] + [[a, b, 1] for a in range(3)
for b in range(3) for _ in range(1)],
columns=["a", "b", "c"],
)
roc, _ = roc_auc(bn, test, "c")
assert len(roc) > 3
assert all(fpr > tpr for fpr, tpr in roc if tpr not in [0.0, 1.0])
def test_roc_of_accurate_predictions(self):
"""TPR should always be better than FPR for accurate predictions"""
# equal class (c) weighting to guarantee high ROC expected
train = pd.DataFrame(
[[a, b, 0] for a in range(0, 2) for b in range(0, 2)
for _ in range(10)] + [[a, b, 1] for a in range(0, 2)
for b in range(0, 2)
for _ in range(a * 10 + b * 10 + 1000)] +
[[a, b, 0] for a in range(2, 4) for b in range(2, 4)
for _ in range(a * 10 + b * 10 + 1000)] + [[a, b, 1]
for a in range(2, 4)
for b in range(2, 4)
for _ in range(10)],
columns=["a", "b", "c"],
)
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
roc, _ = roc_auc(bn, train, "c")
assert all(tpr > fpr for fpr, tpr in roc if tpr not in [0.0, 1.0])
def test_auc_of_accurate_predictions(self):
"""AUC of accurate predictions should be 1"""
# equal class (c) weighting to guarantee high ROC expected
train = pd.DataFrame(
[[a, b, 0] for a in range(0, 2) for b in range(0, 2)
for _ in range(1)] + [[a, b, 1] for a in range(0, 2)
for b in range(0, 2)
for _ in range(a * 10 + b * 10 + 1000)] +
[[a, b, 0] for a in range(2, 4) for b in range(2, 4)
for _ in range(a * 10 + b * 10 + 1000)] + [[a, b, 1]
for a in range(2, 4)
for b in range(2, 4)
for _ in range(1)],
columns=["a", "b", "c"],
)
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
_, auc = roc_auc(bn, train, "c")
assert math.isclose(auc, 1, abs_tol=0.001)
def test_auc_of_incorrect_close_to_zero(self):
"""The AUC of incorrect predictions should be close to zero"""
# regardless of a or b, c=1 is always more likely to varying amounts (to create multiple threshold
# points in roc curve)
train = pd.DataFrame(
[[a, b, 0] for a in range(3) for b in range(3)
for _ in range(1)] + [[a, b, 1] for a in range(3)
for b in range(3)
for _ in range(a * 1000 + b * 1000 + 1000)],
columns=["a", "b", "c"],
)
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
assert np.allclose(bn.cpds["c"].loc[1].values, 1, atol=0.02)
# in test, c=0 is always more likely (opposite of train)
test = pd.DataFrame(
[[a, b, 0] for a in range(3) for b in range(3)
for _ in range(1000)] + [[a, b, 1] for a in range(3)
for b in range(3) for _ in range(1)],
columns=["a", "b", "c"],
)
_, auc = roc_auc(bn, test, "c")
assert math.isclose(auc, 0, abs_tol=0.001)
def test_roc_of_random_has_unit_gradient(self):
"""The ROC curve for random predictions should be a line from (0,0) to (1,1)"""
# regardless of a or b, c=1 is always more likely to varying amounts (to create multiple threshold
# points in roc curve)
train = pd.DataFrame(
[[a, b, 0] for a in range(3) for b in range(3)
for _ in range(1)] + [[a, b, 1] for a in range(3)
for b in range(3)
for _ in range(a * 1000 + b * 1000 + 1000)],
columns=["a", "b", "c"],
)
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
assert np.allclose(bn.cpds["c"].loc[1].values, 1, atol=0.02)
test = pd.DataFrame(
[[a, b, random.randint(0, 1)] for a in range(3) for b in range(3)
for _ in range(1000)],
columns=["a", "b", "c"],
)
roc, _ = roc_auc(bn, test, "c")
assert len(roc) > 3
assert all(math.isclose(a, b, abs_tol=0.03) for a, b in roc)
def test_auc_for_nonnumeric_features(self):
"""AUC of accurate predictions should be 1 even after remapping numbers to strings"""
# equal class (c) weighting to guarantee high ROC expected
train = pd.DataFrame(
[[a, b, 0] for a in range(0, 2) for b in range(0, 2)
for _ in range(1)] + [[a, b, 1] for a in range(0, 2)
for b in range(0, 2)
for _ in range(a * 10 + b * 10 + 1000)] +
[[a, b, 0] for a in range(2, 4) for b in range(2, 4)
for _ in range(a * 10 + b * 10 + 1000)] + [[a, b, 1]
for a in range(2, 4)
for b in range(2, 4)
for _ in range(1)],
columns=["a", "b", "c"],
)
# remap values in column c
train["c"] = train["c"].map({0: "f", 1: "g"})
cg = StructureModel()
cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)])
bn = BayesianNetwork(cg)
bn.fit_node_states(train)
bn.fit_cpds(train)
_, auc = roc_auc(bn, train, "c")
assert math.isclose(auc, 1, abs_tol=0.001)
def test_intercept(self, distribution, n_categories):
graph = StructureModel()
graph.add_node("A")
data_noint = generate_categorical_dataframe(
graph,
100000,
distribution,
noise_scale=0.1,
n_categories=n_categories,
seed=10,
intercept=False,
)
data_intercept = generate_categorical_dataframe(
graph,
100000,
distribution,
noise_scale=0.1,
n_categories=n_categories,
seed=10,
intercept=True,
)
assert np.all(~np.isclose(data_intercept.mean(axis=0),
data_noint.mean(axis=0),
atol=0.05,
rtol=0))
def test_number_of_nodes(self, num_nodes):
""" Length of each row in generated data equals num_nodes """
graph = StructureModel()
edges = [(n, n + 1, 1) for n in range(num_nodes - 1)]
graph.add_weighted_edges_from(edges)
data = generate_binary_data(graph, 100, seed=10)
assert all(len(sample) == num_nodes for sample in data)
def test_number_of_columns(self, num_nodes, n_categories):
""" Length of dataframe is in the correct shape"""
graph = StructureModel()
edges = [(n, n + 1, 1) for n in range(num_nodes - 1)]
graph.add_weighted_edges_from(edges)
data = generate_categorical_dataframe(graph,
100,
seed=10,
n_categories=n_categories)
assert data.shape[1] == (num_nodes * n_categories)
def test_baseline_probability_probit(self, graph, distribution):
""" Test whether probability centered around 50% if no intercept given"""
graph = StructureModel()
graph.add_nodes_from(["A"])
data = generate_binary_data(
graph,
1000000,
distribution=distribution,
noise_scale=0.1,
seed=10,
intercept=False,
)
assert 0.45 < data[:, 0].mean() < 0.55
def test_intercept_probability_logit(self, graph, distribution):
""" Test whether probability is not centered around 50% when using an intercept"""
graph = StructureModel()
graph.add_nodes_from(["A"])
data = generate_binary_data(
graph,
1000000,
distribution=distribution,
noise_scale=0.1,
seed=10,
intercept=True,
)
mean_prob = data[:, 0].mean()
assert not np.isclose(mean_prob, 0.5, atol=0.05)
def test_intercept_probability(self, graph, distribution, n_categories):
""" Test whether probability is not centered around 50% when using an intercept"""
graph = StructureModel()
graph.add_nodes_from(["A"])
data = generate_categorical_dataframe(
graph,
1000000,
distribution=distribution,
n_categories=n_categories,
noise_scale=0.1,
seed=10,
intercept=True,
)
assert not np.allclose(
data.mean(axis=0), 1 / n_categories, atol=0.01, rtol=0)
def test_baseline_probability(self, graph, distribution, n_categories):
""" Test whether probability centered around 50% if no intercept given"""
graph = StructureModel()
graph.add_nodes_from(["A"])
data = generate_categorical_dataframe(
graph,
10000,
distribution=distribution,
n_categories=n_categories,
noise_scale=1.0,
seed=10,
intercept=False,
)
# without intercept, the probabilities should be fairly uniform
assert np.allclose(data.mean(axis=0),
1 / n_categories,
atol=0.01,
rtol=0)
def test_intercept(self, distribution):
graph = StructureModel()
graph.add_node("123")
data_noint = generate_binary_data(graph,
100000,
distribution,
noise_scale=0,
seed=10,
intercept=False)
data_intercept = generate_binary_data(graph,
100000,
distribution,
noise_scale=0,
seed=10,
intercept=True)
assert not np.isclose(data_noint[:, 0].mean(),
data_intercept[:, 0].mean())
def test_intercept(self, distribution):
graph = StructureModel()
graph.add_node("123")
data_noint = generate_continuous_data(
graph,
n_samples=100000,
distribution=distribution,
noise_scale=0,
seed=10,
intercept=False,
)
data_intercept = generate_continuous_data(
graph,
n_samples=100000,
distribution=distribution,
noise_scale=0,
seed=10,
intercept=True,
)
assert not np.isclose(data_noint[:, 0].mean(),
data_intercept[:, 0].mean())
assert np.isclose(data_noint[:, 0].std(), data_intercept[:, 0].std())
def add_to_node(self, sm: StructureModel) -> StructureModel:
"""
Adds self to a node of a structure model corresponding to
all indexes in self.idx_group.
Args:
sm: The input StructureModel
Returns:
Updated StructureModel
"""
for idx in self.idx_group:
sm.nodes[idx]["dist_type"] = self
return sm
def generator(num_nodes, seed, weight=None):
np.random.seed(seed)
sm = StructureModel()
nodes = list("".join(x) for x in product(
string.ascii_lowercase, string.ascii_lowercase))[:num_nodes]
np.random.shuffle(nodes)
sm.add_nodes_from(nodes)
# one edge:
sm.add_weighted_edges_from([("aa", "ab", weight)])
return sm
def structToGraph(weightedGraph: StructureModel,
nodeColor: Color = LIGHT_BLUE,
edgeColor: Color = CHERRY) -> gz.Digraph:
g = gz.Digraph('G')
adjacencies: List[Tuple[Name, Dict[Name, OriginAndWeightInfo]]] = list(
weightedGraph.adjacency())
for sourceVar, edgeDict in adjacencies:
edgeList: List[Name, OriginAndWeightInfo] = list(edgeDict.items())
for endVar, otherInfoDict in edgeList:
g.attr('node', shape='oval') #, color='red')
g.node(sourceVar,
sourceVar) # name, label # variables[head]['desc'])
g.node(endVar, endVar) # name, label
g.node_attr.update(style='filled',
gradientangle='90',
penwidth='1',
fillcolor=nodeColor + ":white",
color=edgeColor,
fontsize='12',
fontpath=ACME_FONT_PATH,
fontname=ACME_FONT_NAME) # + '.otf')
# Setting weighted edge here (if present)
if 'weight' in otherInfoDict.keys():
g.edge(tail_name=sourceVar,
head_name=endVar,
label=str(otherInfoDict['weight']))
else:
g.edge(tail_name=sourceVar, head_name=endVar)
g.edge_attr.update(color=edgeColor,
penwidth='1',
fontsize='10',
fontpath=PLAY_FONT_NAME,
fontname=PLAY_FONT_NAME)
return g
def test_incorrect_weight_dist(self):
sm = StructureModel()
nodes = list(str(x) for x in range(6))
np.random.shuffle(nodes)
sm.add_nodes_from(nodes)
sm.add_weighted_edges_from([("0", "1", None), ("2", "4", None)])
with pytest.raises(ValueError, match="Unknown weight distribution"):
_ = sem_generator(
graph=sm,
schema=None,
default_type="continuous",
distributions={"weight": "unknown"},
noise_std=2.0,
n_samples=1000,
intercept=False,
seed=10,
)
def _learn_structure_lasso(
X: np.ndarray,
beta: float,
bnds,
max_iter: int = 100,
h_tol: float = 1e-8,
w_threshold: float = 0.0,
) -> StructureModel:
"""
Based on initial implementation at https://github.com/xunzheng/notears
"""
def _h(w_vec: np.ndarray) -> float:
"""
Constraint function of the NOTEARS algorithm with lasso regularisation.
Args:
w_vec: weight vector (wpos and wneg).
Returns:
float: DAGness of the adjacency matrix (0 == DAG, >0 == cyclic).
"""
W = w_vec.reshape([d, d])
return np.trace(slin.expm(W * W)) - d
def _func(w_vec: np.ndarray) -> float:
"""
Objective function that the NOTEARS algorithm with lasso regularisation tries to minimise.
Args:
w_vec: weight vector (wpos and wneg).
Returns:
float: objective.
"""
w_pos = w_vec[:d**2]
w_neg = w_vec[d**2:]
wmat_pos = w_pos.reshape([d, d])
wmat_neg = w_neg.reshape([d, d])
wmat = wmat_pos - wmat_neg
loss = 0.5 / n * np.square(
np.linalg.norm(X.dot(np.eye(d, d) - wmat), "fro"))
h_val = _h(wmat)
return loss + 0.5 * rho * h_val * h_val + alpha * h_val + beta * w_vec.sum(
)
def _grad(w_vec: np.ndarray) -> np.ndarray:
"""
Gradient function used to compute next step in NOTEARS algorithm with lasso regularisation.
Args:
w_vec: weight vector (wpos and wneg).
Returns:
np.ndarray: gradient vector.
"""
w_pos = w_vec[:d**2]
w_neg = w_vec[d**2:]
grad_vec = np.zeros(2 * d**2)
wmat_pos = w_pos.reshape([d, d])
wmat_neg = w_neg.reshape([d, d])
wmat = wmat_pos - wmat_neg
loss_grad = -1.0 / n * X.T.dot(X).dot(np.eye(d, d) - wmat)
exp_hdmrd = slin.expm(wmat * wmat)
obj_grad = (
loss_grad +
(rho * (np.trace(exp_hdmrd) - d) + alpha) * exp_hdmrd.T * wmat * 2)
lbd_grad = beta * np.ones(d * d)
grad_vec[:d**2] = obj_grad.flatten() + lbd_grad
grad_vec[d**2:] = -obj_grad.flatten() + lbd_grad
return grad_vec
if X.size == 0:
raise ValueError("Input data X is empty, cannot learn any structure")
logging.info(
"Learning structure using 'NOTEARS' optimisation with lasso regularisation."
)
n, d = X.shape
w_est, w_new = np.zeros(2 * d * d), np.zeros(2 * d * d)
rho, alpha, h_val, h_new = 1.0, 0.0, np.inf, np.inf
for n_iter in range(max_iter):
while rho < 1e20:
sol = sopt.minimize(_func,
w_est,
method="L-BFGS-B",
jac=_grad,
bounds=bnds)
w_new = sol.x
h_new = _h(w_new[:d**2].reshape([d, d]) -
w_new[d**2:].reshape([d, d]))
if h_new > 0.25 * h_val:
rho *= 10
else:
break
w_est, h_val = w_new, h_new
alpha += rho * h_val
if h_val <= h_tol:
break
if h_val > h_tol and n_iter == max_iter - 1:
warnings.warn("Failed to converge. Consider increasing max_iter.")
w_new = w_est[:d**2].reshape([d, d]) - w_est[d**2:].reshape([d, d])
w_new[np.abs(w_new) < w_threshold] = 0
return StructureModel(w_new.reshape([d, d]))
def _learn_structure(
X: np.ndarray,
bnds,
max_iter: int = 100,
h_tol: float = 1e-8,
w_threshold: float = 0.0,
) -> StructureModel:
"""
Based on initial implementation at https://github.com/xunzheng/notears
"""
def _h(w: np.ndarray) -> float:
"""
Constraint function of the NOTEARS algorithm.
Args:
w: current adjacency matrix.
Returns:
float: DAGness of the adjacency matrix (0 == DAG, >0 == cyclic).
"""
W = w.reshape([d, d])
return np.trace(slin.expm(W * W)) - d
def _func(w: np.ndarray) -> float:
"""
Objective function that the NOTEARS algorithm tries to minimise.
Args:
w: current adjacency matrix.
Returns:
float: objective.
"""
W = w.reshape([d, d])
loss = 0.5 / n * np.square(
np.linalg.norm(X.dot(np.eye(d, d) - W), "fro"))
h = _h(W)
return loss + 0.5 * rho * h * h + alpha * h
def _grad(w: np.ndarray) -> np.ndarray:
"""
Gradient function used to compute next step in NOTEARS algorithm.
Args:
w: the current adjacency matrix.
Returns:
np.ndarray: gradient vector.
"""
W = w.reshape([d, d])
loss_grad = -1.0 / n * X.T.dot(X).dot(np.eye(d, d) - W)
E = slin.expm(W * W)
obj_grad = loss_grad + (rho * (np.trace(E) - d) + alpha) * E.T * W * 2
return obj_grad.flatten()
if X.size == 0:
raise ValueError("Input data X is empty, cannot learn any structure")
logging.info("Learning structure using 'NOTEARS' optimisation.")
# n examples, d properties
n, d = X.shape
# initialise matrix to zeros
w_est, w_new = np.zeros(d * d), np.zeros(d * d)
# initialise weights and constraints
rho, alpha, h, h_new = 1.0, 0.0, np.inf, np.inf
# start optimisation
for n_iter in range(max_iter):
while rho < 1e20:
sol = sopt.minimize(_func,
w_est,
method="L-BFGS-B",
jac=_grad,
bounds=bnds)
w_new = sol.x
h_new = _h(w_new)
if h_new > 0.25 * h:
rho *= 10
else:
break
w_est, h = w_new, h_new
alpha += rho * h
if h <= h_tol:
break
if h > h_tol and n_iter == max_iter - 1:
warnings.warn("Failed to converge. Consider increasing max_iter.")
w_est[np.abs(w_est) <= w_threshold] = 0
return StructureModel(w_est.reshape([d, d]))
def from_pandas_lasso(
X: pd.DataFrame,
beta: float,
max_iter: int = 100,
h_tol: float = 1e-8,
w_threshold: float = 0.0,
tabu_edges: List[Tuple[str, str]] = None,
tabu_parent_nodes: List[str] = None,
tabu_child_nodes: List[str] = None,
) -> StructureModel:
"""
Learn the `StructureModel`, the graph structure with lasso regularisation
describing conditional dependencies between variables in data presented as a pandas dataframe.
Based on DAGs with NO TEARS.
@inproceedings{zheng2018dags,
author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.},
booktitle = {Advances in Neural Information Processing Systems},
title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}},
year = {2018},
codebase = {https://github.com/xunzheng/notears}
}
Args:
X: input data.
beta: Constant that multiplies the lasso term.
max_iter: max number of dual ascent steps during optimisation.
h_tol: exit if h(W) < h_tol (as opposed to strict definition of 0).
w_threshold: fixed threshold for absolute edge weights.
tabu_edges: list of edges(from, to) not to be included in the graph.
tabu_parent_nodes: list of nodes banned from being a parent of any other nodes.
tabu_child_nodes: list of nodes banned from being a child of any other nodes.
Returns:
StructureModel: graph of conditional dependencies between data variables.
Raises:
ValueError: If X does not contain data.
"""
data = deepcopy(X)
non_numeric_cols = data.select_dtypes(exclude="number").columns
if not non_numeric_cols.empty:
raise ValueError(
"All columns must have numeric data. "
"Consider mapping the following columns to int {non_numeric_cols}".
format(non_numeric_cols=non_numeric_cols))
col_idx = {c: i for i, c in enumerate(data.columns)}
idx_col = {i: c for c, i in col_idx.items()}
if tabu_edges:
tabu_edges = [(col_idx[u], col_idx[v]) for u, v in tabu_edges]
if tabu_parent_nodes:
tabu_parent_nodes = [col_idx[n] for n in tabu_parent_nodes]
if tabu_child_nodes:
tabu_child_nodes = [col_idx[n] for n in tabu_child_nodes]
g = from_numpy_lasso(
data.values,
beta,
max_iter,
h_tol,
w_threshold,
tabu_edges,
tabu_parent_nodes,
tabu_child_nodes,
)
sm = StructureModel()
sm.add_nodes_from(data.columns)
sm.add_weighted_edges_from(
[(idx_col[u], idx_col[v], w) for u, v, w in g.edges.data("weight")],
origin="learned",
)
return sm
def from_pandas(
X: pd.DataFrame,
max_iter: int = 100,
h_tol: float = 1e-8,
w_threshold: float = 0.0,
tabu_edges: List[Tuple[str, str]] = None,
tabu_parent_nodes: List[str] = None,
tabu_child_nodes: List[str] = None,
) -> StructureModel:
"""
Learn the `StructureModel`, the graph structure describing conditional dependencies between variables
in data presented as a pandas dataframe.
The optimisation is to minimise a score function :math:`F(W)` over the graph's
weighted adjacency matrix, :math:`W`, subject to the a constraint function :math:`h(W)`,
where :math:`h(W) == 0` characterises an acyclic graph.
:math:`h(W) > 0` is a continuous, differentiable function that encapsulated how acyclic the graph is
(less == more acyclic).
Full details of this approach to structure learning are provided in the publication:
Based on DAGs with NO TEARS.
@inproceedings{zheng2018dags,
author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.},
booktitle = {Advances in Neural Information Processing Systems},
title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}},
year = {2018},
codebase = {https://github.com/xunzheng/notears}
}
Args:
X: input data.
max_iter: max number of dual ascent steps during optimisation.
h_tol: exit if h(W) < h_tol (as opposed to strict definition of 0).
w_threshold: fixed threshold for absolute edge weights.
tabu_edges: list of edges(from, to) not to be included in the graph.
tabu_parent_nodes: list of nodes banned from being a parent of any other nodes.
tabu_child_nodes: list of nodes banned from being a child of any other nodes.
Returns:
StructureModel: graph of conditional dependencies between data variables.
Raises:
ValueError: If X does not contain data.
"""
data = deepcopy(X)
non_numeric_cols = data.select_dtypes(exclude="number").columns
if len(non_numeric_cols) > 0:
raise ValueError(
"All columns must have numeric data. "
"Consider mapping the following columns to int {non_numeric_cols}".
format(non_numeric_cols=non_numeric_cols))
col_idx = {c: i for i, c in enumerate(data.columns)}
idx_col = {i: c for c, i in col_idx.items()}
if tabu_edges:
tabu_edges = [(col_idx[u], col_idx[v]) for u, v in tabu_edges]
if tabu_parent_nodes:
tabu_parent_nodes = [col_idx[n] for n in tabu_parent_nodes]
if tabu_child_nodes:
tabu_child_nodes = [col_idx[n] for n in tabu_child_nodes]
g = from_numpy(
data.values,
max_iter,
h_tol,
w_threshold,
tabu_edges,
tabu_parent_nodes,
tabu_child_nodes,
)
sm = StructureModel()
sm.add_nodes_from(data.columns)
sm.add_weighted_edges_from(
[(idx_col[u], idx_col[v], w) for u, v, w in g.edges.data("weight")],
origin="learned",
)
return sm
def from_numpy(X: np.ndarray,
dist_type_schema: Dict[int, str] = None,
lasso_beta: float = 0.0,
ridge_beta: float = 0.0,
use_bias: bool = False,
hidden_layer_units: Iterable[int] = None,
w_threshold: float = None,
max_iter: int = 100,
tabu_edges: List[Tuple[int, int]] = None,
tabu_parent_nodes: List[int] = None,
tabu_child_nodes: List[int] = None,
**kwargs) -> StructureModel:
"""
Learn the `StructureModel`, the graph structure with lasso regularisation
describing conditional dependencies between variables in data presented as a numpy array.
Based on DAGs with NO TEARS.
@inproceedings{zheng2018dags,
author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.},
booktitle = {Advances in Neural Information Processing Systems},
title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}},
year = {2018},
codebase = {https://github.com/xunzheng/notears}
}
Args:
X: 2d input data, axis=0 is data rows, axis=1 is data columns. Data must be row oriented.
dist_type_schema: The dist type schema corresponding to the passed in data X.
It maps the positional column in X to the string alias of a dist type.
A list of alias names can be found in ``dist_type/__init__.py``.
If None, assumes that all data in X is continuous.
lasso_beta: Constant that multiplies the lasso term (l1 regularisation).
NOTE when using nonlinearities, the l1 loss only applies to the dag_layer.
use_bias: Whether to fit a bias parameter in the NOTEARS algorithm.
ridge_beta: Constant that multiplies the ridge term (l2 regularisation).
When using nonlinear layers use of this parameter is recommended.
hidden_layer_units: An iterable where its length determine the number of layers used,
and the numbers determine the number of nodes used for the layer in order.
w_threshold: fixed threshold for absolute edge weights.
max_iter: max number of dual ascent steps during optimisation.
tabu_edges: list of edges(from, to) not to be included in the graph.
tabu_parent_nodes: list of nodes banned from being a parent of any other nodes.
tabu_child_nodes: list of nodes banned from being a child of any other nodes.
**kwargs: additional arguments for NOTEARS MLP model
Returns:
StructureModel: a graph of conditional dependencies between data variables.
Raises:
ValueError: If X does not contain data.
ValueError: If schema does not correspond to columns.
"""
# n examples, d properties
if not X.size:
raise ValueError("Input data X is empty, cannot learn any structure")
logging.info("Learning structure using 'NOTEARS' optimisation.")
# Check array for NaN or inf values
check_array(X)
if dist_type_schema is not None:
# make sure that there is one provided key per column
if set(range(X.shape[1])).symmetric_difference(
set(dist_type_schema.keys())):
raise ValueError(
"Difference indices and expected indices. Got {} schema".
format(dist_type_schema))
# if dist_type_schema is None, assume all columns are continuous, else init the alias mapped object
dist_types = (
[DistTypeContinuous(idx=idx)
for idx in np.arange(X.shape[1])] if dist_type_schema is None else [
dist_type_aliases[alias](idx=idx)
for idx, alias in dist_type_schema.items()
])
# shape of X before preprocessing
_, d_orig = X.shape
# perform dist type pre-processing (i.e. column expansion)
for dist_type in dist_types:
# NOTE: preprocess_X must be called first to perform possible column expansions
X = dist_type.preprocess_X(X)
tabu_edges = dist_type.preprocess_tabu_edges(tabu_edges)
tabu_parent_nodes = dist_type.preprocess_tabu_nodes(tabu_parent_nodes)
tabu_child_nodes = dist_type.preprocess_tabu_nodes(tabu_child_nodes)
# shape of X after preprocessing
_, d = X.shape
# if None or empty, convert into a list with single item
if hidden_layer_units is None:
hidden_layer_units = [0]
elif isinstance(hidden_layer_units, list) and not hidden_layer_units:
hidden_layer_units = [0]
# if no hidden layer units, still take 1 iteration step with bounds
hidden_layer_bnds = hidden_layer_units[0] if hidden_layer_units[0] else 1
# Flip i and j because Pytorch flattens the vector in another direction
bnds = [
(0, 0) if i == j else
(0, 0) if tabu_edges is not None and (i, j) in tabu_edges else
(0, 0) if tabu_parent_nodes is not None and i in tabu_parent_nodes else
(0, 0) if tabu_child_nodes is not None and j in tabu_child_nodes else
(None, None) for j in range(d) for _ in range(hidden_layer_bnds)
for i in range(d)
]
model = NotearsMLP(n_features=d,
dist_types=dist_types,
hidden_layer_units=hidden_layer_units,
lasso_beta=lasso_beta,
ridge_beta=ridge_beta,
bounds=bnds,
use_bias=use_bias,
**kwargs)
model.fit(X, max_iter=max_iter)
sm = StructureModel(model.adj)
if w_threshold:
sm.remove_edges_below_threshold(w_threshold)
# extract the mean effect and add as edge attribute
mean_effect = model.adj_mean_effect
for u, v, edge_dict in sm.edges.data(True):
sm.add_edge(
u,
v,
origin="learned",
weight=edge_dict["weight"],
mean_effect=mean_effect[u, v],
)
# set bias as node attribute
bias = model.bias
for node in sm.nodes():
value = None
if bias is not None:
value = bias[node]
sm.nodes[node]["bias"] = value
# attach each dist_type object to corresponding node(s)
for dist_type in dist_types:
sm = dist_type.add_to_node(sm)
# preserve the structure_learner as a graph attribute
sm.graph["structure_learner"] = model
# collapse the adj down and store as graph attr
adj = deepcopy(model.adj)
for dist_type in dist_types:
adj = dist_type.collapse_adj(adj)
sm.graph["graph_collapsed"] = StructureModel(adj[:d_orig, :d_orig])
return sm
def from_pandas(X: pd.DataFrame,
dist_type_schema: Dict[Union[str, int], str] = None,
lasso_beta: float = 0.0,
ridge_beta: float = 0.0,
use_bias: bool = False,
hidden_layer_units: Iterable[int] = None,
max_iter: int = 100,
w_threshold: float = None,
tabu_edges: List[Tuple[str, str]] = None,
tabu_parent_nodes: List[str] = None,
tabu_child_nodes: List[str] = None,
**kwargs) -> StructureModel:
"""
Learn the `StructureModel`, the graph structure describing conditional dependencies between variables
in data presented as a pandas dataframe.
The optimisation is to minimise a score function :math:`F(W)` over the graph's
weighted adjacency matrix, :math:`W`, subject to the a constraint function :math:`h(W)`,
where :math:`h(W) == 0` characterises an acyclic graph.
:math:`h(W) > 0` is a continuous, differentiable function that encapsulated how acyclic the graph is
(less == more acyclic).
Full details of this approach to structure learning are provided in the publication:
Based on DAGs with NO TEARS.
@inproceedings{zheng2018dags,
author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.},
booktitle = {Advances in Neural Information Processing Systems},
title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}},
year = {2018},
codebase = {https://github.com/xunzheng/notears}
}
Args:
X: 2d input data, axis=0 is data rows, axis=1 is data columns. Data must be row oriented.
dist_type_schema: The dist type schema corresponding to the passed in data X.
It maps the pandas column name in X to the string alias of a dist type.
A list of alias names can be found in ``dist_type/__init__.py``.
If None, assumes that all data in X is continuous.
lasso_beta: Constant that multiplies the lasso term (l1 regularisation).
NOTE when using nonlinearities, the l1 loss only applies to the dag_layer.
use_bias: Whether to fit a bias parameter in the NOTEARS algorithm.
ridge_beta: Constant that multiplies the ridge term (l2 regularisation).
When using nonlinear layers use of this parameter is recommended.
hidden_layer_units: An iterable where its length determine the number of layers used,
and the numbers determine the number of nodes used for the layer in order.
w_threshold: fixed threshold for absolute edge weights.
max_iter: max number of dual ascent steps during optimisation.
tabu_edges: list of edges(from, to) not to be included in the graph.
tabu_parent_nodes: list of nodes banned from being a parent of any other nodes.
tabu_child_nodes: list of nodes banned from being a child of any other nodes.
**kwargs: additional arguments for NOTEARS MLP model
Returns:
StructureModel: graph of conditional dependencies between data variables.
Raises:
ValueError: If X does not contain data.
"""
data = deepcopy(X)
# if dist_type_schema is not None, convert dist_type_schema from cols to idx
dist_type_schema = (dist_type_schema if dist_type_schema is None else {
X.columns.get_loc(col): alias
for col, alias in dist_type_schema.items()
})
non_numeric_cols = data.select_dtypes(exclude="number").columns
if len(non_numeric_cols) > 0:
raise ValueError(
"All columns must have numeric data. "
"Consider mapping the following columns to int {non_numeric_cols}".
format(non_numeric_cols=non_numeric_cols))
col_idx = {c: i for i, c in enumerate(data.columns)}
idx_col = {i: c for c, i in col_idx.items()}
if tabu_edges:
tabu_edges = [(col_idx[u], col_idx[v]) for u, v in tabu_edges]
if tabu_parent_nodes:
tabu_parent_nodes = [col_idx[n] for n in tabu_parent_nodes]
if tabu_child_nodes:
tabu_child_nodes = [col_idx[n] for n in tabu_child_nodes]
g = from_numpy(X=data.values,
dist_type_schema=dist_type_schema,
lasso_beta=lasso_beta,
ridge_beta=ridge_beta,
use_bias=use_bias,
hidden_layer_units=hidden_layer_units,
w_threshold=w_threshold,
max_iter=max_iter,
tabu_edges=tabu_edges,
tabu_parent_nodes=tabu_parent_nodes,
tabu_child_nodes=tabu_child_nodes,
**kwargs)
# set comprehension to ensure only unique dist types are extraced
# NOTE: this prevents double-renaming caused by the same dist type used on expanded columns
unique_dist_types = {node[1]["dist_type"] for node in g.nodes(data=True)}
# use the dist types to update the idx_col mapping
idx_col_expanded = deepcopy(idx_col)
for dist_type in unique_dist_types:
idx_col_expanded = dist_type.update_idx_col(idx_col_expanded)
sm = StructureModel()
# add expanded set of nodes
sm.add_nodes_from(list(idx_col_expanded.values()))
# recover the edge weights from g
for u, v, edge_dict in g.edges.data(True):
sm.add_edge(
idx_col_expanded[u],
idx_col_expanded[v],
origin="learned",
weight=edge_dict["weight"],
mean_effect=edge_dict["mean_effect"],
)
# retrieve all graphs attrs
for key, val in g.graph.items():
sm.graph[key] = val
# recover the node biases from g
for node in g.nodes(data=True):
node_name = idx_col_expanded[node[0]]
sm.nodes[node_name]["bias"] = node[1]["bias"]
# recover and preseve the node dist_types
for node_data in g.nodes(data=True):
node_name = idx_col_expanded[node_data[0]]
sm.nodes[node_name]["dist_type"] = node_data[1]["dist_type"]
# recover the collapsed model from g
sm_collapsed = StructureModel()
sm_collapsed.add_nodes_from(list(idx_col.values()))
for u, v, edge_dict in g.graph["graph_collapsed"].edges.data(True):
sm_collapsed.add_edge(
idx_col[u],
idx_col[v],
origin="learned",
weight=edge_dict["weight"],
)
sm.graph["graph_collapsed"] = sm_collapsed
return sm
def _generate_inter_structure(
num_nodes: int,
p: int,
degree: float,
graph_type: str,
w_min: float,
w_max: float,
w_decay: float = 1.0,
neg: float = 0.5,
) -> StructureModel:
"""Simulate random DAG between two time slices.
Args:
num_nodes: number of nodes per slice
p: number of slices that influence current slice
degree: expected in-degree of current time slice
graph_type: {'erdos-renyi' 'full'}
w_min: minimum weight for inter-slice nodes
w_max: maximum weight for inter-slice nodes
w_decay: exponent of weights decay for slices that are farther apart. Default is 1.0, which implies no decay
neg: the proportion of edge weights expected to be negative. By default, 50% of the edges are expected
to be negative weight (`neg == 0.5`).
Returns:
G_inter: weighted, bipartite DAG for inter-slice connections
Raises:
ValueError: if graph type not known
"""
if w_min > w_max:
raise ValueError(
"Absolute minimum weight must be less than or equal to maximum weight: "
f"{w_min} > {w_max}")
if graph_type == "erdos-renyi":
prob = degree / num_nodes
b = (np.random.rand(p * num_nodes, num_nodes) < prob).astype(float)
elif graph_type == "full": # ignore degree, only for experimental use
b = np.ones([p * num_nodes, num_nodes])
else:
raise ValueError(f"Unknown inter-slice graph type `{graph_type}`. "
"Valid types are 'erdos-renyi' and 'full'")
u = []
for i in range(p):
u_i = np.random.uniform(
low=w_min, high=w_max, size=[num_nodes, num_nodes]) / (w_decay**i)
u_i[np.random.rand(num_nodes, num_nodes) < neg] *= -1
u.append(u_i)
u = np.concatenate(u, axis=0) if u else np.empty(b.shape)
a = (b != 0).astype(float) * u
df = pd.DataFrame(
a,
index=[
f"{var}_lag{l_val}" for l_val in range(1, p + 1)
for var in range(num_nodes)
],
columns=[f"{var}_lag0" for var in range(num_nodes)],
)
idxs, cols = list(df.index), list(df.columns)
for i in idxs:
df[i] = 0
for i in cols:
df.loc[i, :] = 0
g_inter = StructureModel(df)
return g_inter
def generate_structure_dynamic( # pylint: disable=too-many-arguments
num_nodes: int,
p: int,
degree_intra: float,
degree_inter: float,
graph_type_intra: str = "erdos-renyi",
graph_type_inter: str = "erdos-renyi",
w_min_intra: float = 0.5,
w_max_intra: float = 0.5,
w_min_inter: float = 0.5,
w_max_inter: float = 0.5,
w_decay: float = 1.0,
) -> StructureModel:
"""
Generates a dynamic DAG at random.
Args:
num_nodes: Number of nodes
p: maximum lag to be considered in the structure
degree_intra: expected degree on nodes from the current state
degree_inter: expected degree on nodes from the lagged nodes
graph_type_intra:
- erdos-renyi: constructs a graph such that the probability of any given edge is degree / (num_nodes - 1)
- barabasi-albert: constructs a scale-free graph from an initial connected graph of (degree / 2) nodes
- full: constructs a fully-connected graph - degree has no effect
graph_type_inter:
- erdos-renyi: constructs a graph such that the probability of any given edge is degree / (num_nodes - 1)
- full: connect all past nodes to all present nodes
w_min_intra: minimum weight for intra-slice nodes
w_max_intra: maximum weight for intra-slice nodes
w_min_inter: minimum weight for inter-slice nodes
w_max_inter: maximum weight for inter-slice nodes
w_decay: exponent of weights decay for slices that are farther apart. Default is 1.0, which implies no decay
Raises:
ValueError: if graph type unknown or `num_nodes < 2`
Returns:
StructureModel containing all simulated nodes and edges (intra- and inter-slice)
"""
sm_intra = generate_structure(
num_nodes=num_nodes,
degree=degree_intra,
graph_type=graph_type_intra,
w_min=w_min_intra,
w_max=w_max_intra,
)
sm_inter = _generate_inter_structure(
num_nodes=num_nodes,
p=p,
degree=degree_inter,
graph_type=graph_type_inter,
w_min=w_min_inter,
w_max=w_max_inter,
w_decay=w_decay,
)
res = StructureModel()
res.add_nodes_from(sm_inter.nodes)
res.add_nodes_from([f"{u}_lag0" for u in sm_intra.nodes])
res.add_weighted_edges_from(sm_inter.edges.data("weight"))
res.add_weighted_edges_from([(f"{u}_lag0", f"{v}_lag0", w)
for u, v, w in sm_intra.edges.data("weight")])
return res
def naive_bayes_plus_parents(
categories: int = 3,
samples: int = 500,
parents: int = 3,
children: int = 3,
p_z: float = 0.9,
p_c: float = 0.9,
percentage_not_missing: float = 0,
seed: int = 22,
) -> Tuple[pd.DataFrame, StructureModel, Dict, np.array]:
"""
p0 ... pn
\\ | /
z
/ | \\
c0 ... cm
z = mode of parents with probability p_z, otherwise mode of parents + 1 mod n_categories
c0 = z with prob. p_c, otherwise it is z + 1 mod n_categories
if no p are give, sample z from the categories uniformly
Args:
categories: number of categories
samples: number of samples
parents: number of parents, n as shown above
children: number of children, m as above
p_z: probability that z = mode(parents)
p_c: probability that children equals parent
percentage_not_missing: percentage of the LV that is provided. The default is 0, i.e. the LV is not observed
seed: seed for random generator
Returns:
data: sampled pandas dataframe, missing data on z
sm: structure model
node_states: dictionary of list of states for each node
true_lv_values: true values of latent variable
"""
def mode(lst: Iterable) -> Any:
return Counter(lst).most_common()[0][0] if len(lst) > 0 else np.nan
np.random.seed(seed)
par_samples = np.random.choice(categories, size=[samples, parents])
if parents == 0:
true_lv_values = np.random.choice(categories, size=[samples, 1])
else:
true_lv_values = np.array(
[
[(mode(el) + np.random.choice(2, p=[p_z, 1 - p_z])) % categories]
for el in par_samples
]
)
child_samples = np.random.random(size=[samples, children])
aux = true_lv_values.repeat(children, axis=1)
child_samples = np.where(child_samples < p_c, aux, (aux + 1) % categories)
df = pd.concat(
[
pd.DataFrame(par_samples, columns=[f"p_{i}" for i in range(parents)]),
pd.DataFrame(child_samples, columns=[f"c_{i}" for i in range(children)]),
pd.DataFrame(true_lv_values, columns=["z"]),
],
axis=1,
)
df.loc[int(samples * percentage_not_missing) :, "z"] = np.nan
sm = StructureModel()
sm.add_edges_from([(f"p_{i}", "z") for i in range(parents)])
sm.add_edges_from([("z", f"c_{i}") for i in range(children)])
node_states = {"z": list(range(categories))}
for i in range(parents):
node_states[f"p_{i}"] = list(range(categories))
for i in range(children):
node_states[f"c_{i}"] = list(range(categories))
return df, sm, node_states, true_lv_values