def naive_bayes(data, target, estimator='mle'):
"""
Learn naive bayes model from data.
The Naive Bayes model is a Tree-based
model where all random variables have
the same parent (the "target" variable).
From a probabilistic standpoint, the implication
of this model is that all random variables
(i.e. features) are assumed to be
conditionally independent of any other random variable,
conditioned upon the single parent (target) variable.
It turns out that this model performs quite well
as a classifier, and can be used as such. Moreover,
this model is quite fast and simple to learn/create
from a computational standpoint.
Note that this function not only learns the structure,
but ALSO learns the parameters.
Arguments
---------
*data* : a nested numpy array
*target* : an integer
The target variable column in *data*
Returns
-------
*bn* : a BayesNet object,
with the structure instantiated.
Effects
-------
None
Notes
-----
"""
value_dict = dict(
zip(range(data.shape[1]), [list(np.unique(col)) for col in data.T]))
edge_dict = {target: [v for v in value_dict if v != target]}
edge_dict.update(dict([(rv, []) for rv in value_dict if rv != target]))
bn = BayesNet(edge_dict, value_dict)
if estimator == 'bayes':
bayes_estimator(bn, data)
else:
mle_estimator(bn, data)
return bn
def chow_liu(data, edges_only=False):
"""
Perform Chow-Liu structure learning algorithm
over an entire dataset, and return the BN-tree.
Arguments
---------
*data* : a nested numpy array
The data from which we will learn. It should be
the entire dataset.
Returns
-------
*bn* : a BayesNet object
The structure-learned BN.
Effects
-------
None
Notes
-----
"""
value_dict = dict(
zip(range(data.shape[1]), [list(np.unique(col)) for col in data.T]))
n_rv = data.shape[1]
edge_list = [(i,j,mi_test(data[:,(i,j)],chi2_test=False)) \
for i in range(n_rv) for j in range(i+1,n_rv)]
edge_list.sort(key=operator.itemgetter(2), reverse=True) # sort by weight
vertex_cache = {edge_list[0][0]} # start with first vertex..
mst = dict((rv, []) for rv in range(n_rv))
for i, j, w in edge_list:
if i in vertex_cache and j not in vertex_cache:
mst[i].append(j)
vertex_cache.add(j)
elif i not in vertex_cache and j in vertex_cache:
mst[j].append(i)
vertex_cache.add(i)
if edges_only == True:
return mst, value_dict
bn = BayesNet(mst, value_dict)
return bn
def bridge(c_bn, f_bn, data): """ Make a Multi-Dimensional Bayesian Network by bridging two Bayesian network structures. This happens by placing edges from c_bn -> f_bn using a heuristic optimization procedure. This can be used to create a Multi-Dimensional Bayesian Network classifier from two already-learned Bayesian networks - one of which is a BN containing all the class variables, the other containing all the feature variables. Arguments --------- *c_bn* : a BayesNet object with known structure *f_bn* : a BayesNet object with known structure. Returns ------- *m_bn* : a merged/bridge BayesNet object, whose structure contains *c_bn*, *f_bn*, and some bridge edges between them. """ restrict = [] for u in c_cols: for v in f_cols: restrict.append((u,v)) # only allow edges from c_bn -> f_bn bridge_bn = hc_rr(data, restriction=restrict) m_bn = bridge_bn.E m_bn.update(c_bn.E) m_bn.update(f_bn.E) mbc_bn = BayesNet(E=m_bn)
def hc_rr(data,
M=5,
R=8,
metric='AIC',
max_iter=300,
debug=False,
init_nodes=None,
restriction=None):
"""
Arguments
---------
*data* : a nested numpy array
The data from which the Bayesian network
structure will be learned.
*metric* : a string
Which score metric to use.
Options:
- AIC
- BIC / MDL
- LL (log-likelihood)
*max_iter* : an integer
The maximum number of iterations of the
hill-climbing algorithm to run. Note that
the algorithm will terminate on its own if no
improvement is made in a given iteration.
*debug* : boolean
Whether to print(the scores/moves of the)
algorithm as its happening.
*init_nodes* : a list of initialize nodes (number of nodes according to the dataset)
*restriction* : a list of 2-tuples
For MMHC algorithm, the list of allowable edge additions.
Returns
-------
*bn* : a BayesNet object
"""
nrow = data.shape[0]
ncol = data.shape[1]
names = range(ncol)
# INITIALIZE NETWORK W/ NO EDGES
# maintain children and parents dict for fast lookups
c_dict = dict([(n, []) for n in names])
p_dict = dict([(n, []) for n in names])
# COMPUTE INITIAL LIKELIHOOD SCORE
value_dict = dict([(n, np.unique(data[:, i]))
for i, n in enumerate(names)])
bn = BayesNet(c_dict)
mle_estimator(bn, data)
max_score = info_score(bn, data, metric)
_iter = 0
improvement = True
_restarts = 0
while improvement:
improvement = False
max_delta = 0
if debug:
print('ITERATION: ', _iter)
### TEST ARC ADDITIONS ###
for u in bn.nodes():
for v in bn.nodes():
if v not in c_dict[u] and u != v and not would_cause_cycle(
c_dict, u, v):
# FOR MMHC ALGORITHM -> Edge Restrictions
if (init_nodes is None or not (v in init_nodes)):
if restriction is None or (u, v) in restriction:
# SCORE FOR 'V' -> gaining a parent
old_cols = (v, ) + tuple(
p_dict[v]) # without 'u' as parent
mi_old = mutual_information(data[:, old_cols])
new_cols = old_cols + (u, ) # with'u' as parent
mi_new = mutual_information(data[:, new_cols])
delta_score = nrow * (mi_old - mi_new)
if delta_score > max_delta:
if debug:
print('Improved Arc Addition: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Addition'
max_arc = (u, v)
### TEST ARC DELETIONS ###
for u in bn.nodes():
for v in bn.nodes():
if v in c_dict[u]:
# SCORE FOR 'V' -> losing a parent
old_cols = (v, ) + tuple(p_dict[v]) # with 'u' as parent
mi_old = mutual_information(data[:, old_cols])
new_cols = tuple([i for i in old_cols
if i != u]) # without 'u' as parent
mi_new = mutual_information(data[:, new_cols])
delta_score = nrow * (mi_old - mi_new)
if delta_score > max_delta:
if debug:
print('Improved Arc Deletion: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Deletion'
max_arc = (u, v)
### TEST ARC REVERSALS ###
for u in bn.nodes():
for v in bn.nodes():
if v in c_dict[u] and not would_cause_cycle(
c_dict, v, u, reverse=True) and (
init_nodes is None or
not (u in init_nodes)) and (restriction is None or
(v, u) in restriction):
# SCORE FOR 'U' -> gaining 'v' as parent
old_cols = (u, ) + tuple(
p_dict[v]) # without 'v' as parent
mi_old = mutual_information(data[:, old_cols])
new_cols = old_cols + (v, ) # with 'v' as parent
mi_new = mutual_information(data[:, new_cols])
delta1 = nrow * (mi_old - mi_new)
# SCORE FOR 'V' -> losing 'u' as parent
old_cols = (v, ) + tuple(p_dict[v]) # with 'u' as parent
mi_old = mutual_information(data[:, old_cols])
new_cols = tuple([u for i in old_cols
if i != u]) # without 'u' as parent
mi_new = mutual_information(data[:, new_cols])
delta2 = nrow * (mi_old - mi_new)
# COMBINED DELTA-SCORES
delta_score = delta1 + delta2
if delta_score > max_delta:
if debug:
print('Improved Arc Reversal: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Reversal'
max_arc = (u, v)
### DETERMINE IF/WHERE IMPROVEMENT WAS MADE ###
if max_delta != 0:
improvement = True
u, v = max_arc
if max_operation == 'Addition':
if debug:
print('ADDING: ', max_arc, '\n')
c_dict[u].append(v)
p_dict[v].append(u)
elif max_operation == 'Deletion':
if debug:
print('DELETING: ', max_arc, '\n')
c_dict[u].remove(v)
p_dict[v].remove(u)
elif max_operation == 'Reversal':
if debug:
print('REVERSING: ', max_arc, '\n')
c_dict[u].remove(v)
p_dict[v].remove(u)
c_dict[v].append(u)
p_dict[u].append(v)
else:
if debug:
print('No Improvement on Iter: ', _iter)
#### RESTART WITH RANDOM MOVES ####
if _restarts < R:
improvement = True # make another pass of hill climbing
_iter = 0 # reset iterations
if debug:
print('Restart - ', _restarts)
_restarts += 1
for _ in range(M):
# 0 = Addition, 1 = Deletion, 2 = Reversal
operation = np.random.choice([0, 1, 2])
if operation == 0:
u, v = np.random.choice(list(bn.nodes()),
size=2,
replace=False)
# IF EDGE DOESN'T EXIST, ADD IT
if u not in p_dict[
v] and u != v and not would_cause_cycle(
c_dict, u,
v) and (init_nodes is None
or not (v in init_nodes)) and (
restriction is None or
(u, v) in restriction):
if debug:
print('RESTART - ADDING: ', (u, v))
c_dict[u].append(v)
p_dict[v].append(u)
elif operation == 1:
u, v = np.random.choice(list(bn.nodes()),
size=2,
replace=False)
# IF EDGE EXISTS, DELETE IT
if u in p_dict[v]:
if debug:
print('RESTART - DELETING: ', (u, v))
c_dict[u].remove(v)
p_dict[v].remove(u)
elif operation == 2:
u, v = np.random.choice(list(bn.nodes()),
size=2,
replace=False)
# IF EDGE EXISTS, REVERSE IT
if u in p_dict[v] and not would_cause_cycle(
c_dict, v, u, reverse=True) and (
init_nodes is None or not (u in init_nodes)
) and (restriction is None or
(v, u) in restriction):
if debug:
print('RESTART - REVERSING: ', (u, v))
c_dict[u].remove(v)
p_dict[v].remove(u)
c_dict[v].append(u)
p_dict[u].append(v)
break
### TEST FOR MAX ITERATION ###
_iter += 1
if _iter > max_iter:
if debug:
print('Max Iteration Reached')
break
bn = BayesNet(c_dict)
return bn
def iamb(data, alpha=0.05, feature_selection=None, debug=False):
"""
IAMB Algorithm for learning the structure of a
Discrete Bayesian Network from data.
Arguments
---------
*data* : a nested numpy array
*alpha* : a float
The type II error rate.
*feature_selection* : None or a string
Whether to use IAMB as a structure learning
or feature selection algorithm.
Returns
-------
*bn* : a BayesNet object or
*mb* : the markov blanket of a node
Effects
-------
None
Notes
-----
- Works but there are definitely some bugs.
Speed Test:
*** 5 vars, 624 obs ***
- 196 ms
"""
n_rv = data.shape[1]
Mb = dict([(rv,[]) for rv in range(n_rv)])
if feature_selection is None:
_T = range(n_rv)
else:
assert (not isinstance(feature_selection, list)), 'feature_selection must be only one value'
_T = [feature_selection]
# LEARN MARKOV BLANKET
for T in _T:
V = set(range(n_rv)) - {T}
Mb_change=True
# GROWING PHASE
while Mb_change:
Mb_change = False
# find X_max in V-Mb(T)-{T} that maximizes
# mutual information of X,T|Mb(T)
# i.e. max of mi_test(data[:,(X,T,Mb(T))])
max_val = -1
max_x = None
for X in V - set(Mb[T]) - {T}:
cols = (X,T)+tuple(Mb[T])
mi_val = mi_test(data[:,cols],test=False)
if mi_val > max_val:
max_val = mi_val
max_x = X
# if Xmax is dependent on T given Mb(T)
cols = (max_x,T) + tuple(Mb[T])
if max_x is not None and are_independent(data[:,cols]):
Mb[T].append(X)
Mb_change = True
if debug:
print('Adding %s to MB of %s' % (str(X), str(T)))
# SHRINKING PHASE
for X in Mb[T]:
# if x is independent of t given Mb(T) - {x}
cols = (X,T) + tuple(set(Mb[T]) - {X})
if are_independent(data[:,cols],alpha):
Mb[T].remove(X)
if debug:
print('Removing %s from MB of %s' % (str(X), str(T)))
if feature_selection is None:
# RESOLVE GRAPH STRUCTURE
edge_dict = resolve_markov_blanket(Mb, data)
if debug:
print('Unoriented edge dict:\n %s' % str(edge_dict))
print('MB: %s' % str(Mb))
# ORIENT EDGES
oriented_edge_dict = orient_edges_gs2(edge_dict,Mb,data,alpha)
if debug:
print('Oriented edge dict:\n %s' % str(oriented_edge_dict))
# CREATE BAYESNET OBJECT
value_dict = dict(zip(range(data.shape[1]),
[list(np.unique(col)) for col in data.T]))
bn=BayesNet(oriented_edge_dict,value_dict)
return bn
else:
return Mb[_T]
def tabu(data, k=5, metric='AIC', max_iter=100, debug=False, restriction=None):
"""
Tabu search for score-based structure learning.
The algorithm maintains a list called "tabu_list",
which consists of 3-tuples, where the first two
elements constitute the edge which is tabued, and
the third element is a string - either 'Addition',
'Deletion', or 'Reversal' denoting the operation
associated with the edge.
Arguments
---------
*data* : a nested numpy array
The data from which the Bayesian network
structure will be learned.
*metric* : a string
Which score metric to use.
Options:
- AIC
- BIC / MDL
- LL (log-likelihood)
*max_iter* : an integer
The maximum number of iterations of the
hill-climbing algorithm to run. Note that
the algorithm will terminate on its own if no
improvement is made in a given iteration.
*debug* : boolean
Whether to print(the scores/moves of the)
algorithm as its happening.
*restriction* : a list of 2-tuples
For MMHC algorithm, the list of allowable edge additions.
Returns
-------
*bn* : a BayesNet object
"""
nrow = data.shape[0]
ncol = data.shape[1]
names = range(ncol)
# INITIALIZE NETWORK W/ NO EDGES
# maintain children and parents dict for fast lookups
c_dict = dict([(n,[]) for n in names])
p_dict = dict([(n,[]) for n in names])
# COMPUTE INITIAL LIKELIHOOD SCORE
value_dict = dict([(n, np.unique(data[:,i])) for i,n in enumerate(names)])
bn = BayesNet(c_dict)
mle_estimator(bn, data)
max_score = info_score(bn, nrow, metric)
tabu_list = [None]*k
_iter = 0
improvement = True
while improvement:
improvement = False
max_delta = 0
if debug:
print('ITERATION: ' , _iter)
### TEST ARC ADDITIONS ###
for u in bn.nodes():
for v in bn.nodes():
# CHECK TABU LIST - can't delete an addition on the tabu list
if (u,v,'Deletion') not in tabu_list:
# CHECK EDGE EXISTENCE AND CYCLICITY
if v not in c_dict[u] and u!=v and not would_cause_cycle(c_dict, u, v):
# FOR MMHC ALGORITHM -> Edge Restrictions
if restriction is None or (u,v) in restriction:
# SCORE FOR 'V' -> gaining a parent
old_cols = (v,) + tuple(p_dict[v]) # without 'u' as parent
mi_old = mutual_information(data[:,old_cols])
new_cols = old_cols + (u,) # with'u' as parent
mi_new = mutual_information(data[:,new_cols])
delta_score = nrow * (mi_old - mi_new)
if delta_score > max_delta:
if debug:
print('Improved Arc Addition: ' , (u,v))
print('Delta Score: ' , delta_score)
max_delta = delta_score
max_operation = 'Addition'
max_arc = (u,v)
### TEST ARC DELETIONS ###
for u in bn.nodes():
for v in bn.nodes():
# CHECK TABU LIST - can't add back a deletion on the tabu list
if (u,v,'Addition') not in tabu_list:
if v in c_dict[u]:
# SCORE FOR 'V' -> losing a parent
old_cols = (v,) + tuple(p_dict[v]) # with 'u' as parent
mi_old = mutual_information(data[:,old_cols])
new_cols = tuple([i for i in old_cols if i != u]) # without 'u' as parent
mi_new = mutual_information(data[:,new_cols])
delta_score = nrow * (mi_old - mi_new)
if delta_score > max_delta:
if debug:
print('Improved Arc Deletion: ' , (u,v))
print('Delta Score: ' , delta_score)
max_delta = delta_score
max_operation = 'Deletion'
max_arc = (u,v)
### TEST ARC REVERSALS ###
for u in bn.nodes():
for v in bn.nodes():
# CHECK TABU LIST - can't reverse back a reversal on the tabu list
if (u,v,'Reversal') not in tabu_list:
if v in c_dict[u] and not would_cause_cycle(c_dict,v,u, reverse=True):
# SCORE FOR 'U' -> gaining 'v' as parent
old_cols = (u,) + tuple(p_dict[v]) # without 'v' as parent
mi_old = mutual_information(data[:,old_cols])
new_cols = old_cols + (v,) # with 'v' as parent
mi_new = mutual_information(data[:,new_cols])
delta1 = nrow * (mi_old - mi_new)
# SCORE FOR 'V' -> losing 'u' as parent
old_cols = (v,) + tuple(p_dict[v]) # with 'u' as parent
mi_old = mutual_information(data[:,old_cols])
new_cols = tuple([u for i in old_cols if i != u]) # without 'u' as parent
mi_new = mutual_information(data[:,new_cols])
delta2 = nrow * (mi_old - mi_new)
# COMBINED DELTA-SCORES
delta_score = delta1 + delta2
if delta_score > max_delta:
if debug:
print('Improved Arc Reversal: ' , (u,v))
print('Delta Score: ' , delta_score)
max_delta = delta_score
max_operation = 'Reversal'
max_arc = (u,v)
### DETERMINE IF/WHERE IMPROVEMENT WAS MADE ###
if max_delta != 0:
improvement = True
u,v = max_arc
if max_operation == 'Addition':
if debug:
print('ADDING: ' , max_arc , '\n')
c_dict[u].append(v)
p_dict[v].append(u)
tabu_list[_iter % 5] = (u,v,'Addition')
elif max_operation == 'Deletion':
if debug:
print('DELETING: ' , max_arc , '\n')
c_dict[u].remove(v)
p_dict[v].remove(u)
tabu_list[_iter % 5] = (u,v,'Deletion')
elif max_operation == 'Reversal':
if debug:
print('REVERSING: ' , max_arc, '\n')
c_dict[u].remove(v)
p_dict[v].remove(u)
c_dict[v].append(u)
p_dict[u].append(v)
tabu_list[_iter % 5] = (u,v,'Reversal')
else:
if debug:
print('No Improvement on Iter: ' , _iter)
### TEST FOR MAX ITERATION ###
_iter += 1
if _iter > max_iter:
if debug:
print('Max Iteration Reached')
break
bn = BayesNet(c_dict)
return bn
def hc(data,
metric='MI',
max_iter=200,
debug=False,
init_nodes=None,
restriction=None,
init_edges=None,
remove_geo_edges=True,
black_list=None):
"""
Greedy Hill Climbing search proceeds by choosing the move
which maximizes the increase in fitness of the
network at the current step. It continues until
it reaches a point where there does not exist any
feasible single move that increases the network fitness.
It is called "greedy" because it simply does what is
best at the current iteration only, and thus does not
look ahead to what may be better later on in the search.
For computational saving, a Priority Queue (python's heapq)
can be used to maintain the best operators and reduce the
complexity of picking the best operator from O(n^2) to O(nlogn).
This works by maintaining the heapq of operators sorted by their
delta score, and each time a move is made, we only have to recompute
the O(n) delta-scores which were affected by the move. The rest of
the operator delta-scores are not affected.
For additional computational efficiency, we can cache the
sufficient statistics for various families of distributions -
therefore, computing the mutual information for a given family
only needs to happen once.
The possible moves are the following:
- add edge
- delete edge
- invert edge
Arguments
---------
*data* : pd.DataFrame
The data from which the Bayesian network
structure will be learned.
*metric* : a string
Which score metric to use.
Options:
- AIC
- BIC / MDL
- LL (log-likelihood)
*max_iter* : an integer
The maximum number of iterations of the
hill-climbing algorithm to run. Note that
the algorithm will terminate on its own if no
improvement is made in a given iteration.
*debug* : boolean
Whether to print(the scores/moves of the)
algorithm as its happening.
*init_nodes* : a list of initialize nodes (number of nodes according to the dataset)
*restriction* : a list of 2-tuples
For MMHC algorithm, the list of allowable edge additions.
Returns
-------
*bn* : a BayesNet object
"""
nrow = data.shape[0]
ncol = data.shape[1]
names = range(ncol)
# INITIALIZE NETWORK W/ NO EDGES
# maintain children and parents dict for fast lookups
c_dict = dict([(n, []) for n in names])
p_dict = dict([(n, []) for n in names])
if init_edges:
for edge in init_edges:
c_dict[edge[0]].append(edge[1])
p_dict[edge[1]].append(edge[0])
bn = BayesNet(c_dict)
mutual_information = mi_gauss
if metric == 'BIC':
mutual_information = BIC_local
if metric == 'AIC':
mutual_information = AIC_local
if metric == 'LL':
mutual_information = log_lik_local
data = data.values
cache = dict()
_iter = 0
improvement = True
while improvement:
improvement = False
max_delta = 0
if debug:
print('ITERATION: ', _iter)
### TEST ARC ADDITIONS ###
for u in bn.nodes():
for v in bn.nodes():
if v not in c_dict[u] and u != v and not would_cause_cycle(
c_dict, u, v) and len(p_dict[v]) != 3:
# FOR MMHC ALGORITHM -> Edge Restrictions
if (init_nodes is None or not (v in init_nodes)) and (
restriction is None or
(u, v) in restriction) and (black_list is None or not (
(u, v) in black_list)):
# SCORE FOR 'V' -> gaining a parent
old_cols = (v, ) + tuple(
p_dict[v]) # without 'u' as parent
if not old_cols in cache:
cache[old_cols] = mutual_information(
data[:, old_cols])
mi_old = cache[old_cols]
# mi_old = mutual_information(data[:,old_cols])
new_cols = old_cols + (u, ) # with'u' as parent
if not new_cols in cache:
cache[new_cols] = mutual_information(
data[:, new_cols])
mi_new = cache[new_cols]
# mi_new = mutual_information(data[:,new_cols])
delta_score = nrow * (mi_old - mi_new)
if delta_score > max_delta:
if debug:
print('Improved Arc Addition: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Addition'
max_arc = (u, v)
# ### TEST ARC DELETIONS ###
for u in bn.nodes():
for v in bn.nodes():
if v in c_dict[u]:
# SCORE FOR 'V' -> losing a parent
old_cols = (v, ) + tuple(p_dict[v]) # with 'u' as parent
if not old_cols in cache:
cache[old_cols] = mutual_information(data[:, old_cols])
mi_old = cache[old_cols]
# mi_old = mutual_information(data[:,old_cols])
new_cols = tuple([i for i in old_cols
if i != u]) # without 'u' as parent
if not new_cols in cache:
cache[new_cols] = mutual_information(data[:, new_cols])
mi_new = cache[new_cols]
# mi_new = mutual_information(data[:,new_cols])
delta_score = nrow * (mi_old - mi_new)
if (delta_score > max_delta):
if init_edges == None:
if debug:
print('Improved Arc Deletion: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Deletion'
max_arc = (u, v)
else:
if (u, v) in init_edges:
if remove_geo_edges:
if debug:
print('Improved Arc Deletion: ',
(u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Deletion'
max_arc = (u, v)
else:
if debug:
print('Improved Arc Deletion: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Deletion'
max_arc = (u, v)
# ### TEST ARC REVERSALS ###
for u in bn.nodes():
for v in bn.nodes():
if v in c_dict[u] and not would_cause_cycle(
c_dict, v, u,
reverse=True) and len(p_dict[u]) != 3 and (
init_nodes is None or not (u in init_nodes)) and (
restriction is None or
(v, u) in restriction) and (
black_list is None
or not ((v, u) in black_list)):
# SCORE FOR 'U' -> gaining 'v' as parent
old_cols = (u, ) + tuple(
p_dict[v]) # without 'v' as parent
if not old_cols in cache:
cache[old_cols] = mutual_information(data[:, old_cols])
mi_old = cache[old_cols]
# mi_old = mutual_information(data[:,old_cols])
new_cols = old_cols + (v, ) # with 'v' as parent
if not new_cols in cache:
cache[new_cols] = mutual_information(data[:, new_cols])
mi_new = cache[new_cols]
# mi_new = mutual_information(data[:,new_cols])
delta1 = -1 * nrow * (mi_old - mi_new)
# SCORE FOR 'V' -> losing 'u' as parent
old_cols = (v, ) + tuple(p_dict[v]) # with 'u' as parent
if not old_cols in cache:
cache[old_cols] = mutual_information(data[:, old_cols])
mi_old = cache[old_cols]
# mi_old = mutual_information(data[:,old_cols])
new_cols = tuple([u for i in old_cols
if i != u]) # without 'u' as parent
if not new_cols in cache:
cache[new_cols] = mutual_information(data[:, new_cols])
mi_new = cache[new_cols]
# mi_new = mutual_information(data[:,new_cols])
delta2 = nrow * (mi_old - mi_new)
# COMBINED DELTA-SCORES
delta_score = delta1 + delta2
if (delta_score > max_delta):
if init_edges == None:
if debug:
print('Improved Arc Reversal: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Reversal'
max_arc = (u, v)
else:
if (u, v) in init_edges:
if remove_geo_edges:
if debug:
print('Improved Arc Reversal: ',
(u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Reversal'
max_arc = (u, v)
else:
if debug:
print('Improved Arc Reversal: ', (u, v))
print('Delta Score: ', delta_score)
max_delta = delta_score
max_operation = 'Reversal'
max_arc = (u, v)
if max_delta != 0:
improvement = True
u, v = max_arc
if max_operation == 'Addition':
if debug:
print('ADDING: ', max_arc, '\n')
c_dict[u].append(v)
p_dict[v].append(u)
elif max_operation == 'Deletion':
if debug:
print('DELETING: ', max_arc, '\n')
c_dict[u].remove(v)
p_dict[v].remove(u)
elif max_operation == 'Reversal':
if debug:
print('REVERSING: ', max_arc, '\n')
c_dict[u].remove(v)
p_dict[v].remove(u)
c_dict[v].append(u)
p_dict[u].append(v)
else:
if debug:
print('No Improvement on Iter: ', _iter)
### TEST FOR MAX ITERATION ###
_iter += 1
if _iter > max_iter:
if debug:
print('Max Iteration Reached')
break
bn = BayesNet(c_dict)
return bn
def pc(data, alpha=0.05):
"""
Path Condition algorithm for structure learning. This is a
good test, but has some issues with test reliability when
the size of the dataset is small. The Necessary Path
Condition (NPC) algorithm can solve these problems.
Arguments
---------
*bn* : a BayesNet object
The object we wish to modify. This can be a competely
empty BayesNet object, in which case the structure info
will be set. This can be a BayesNet object with already
initialized structure/params, in which case the structure
will be overwritten and the parameters will be cleared.
*data* : a nested numpy array
The data from which we will learn -> will code for
pandas dataframe after numpy works
Returns
-------
*bn* : a BayesNet object
The network created from the learning procedure, with
the nodes/edges initialized/changed
Effects
-------
None
Notes
-----
Speed Test:
** 5 vars, 624 obs ***
- 90.9 ms
"""
n_rv = data.shape[1]
##### FIND EDGES #####
value_dict = dict(zip(range(n_rv),
[list(np.unique(col)) for col in data.T]))
edge_dict = dict([(i,[j for j in range(n_rv) if i!=j]) for i in range(n_rv)])
block_dict = dict([(i,[]) for i in range(n_rv)])
stop = False
i = 1
while not stop:
for x in range(n_rv):
for y in edge_dict[x]:
if i == 0:
pval_xy_z = mi_test(data[:,(x,y)])
if pval_xy_z > alpha:
if y in edge_dict[x]:
edge_dict[x].remove(y)
edge_dict[y].remove(x)
else:
for z in itertools.combinations(edge_dict[x],i):
if y not in z:
cols = (x,y) + z
pval_xy_z = mi_test(data[:,cols])
# if I(X,Y | Z) = TRUE
if pval_xy_z > alpha:
block_dict[x] = {y:z}
block_dict[y] = {x:z}
if y in edge_dict[x]:
edge_dict[x].remove(y)
edge_dict[y].remove(x)
i += 1
stop = True
for x in range(n_rv):
if (len(edge_dict[x]) > i-1):
stop = False
break
# ORIENT EDGES (from collider set)
directed_edge_dict = orient_edges_CS(edge_dict,block_dict)
# CREATE BAYESNET OBJECT
bn=BayesNet(directed_edge_dict,value_dict)
return bn
def gs(data, alpha=0.05, feature_selection=None, debug=False):
"""
Perform growshink algorithm over dataset to learn
Bayesian network structure.
This algorithm is clearly a good candidate for
numba JIT compilation...
STEPS
-----
1. Compute Markov Blanket
2. Compute Graph Structure
3. Orient Edges
4. Remove Cycles
5. Reverse Edges
6. Propagate Directions
Arguments
---------
*data* : a nested numpy array
Data from which you wish to learn structure
*alpha* : a float
Type I error rate for independence test
Returns
-------
*bn* : a BayesNet object
Effects
-------
None
Notes
-----
Speed Test:
*** 5 variables, 624 observations ***
- 63.7 ms
"""
n_rv = data.shape[1]
data, value_dict = replace_strings(data, return_values=True)
if feature_selection is None:
_T = range(n_rv)
else:
assert (not isinstance(feature_selection, list)
), 'feature_selection must be only one value'
_T = [feature_selection]
# STEP 1 : COMPUTE MARKOV BLANKETS
Mb = dict([(rv, []) for rv in range(n_rv)])
for X in _T:
S = []
grow_condition = True
while grow_condition:
grow_condition = False
for Y in range(n_rv):
if X != Y and Y not in S:
# if there exists some Y such that Y is dependent on X given S,
# add Y to S
cols = (X, Y) + tuple(S)
pval = mi_test(data[:, cols])
if pval < alpha: # dependent
grow_condition = True # dependent -> continue searching
S.append(Y)
shrink_condition = True
while shrink_condition:
TEMP_S = []
shrink_condition = False
for Y in S:
s_copy = copy(S)
s_copy.remove(Y) # condition on S-{Y}
# if X independent of Y given S-{Y}, leave Y out
# if X dependent of Y given S-{Y}, keep it in
cols = (X, Y) + tuple(s_copy)
pval = mi_test(data[:, cols])
if pval < alpha: # dependent
TEMP_S.append(Y)
else: # independent -> condition searching
shrink_condition = True
Mb[X] = TEMP_S
if debug:
print('Markov Blanket for %s : %s' % (X, str(TEMP_S)))
if feature_selection is None:
# STEP 2: COMPUTE GRAPH STRUCTURE
# i.e. Resolve Markov Blanket
edge_dict = resolve_markov_blanket(Mb, data)
if debug:
print('Unoriented edge dict:\n %s' % str(edge_dict))
# STEP 3: ORIENT EDGES
oriented_edge_dict = orient_edges_MB(edge_dict, Mb, data, alpha)
if debug:
print('Oriented edge dict:\n %s' % str(oriented_edge_dict))
# CREATE BAYESNET OBJECT
bn = BayesNet(oriented_edge_dict, value_dict)
return bn
else:
return Mb[_T]