def MCMC_SFT(SFTNet, data, N, z0, T):
"""
SFTNet : SFTNet instance
The net to do MCMC over
data : list
The data as outputted by gen_data
N : int
The number of MCMC proposals
z0 : dict
Initial guess for infection times. Keys are node names
values are floats
T : int
How long the process ran for.
"""
n = 1
# initiate step
prob_mod = lambda x : prob_model_given_data(SFTNet, data[1], x,
data[2], data[3], T)
# lambda function that calls prob_model_given_data for
# specified infection times
p0 = prob_mod(z0)
# Initiial probability
t0 =np.asarray(z0.values())
za, zc, zb, zd = list(t0)
# actual times
time_samples = []
# container for samples
probs = []
# container for probabilities
con_cdf = convoluted_cdf_func(20000 **.5, 0, 50)
while n < N:
za = 0
zb = z0['B'] + np.random.normal() * 100
zc = min(z0['C'] + np.random.normal() * 100,
zb + np.random.random()* 50)
zd = min(zb, zc) + np.random.random() * 50
z1 = dict(zip(['A', 'B', 'C', 'D'], [za, zb,zc, zd]))
p1 = prob_mod(z1)
if min(z1.values()) >= 0:
log_q_ratio = qij_over_qji(z0,z1, con_cdf, convoluted_pdf_func)
if (p1 - p0 + log_q_ratio >
np.log(np.random.random())):
print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n'
t0 = z1.values()
p0 = p1
z0 = z1
time_samples.append(t0)
probs.append(p0)
n += 1
return time_samples, probs, z1.keys()
def MCMC_SFT(SFTNet, data, N, z0, T):
"""
SFTNet : SFTNet instance
The net to do MCMC over
data : list
The data as outputted by gen_data
N : int
The number of MCMC proposals
z0 : dict
Initial guess for infection times. Keys are node names
values are floats
T : int
How long the process ran for.
"""
n = 1
# initiate step
prob_mod = lambda x: prob_model_given_data(SFTNet, data[1], x, data[2],
data[3], T)
# lambda function that calls prob_model_given_data for
# specified infection times
p0 = prob_mod(z0)
# Initiial probability
t0 = np.asarray(z0.values())
za, zc, zb, zd = list(t0)
# actual times
time_samples = []
# container for samples
probs = []
# container for probabilities
con_cdf = convoluted_cdf_func(20000**.5, 0, 50)
while n < N:
za = 0
zb = z0['B'] + np.random.normal() * 100
zc = min(z0['C'] + np.random.normal() * 100,
zb + np.random.random() * 50)
zd = min(zb, zc) + np.random.random() * 50
z1 = dict(zip(['A', 'B', 'C', 'D'], [za, zb, zc, zd]))
p1 = prob_mod(z1)
if min(z1.values()) >= 0:
log_q_ratio = qij_over_qji(z0, z1, con_cdf, convoluted_pdf_func)
if (p1 - p0 + log_q_ratio > np.log(np.random.random())):
print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n'
t0 = z1.values()
p0 = p1
z0 = z1
time_samples.append(t0)
probs.append(p0)
n += 1
return time_samples, probs, z1.keys()
def Direct_Sample(SFTNet, data, num_samples, T, s0):
net = copy.deepcopy(SFTNet)
logn_fact = gen_logn_fact(data)
n = 1
nodes_to_change = [nd for nd in net.node_names if s0[nd] == 'normal']
nodes_no_change = [nd for nd in net.node_names if s0[nd] == 'infected']
prob_no_attacker = prob_model_no_attacker(net, data, T)
prob_true_value = prob_model_given_data(net, data, data[-1], T, logn_fact,
s0)
numattackers = len(nodes_no_change)
prob_mod = lambda x: prob_model_given_data(net, data, x, T, logn_fact, s0)
probs = []
while n < num_samples:
t = 0
for nd in net.node_names:
net.node_dict[nd].state = s0[nd]
times = {nd: 0 for nd in nodes_no_change}
# Corresponds to correct order
while t < T:
infected = [nd.name for nd in net.nodes if nd.state == 'infected']
at_risk = set(
chain(*[net.node_dict[nd].sends_to
for nd in infected])) - set(infected)
if len(at_risk) == 0:
break
at_risk_ix = [net.node_names.index(nd) for nd in at_risk]
mt_rates = np.sum(net.get_mal_trans()[:, at_risk_ix], axis=0)
#print at_risk, mt_rates, infected, n
r_rate = np.sum(mt_rates)
t += np.random.exponential(scale=1 / r_rate)
if t < T:
next_infected = np.random.choice(list(at_risk),
p=mt_rates / sum(mt_rates))
times[next_infected] = t
net.node_dict[next_infected].state = 'infected'
#print times, n
probs.append(prob_mod(times)[1])
n += 1
e_probs = np.exp(probs)
return np.log(np.mean(e_probs)), e_probs
def Direct_Sample(SFTNet, data, num_samples, T, s0):
net = copy.deepcopy(SFTNet)
logn_fact = gen_logn_fact(data)
n = 1
nodes_to_change = [nd for nd in net.node_names if s0[nd] == 'normal' ]
nodes_no_change = [nd for nd in net.node_names if s0[nd] == 'infected']
prob_no_attacker = prob_model_no_attacker(net, data, T)
prob_true_value = prob_model_given_data(net, data, data[-1], T, logn_fact, s0)
numattackers = len(nodes_no_change)
prob_mod = lambda x : prob_model_given_data(net, data, x, T,
logn_fact, s0)
probs = []
while n < num_samples:
t = 0
for nd in net.node_names:
net.node_dict[nd].state = s0[nd]
times = {nd: 0 for nd in nodes_no_change}
# Corresponds to correct order
while t<T :
infected = [nd.name for nd in net.nodes if nd.state =='infected']
at_risk = set(chain(*[net.node_dict[nd].sends_to for nd in infected])) - set(infected)
if len(at_risk) == 0:
break
at_risk_ix = [net.node_names.index(nd) for nd in at_risk]
mt_rates = np.sum(net.get_mal_trans()[:, at_risk_ix], axis=0)
#print at_risk, mt_rates, infected, n
r_rate = np.sum(mt_rates)
t += np.random.exponential(scale=1/r_rate)
if t<T:
next_infected = np.random.choice(list(at_risk), p = mt_rates/sum(mt_rates))
times[next_infected] = t
net.node_dict[next_infected].state = 'infected'
#print times, n
probs.append(prob_mod(times)[1])
n+=1
e_probs = np.exp(probs)
return np.log(np.mean(e_probs)), e_probs
def MCMC_MH(SFTNet, data, s0, N, T, proposal_var=100, print_jumps=False):
# TODO Need to profile this
# TODO: Need to make this more general. Not trivial
# TODO : Add sample from possible node orderings
"""
Performs MCMC integration using Metropolis Hastings. Returns
the sampled times, their associated probabilities and the
associated likelihood value. This method corresponds to David's
"half-way" approach in the 2nd version of the ASCII where we
sample (accept / reject) according to P(z | attacker) and then
take the average of P(data | z, attacker) of the accepted
values.
SFTNet : SFTNet instance
The net to do MCMC over
data : list
The data as outputted by gen_data
N : int
The number of MCMC proposals
s0 : dict
State of the net at t=0
T : int
How long the process ran for.
"""
logn_fact = gen_logn_fact(data)
n = 1
nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == "normal"]
nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == "infected"]
prob_no_attacker = prob_model_no_attacker(SFTNet, data, T)
prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T, logn_fact)
prob_mod = lambda x: prob_model_given_data(SFTNet, data, x, T, logn_fact)
guess_times = np.sort(np.random.random(size=len(nodes_to_change)) * T)
z0 = dict(zip(nodes_to_change, guess_times))
for nd in nodes_no_change:
z0[nd] = 0
# lambda function that calls prob_model_given_data for
# specified infection times
p0 = prob_mod(z0)
# Initiial probability
# actual times
time_samples = {node.name: [] for node in SFTNet.nodes}
# container for samples
probs = []
# container for probabilities
z1 = copy.deepcopy(z0)
while n < N:
# if np.random.random() < alpha:
# order = random.sample(orderings, 1)[0]
for nd in nodes_to_change:
z1[nd] = z0[nd] + np.random.normal() * proposal_var
p1 = prob_mod(z1)
if p1[0] - p0[0] > np.log(np.random.random()):
if print_jumps:
print "A Jump at, ", n, "to ", z1, "with prob", p1, "\n"
p0 = p1
z0 = copy.deepcopy(z1)
for key, val in z0.iteritems():
time_samples[key].append(val)
probs.append(p0[:2])
n += 1
probs = np.asarray(probs)
out_ar = np.hstack((np.asarray(time_samples.values()).T, probs))
columns = copy.copy(time_samples.keys())
columns.append("P(z | attacker)")
columns.append("P(data | z, attacker)")
out = pandas.DataFrame(out_ar, columns=columns)
mcmc_results = Results(out, data[-1], prob_no_attacker, prob_true_value, data, metropolis=True)
return mcmc_results
def MCMC_MH(SFTNet, data, s0, N, T, proposal_var=100, print_jumps=False):
# TODO Need to profile this
# TODO: Need to make this more general. Not trivial
# TODO : Add sample from possible node orderings
"""
Performs MCMC integration using Metropolis Hastings. Returns
the sampled times, their associated probabilities and the
associated likelihood value. This method corresponds to David's
"half-way" approach in the 2nd version of the ASCII where we
sample (accept / reject) according to P(z | attacker) and then
take the average of P(data | z, attacker) of the accepted
values.
SFTNet : SFTNet instance
The net to do MCMC over
data : list
The data as outputted by gen_data
N : int
The number of MCMC proposals
s0 : dict
State of the net at t=0
T : int
How long the process ran for.
"""
logn_fact = gen_logn_fact(data)
n = 1
nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == 'normal']
nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == 'infected']
prob_no_attacker = prob_model_no_attacker(SFTNet, data, T)
prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T,
logn_fact)
prob_mod = lambda x: prob_model_given_data(SFTNet, data, x, T, logn_fact)
guess_times = np.sort(np.random.random(size=len(nodes_to_change)) * T)
z0 = dict(zip(nodes_to_change, guess_times))
for nd in nodes_no_change:
z0[nd] = 0
# lambda function that calls prob_model_given_data for
# specified infection times
p0 = prob_mod(z0)
# Initiial probability
# actual times
time_samples = {node.name: [] for node in SFTNet.nodes}
# container for samples
probs = []
# container for probabilities
z1 = copy.deepcopy(z0)
while n < N:
#if np.random.random() < alpha:
# order = random.sample(orderings, 1)[0]
for nd in nodes_to_change:
z1[nd] = z0[nd] + np.random.normal() * proposal_var
p1 = prob_mod(z1)
if (p1[0] - p0[0] > np.log(np.random.random())):
if print_jumps:
print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n'
p0 = p1
z0 = copy.deepcopy(z1)
for key, val in z0.iteritems():
time_samples[key].append(val)
probs.append(p0[:2])
n += 1
probs = np.asarray(probs)
out_ar = np.hstack((np.asarray(time_samples.values()).T, probs))
columns = copy.copy(time_samples.keys())
columns.append('P(z | attacker)')
columns.append('P(data | z, attacker)')
out = pandas.DataFrame(out_ar, columns=columns)
mcmc_results = Results(out,
data[-1],
prob_no_attacker,
prob_true_value,
data,
metropolis=True)
return mcmc_results
def MCMC_sequence(SFTNet, data, s0, N, T, proposal_var=100, print_jumps=False, alpha=1):
# TODO Need to profile this
# TODO: Need to make this more general. Not trivial
# TODO : Add sample from possible node orderings
"""
Performs MCMC integration using Metropolis Hastings. Returns
the sampled times, their associated probabilities and the
associated likelihood value. This method corresponds to David's
"half-way" approach in the 2nd version of the ASCII where we
sample (accept / reject) according to P(z | attacker) and then
take the average of P(data | z, attacker) of the accepted
values.
SFTNet : SFTNet instance
The net to do MCMC over
data : list
The data as outputted by gen_data
N : int
The number of MCMC proposals
s0 : dict
State of the net at t=0
T : int
How long the process ran for.
"""
logn_fact = gen_logn_fact(data)
n = 1
nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == 'normal' ]
nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == 'infected']
prob_no_attacker = prob_model_no_attacker(SFTNet, data, T)
prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T, logn_fact)
numattackers = len(nodes_no_change)
prob_mod = lambda x : prob_model_given_data(SFTNet, data, x, T,
logn_fact)
guess_times = np.sort(np.random.random(size=len(nodes_to_change))*T)
z0 = dict(zip(nodes_to_change, guess_times))
for nd in nodes_no_change:
z0[nd] = 0
order = sorted(z0.iterkeys(), key = lambda k: z0[k])
# lambda function that calls prob_model_given_data for
# specified infection times
p0 = prob_mod(z0)
# Initiial probability
# actual times
time_samples = {node.name : [] for node in SFTNet.nodes}
# container for samples
probs = []
# container for probabilities
z1 = copy.deepcopy(z0)
orders = gen_orderings(SFTNet, s0)
state0 = ['infected'] * numattackers + ['normal'] * len(nodes_to_change)
while n < N:
z1 = dict(zip(nodes_no_change, [0] * numattackers))
last_infect = 0
state = copy.copy(state0)
if np.random.random() < alpha:
new_order = random.choice(orders)
switch_order = True
else :
switch_order = False
new_order = order
for nd in new_order[numattackers:]:
cross_s_ix = SFTNet.cross_S.index(state)
nd_ix = SFTNet.node_names.index(nd)
incoming_rate = np.sum(SFTNet.mal_trans_mats[cross_s_ix][:, nd_ix])
last_infect = last_infect + trunc_expon(incoming_rate, T-last_infect)
z1[nd] = last_infect
state[nd_ix] = 'infected'
p1 = prob_mod(z1)
# Possible change to 2
if (p1[2] -p0[2] > np.log(np.random.random())):
if print_jumps :
print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n'
if switch_order:
#print ' new order ', order, ' at ', n
p0 = p1
z0 = copy.deepcopy(z1)
order = new_order
for key, val in z0.iteritems():
time_samples[key].append(val)
for nd in nodes_to_change:
if nd not in z0.keys():
time_samples[nd].append(T)
probs.append(p0[:2])
n += 1
probs = np.asarray(probs)
out_ar = np.hstack((np.asarray(time_samples.values()).T, probs))
columns = copy.copy(time_samples.keys())
columns.append('P(z | attacker)')
columns.append('P(data | z, attacker)')
out = pandas.DataFrame(out_ar, columns = columns)
mcmc_results = Results(out, data[-1], prob_no_attacker,
prob_true_value, data, metropolis = True)
return mcmc_results
def MCMC_sequence(SFTNet,
data,
s0,
N,
T,
proposal_var=100,
print_jumps=False,
alpha=1):
# TODO Need to profile this
# TODO: Need to make this more general. Not trivial
# TODO : Add sample from possible node orderings
"""
Performs MCMC integration using Metropolis Hastings. Returns
the sampled times, their associated probabilities and the
associated likelihood value. This method corresponds to David's
"half-way" approach in the 2nd version of the ASCII where we
sample (accept / reject) according to P(z | attacker) and then
take the average of P(data | z, attacker) of the accepted
values.
SFTNet : SFTNet instance
The net to do MCMC over
data : list
The data as outputted by gen_data
N : int
The number of MCMC proposals
s0 : dict
State of the net at t=0
T : int
How long the process ran for.
"""
logn_fact = gen_logn_fact(data)
n = 1
nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == 'normal']
nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == 'infected']
prob_no_attacker = prob_model_no_attacker(SFTNet, data, T)
prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T,
logn_fact)
numattackers = len(nodes_no_change)
prob_mod = lambda x: prob_model_given_data(SFTNet, data, x, T, logn_fact)
guess_times = np.sort(np.random.random(size=len(nodes_to_change)) * T)
z0 = dict(zip(nodes_to_change, guess_times))
for nd in nodes_no_change:
z0[nd] = 0
order = sorted(z0.iterkeys(), key=lambda k: z0[k])
# lambda function that calls prob_model_given_data for
# specified infection times
p0 = prob_mod(z0)
# Initiial probability
# actual times
time_samples = {node.name: [] for node in SFTNet.nodes}
# container for samples
probs = []
# container for probabilities
z1 = copy.deepcopy(z0)
orders = gen_orderings(SFTNet, s0)
state0 = ['infected'] * numattackers + ['normal'] * len(nodes_to_change)
while n < N:
z1 = dict(zip(nodes_no_change, [0] * numattackers))
last_infect = 0
state = copy.copy(state0)
if np.random.random() < alpha:
new_order = random.choice(orders)
switch_order = True
else:
switch_order = False
new_order = order
for nd in new_order[numattackers:]:
cross_s_ix = SFTNet.cross_S.index(state)
nd_ix = SFTNet.node_names.index(nd)
incoming_rate = np.sum(SFTNet.mal_trans_mats[cross_s_ix][:, nd_ix])
last_infect = last_infect + trunc_expon(incoming_rate,
T - last_infect)
z1[nd] = last_infect
state[nd_ix] = 'infected'
p1 = prob_mod(z1)
# Possible change to 2
if (p1[2] - p0[2] > np.log(np.random.random())):
if print_jumps:
print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n'
if switch_order:
#print ' new order ', order, ' at ', n
p0 = p1
z0 = copy.deepcopy(z1)
order = new_order
for key, val in z0.iteritems():
time_samples[key].append(val)
for nd in nodes_to_change:
if nd not in z0.keys():
time_samples[nd].append(T)
probs.append(p0[:2])
n += 1
probs = np.asarray(probs)
out_ar = np.hstack((np.asarray(time_samples.values()).T, probs))
columns = copy.copy(time_samples.keys())
columns.append('P(z | attacker)')
columns.append('P(data | z, attacker)')
out = pandas.DataFrame(out_ar, columns=columns)
mcmc_results = Results(out,
data[-1],
prob_no_attacker,
prob_true_value,
data,
metropolis=True)
return mcmc_results