def MCMC_SFT(SFTNet, data, N, z0, T): """ SFTNet : SFTNet instance The net to do MCMC over data : list The data as outputted by gen_data N : int The number of MCMC proposals z0 : dict Initial guess for infection times. Keys are node names values are floats T : int How long the process ran for. """ n = 1 # initiate step prob_mod = lambda x : prob_model_given_data(SFTNet, data[1], x, data[2], data[3], T) # lambda function that calls prob_model_given_data for # specified infection times p0 = prob_mod(z0) # Initiial probability t0 =np.asarray(z0.values()) za, zc, zb, zd = list(t0) # actual times time_samples = [] # container for samples probs = [] # container for probabilities con_cdf = convoluted_cdf_func(20000 **.5, 0, 50) while n < N: za = 0 zb = z0['B'] + np.random.normal() * 100 zc = min(z0['C'] + np.random.normal() * 100, zb + np.random.random()* 50) zd = min(zb, zc) + np.random.random() * 50 z1 = dict(zip(['A', 'B', 'C', 'D'], [za, zb,zc, zd])) p1 = prob_mod(z1) if min(z1.values()) >= 0: log_q_ratio = qij_over_qji(z0,z1, con_cdf, convoluted_pdf_func) if (p1 - p0 + log_q_ratio > np.log(np.random.random())): print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n' t0 = z1.values() p0 = p1 z0 = z1 time_samples.append(t0) probs.append(p0) n += 1 return time_samples, probs, z1.keys()
def MCMC_SFT(SFTNet, data, N, z0, T): """ SFTNet : SFTNet instance The net to do MCMC over data : list The data as outputted by gen_data N : int The number of MCMC proposals z0 : dict Initial guess for infection times. Keys are node names values are floats T : int How long the process ran for. """ n = 1 # initiate step prob_mod = lambda x: prob_model_given_data(SFTNet, data[1], x, data[2], data[3], T) # lambda function that calls prob_model_given_data for # specified infection times p0 = prob_mod(z0) # Initiial probability t0 = np.asarray(z0.values()) za, zc, zb, zd = list(t0) # actual times time_samples = [] # container for samples probs = [] # container for probabilities con_cdf = convoluted_cdf_func(20000**.5, 0, 50) while n < N: za = 0 zb = z0['B'] + np.random.normal() * 100 zc = min(z0['C'] + np.random.normal() * 100, zb + np.random.random() * 50) zd = min(zb, zc) + np.random.random() * 50 z1 = dict(zip(['A', 'B', 'C', 'D'], [za, zb, zc, zd])) p1 = prob_mod(z1) if min(z1.values()) >= 0: log_q_ratio = qij_over_qji(z0, z1, con_cdf, convoluted_pdf_func) if (p1 - p0 + log_q_ratio > np.log(np.random.random())): print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n' t0 = z1.values() p0 = p1 z0 = z1 time_samples.append(t0) probs.append(p0) n += 1 return time_samples, probs, z1.keys()
def Direct_Sample(SFTNet, data, num_samples, T, s0): net = copy.deepcopy(SFTNet) logn_fact = gen_logn_fact(data) n = 1 nodes_to_change = [nd for nd in net.node_names if s0[nd] == 'normal'] nodes_no_change = [nd for nd in net.node_names if s0[nd] == 'infected'] prob_no_attacker = prob_model_no_attacker(net, data, T) prob_true_value = prob_model_given_data(net, data, data[-1], T, logn_fact, s0) numattackers = len(nodes_no_change) prob_mod = lambda x: prob_model_given_data(net, data, x, T, logn_fact, s0) probs = [] while n < num_samples: t = 0 for nd in net.node_names: net.node_dict[nd].state = s0[nd] times = {nd: 0 for nd in nodes_no_change} # Corresponds to correct order while t < T: infected = [nd.name for nd in net.nodes if nd.state == 'infected'] at_risk = set( chain(*[net.node_dict[nd].sends_to for nd in infected])) - set(infected) if len(at_risk) == 0: break at_risk_ix = [net.node_names.index(nd) for nd in at_risk] mt_rates = np.sum(net.get_mal_trans()[:, at_risk_ix], axis=0) #print at_risk, mt_rates, infected, n r_rate = np.sum(mt_rates) t += np.random.exponential(scale=1 / r_rate) if t < T: next_infected = np.random.choice(list(at_risk), p=mt_rates / sum(mt_rates)) times[next_infected] = t net.node_dict[next_infected].state = 'infected' #print times, n probs.append(prob_mod(times)[1]) n += 1 e_probs = np.exp(probs) return np.log(np.mean(e_probs)), e_probs
def Direct_Sample(SFTNet, data, num_samples, T, s0): net = copy.deepcopy(SFTNet) logn_fact = gen_logn_fact(data) n = 1 nodes_to_change = [nd for nd in net.node_names if s0[nd] == 'normal' ] nodes_no_change = [nd for nd in net.node_names if s0[nd] == 'infected'] prob_no_attacker = prob_model_no_attacker(net, data, T) prob_true_value = prob_model_given_data(net, data, data[-1], T, logn_fact, s0) numattackers = len(nodes_no_change) prob_mod = lambda x : prob_model_given_data(net, data, x, T, logn_fact, s0) probs = [] while n < num_samples: t = 0 for nd in net.node_names: net.node_dict[nd].state = s0[nd] times = {nd: 0 for nd in nodes_no_change} # Corresponds to correct order while t<T : infected = [nd.name for nd in net.nodes if nd.state =='infected'] at_risk = set(chain(*[net.node_dict[nd].sends_to for nd in infected])) - set(infected) if len(at_risk) == 0: break at_risk_ix = [net.node_names.index(nd) for nd in at_risk] mt_rates = np.sum(net.get_mal_trans()[:, at_risk_ix], axis=0) #print at_risk, mt_rates, infected, n r_rate = np.sum(mt_rates) t += np.random.exponential(scale=1/r_rate) if t<T: next_infected = np.random.choice(list(at_risk), p = mt_rates/sum(mt_rates)) times[next_infected] = t net.node_dict[next_infected].state = 'infected' #print times, n probs.append(prob_mod(times)[1]) n+=1 e_probs = np.exp(probs) return np.log(np.mean(e_probs)), e_probs
def MCMC_MH(SFTNet, data, s0, N, T, proposal_var=100, print_jumps=False): # TODO Need to profile this # TODO: Need to make this more general. Not trivial # TODO : Add sample from possible node orderings """ Performs MCMC integration using Metropolis Hastings. Returns the sampled times, their associated probabilities and the associated likelihood value. This method corresponds to David's "half-way" approach in the 2nd version of the ASCII where we sample (accept / reject) according to P(z | attacker) and then take the average of P(data | z, attacker) of the accepted values. SFTNet : SFTNet instance The net to do MCMC over data : list The data as outputted by gen_data N : int The number of MCMC proposals s0 : dict State of the net at t=0 T : int How long the process ran for. """ logn_fact = gen_logn_fact(data) n = 1 nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == "normal"] nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == "infected"] prob_no_attacker = prob_model_no_attacker(SFTNet, data, T) prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T, logn_fact) prob_mod = lambda x: prob_model_given_data(SFTNet, data, x, T, logn_fact) guess_times = np.sort(np.random.random(size=len(nodes_to_change)) * T) z0 = dict(zip(nodes_to_change, guess_times)) for nd in nodes_no_change: z0[nd] = 0 # lambda function that calls prob_model_given_data for # specified infection times p0 = prob_mod(z0) # Initiial probability # actual times time_samples = {node.name: [] for node in SFTNet.nodes} # container for samples probs = [] # container for probabilities z1 = copy.deepcopy(z0) while n < N: # if np.random.random() < alpha: # order = random.sample(orderings, 1)[0] for nd in nodes_to_change: z1[nd] = z0[nd] + np.random.normal() * proposal_var p1 = prob_mod(z1) if p1[0] - p0[0] > np.log(np.random.random()): if print_jumps: print "A Jump at, ", n, "to ", z1, "with prob", p1, "\n" p0 = p1 z0 = copy.deepcopy(z1) for key, val in z0.iteritems(): time_samples[key].append(val) probs.append(p0[:2]) n += 1 probs = np.asarray(probs) out_ar = np.hstack((np.asarray(time_samples.values()).T, probs)) columns = copy.copy(time_samples.keys()) columns.append("P(z | attacker)") columns.append("P(data | z, attacker)") out = pandas.DataFrame(out_ar, columns=columns) mcmc_results = Results(out, data[-1], prob_no_attacker, prob_true_value, data, metropolis=True) return mcmc_results
def MCMC_MH(SFTNet, data, s0, N, T, proposal_var=100, print_jumps=False): # TODO Need to profile this # TODO: Need to make this more general. Not trivial # TODO : Add sample from possible node orderings """ Performs MCMC integration using Metropolis Hastings. Returns the sampled times, their associated probabilities and the associated likelihood value. This method corresponds to David's "half-way" approach in the 2nd version of the ASCII where we sample (accept / reject) according to P(z | attacker) and then take the average of P(data | z, attacker) of the accepted values. SFTNet : SFTNet instance The net to do MCMC over data : list The data as outputted by gen_data N : int The number of MCMC proposals s0 : dict State of the net at t=0 T : int How long the process ran for. """ logn_fact = gen_logn_fact(data) n = 1 nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == 'normal'] nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == 'infected'] prob_no_attacker = prob_model_no_attacker(SFTNet, data, T) prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T, logn_fact) prob_mod = lambda x: prob_model_given_data(SFTNet, data, x, T, logn_fact) guess_times = np.sort(np.random.random(size=len(nodes_to_change)) * T) z0 = dict(zip(nodes_to_change, guess_times)) for nd in nodes_no_change: z0[nd] = 0 # lambda function that calls prob_model_given_data for # specified infection times p0 = prob_mod(z0) # Initiial probability # actual times time_samples = {node.name: [] for node in SFTNet.nodes} # container for samples probs = [] # container for probabilities z1 = copy.deepcopy(z0) while n < N: #if np.random.random() < alpha: # order = random.sample(orderings, 1)[0] for nd in nodes_to_change: z1[nd] = z0[nd] + np.random.normal() * proposal_var p1 = prob_mod(z1) if (p1[0] - p0[0] > np.log(np.random.random())): if print_jumps: print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n' p0 = p1 z0 = copy.deepcopy(z1) for key, val in z0.iteritems(): time_samples[key].append(val) probs.append(p0[:2]) n += 1 probs = np.asarray(probs) out_ar = np.hstack((np.asarray(time_samples.values()).T, probs)) columns = copy.copy(time_samples.keys()) columns.append('P(z | attacker)') columns.append('P(data | z, attacker)') out = pandas.DataFrame(out_ar, columns=columns) mcmc_results = Results(out, data[-1], prob_no_attacker, prob_true_value, data, metropolis=True) return mcmc_results
def MCMC_sequence(SFTNet, data, s0, N, T, proposal_var=100, print_jumps=False, alpha=1): # TODO Need to profile this # TODO: Need to make this more general. Not trivial # TODO : Add sample from possible node orderings """ Performs MCMC integration using Metropolis Hastings. Returns the sampled times, their associated probabilities and the associated likelihood value. This method corresponds to David's "half-way" approach in the 2nd version of the ASCII where we sample (accept / reject) according to P(z | attacker) and then take the average of P(data | z, attacker) of the accepted values. SFTNet : SFTNet instance The net to do MCMC over data : list The data as outputted by gen_data N : int The number of MCMC proposals s0 : dict State of the net at t=0 T : int How long the process ran for. """ logn_fact = gen_logn_fact(data) n = 1 nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == 'normal' ] nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == 'infected'] prob_no_attacker = prob_model_no_attacker(SFTNet, data, T) prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T, logn_fact) numattackers = len(nodes_no_change) prob_mod = lambda x : prob_model_given_data(SFTNet, data, x, T, logn_fact) guess_times = np.sort(np.random.random(size=len(nodes_to_change))*T) z0 = dict(zip(nodes_to_change, guess_times)) for nd in nodes_no_change: z0[nd] = 0 order = sorted(z0.iterkeys(), key = lambda k: z0[k]) # lambda function that calls prob_model_given_data for # specified infection times p0 = prob_mod(z0) # Initiial probability # actual times time_samples = {node.name : [] for node in SFTNet.nodes} # container for samples probs = [] # container for probabilities z1 = copy.deepcopy(z0) orders = gen_orderings(SFTNet, s0) state0 = ['infected'] * numattackers + ['normal'] * len(nodes_to_change) while n < N: z1 = dict(zip(nodes_no_change, [0] * numattackers)) last_infect = 0 state = copy.copy(state0) if np.random.random() < alpha: new_order = random.choice(orders) switch_order = True else : switch_order = False new_order = order for nd in new_order[numattackers:]: cross_s_ix = SFTNet.cross_S.index(state) nd_ix = SFTNet.node_names.index(nd) incoming_rate = np.sum(SFTNet.mal_trans_mats[cross_s_ix][:, nd_ix]) last_infect = last_infect + trunc_expon(incoming_rate, T-last_infect) z1[nd] = last_infect state[nd_ix] = 'infected' p1 = prob_mod(z1) # Possible change to 2 if (p1[2] -p0[2] > np.log(np.random.random())): if print_jumps : print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n' if switch_order: #print ' new order ', order, ' at ', n p0 = p1 z0 = copy.deepcopy(z1) order = new_order for key, val in z0.iteritems(): time_samples[key].append(val) for nd in nodes_to_change: if nd not in z0.keys(): time_samples[nd].append(T) probs.append(p0[:2]) n += 1 probs = np.asarray(probs) out_ar = np.hstack((np.asarray(time_samples.values()).T, probs)) columns = copy.copy(time_samples.keys()) columns.append('P(z | attacker)') columns.append('P(data | z, attacker)') out = pandas.DataFrame(out_ar, columns = columns) mcmc_results = Results(out, data[-1], prob_no_attacker, prob_true_value, data, metropolis = True) return mcmc_results
def MCMC_sequence(SFTNet, data, s0, N, T, proposal_var=100, print_jumps=False, alpha=1): # TODO Need to profile this # TODO: Need to make this more general. Not trivial # TODO : Add sample from possible node orderings """ Performs MCMC integration using Metropolis Hastings. Returns the sampled times, their associated probabilities and the associated likelihood value. This method corresponds to David's "half-way" approach in the 2nd version of the ASCII where we sample (accept / reject) according to P(z | attacker) and then take the average of P(data | z, attacker) of the accepted values. SFTNet : SFTNet instance The net to do MCMC over data : list The data as outputted by gen_data N : int The number of MCMC proposals s0 : dict State of the net at t=0 T : int How long the process ran for. """ logn_fact = gen_logn_fact(data) n = 1 nodes_to_change = [nd for nd in SFTNet.node_names if s0[nd] == 'normal'] nodes_no_change = [nd for nd in SFTNet.node_names if s0[nd] == 'infected'] prob_no_attacker = prob_model_no_attacker(SFTNet, data, T) prob_true_value = prob_model_given_data(SFTNet, data, data[-1], T, logn_fact) numattackers = len(nodes_no_change) prob_mod = lambda x: prob_model_given_data(SFTNet, data, x, T, logn_fact) guess_times = np.sort(np.random.random(size=len(nodes_to_change)) * T) z0 = dict(zip(nodes_to_change, guess_times)) for nd in nodes_no_change: z0[nd] = 0 order = sorted(z0.iterkeys(), key=lambda k: z0[k]) # lambda function that calls prob_model_given_data for # specified infection times p0 = prob_mod(z0) # Initiial probability # actual times time_samples = {node.name: [] for node in SFTNet.nodes} # container for samples probs = [] # container for probabilities z1 = copy.deepcopy(z0) orders = gen_orderings(SFTNet, s0) state0 = ['infected'] * numattackers + ['normal'] * len(nodes_to_change) while n < N: z1 = dict(zip(nodes_no_change, [0] * numattackers)) last_infect = 0 state = copy.copy(state0) if np.random.random() < alpha: new_order = random.choice(orders) switch_order = True else: switch_order = False new_order = order for nd in new_order[numattackers:]: cross_s_ix = SFTNet.cross_S.index(state) nd_ix = SFTNet.node_names.index(nd) incoming_rate = np.sum(SFTNet.mal_trans_mats[cross_s_ix][:, nd_ix]) last_infect = last_infect + trunc_expon(incoming_rate, T - last_infect) z1[nd] = last_infect state[nd_ix] = 'infected' p1 = prob_mod(z1) # Possible change to 2 if (p1[2] - p0[2] > np.log(np.random.random())): if print_jumps: print 'A Jump at, ', n, 'to ', z1, 'with prob', p1, '\n' if switch_order: #print ' new order ', order, ' at ', n p0 = p1 z0 = copy.deepcopy(z1) order = new_order for key, val in z0.iteritems(): time_samples[key].append(val) for nd in nodes_to_change: if nd not in z0.keys(): time_samples[nd].append(T) probs.append(p0[:2]) n += 1 probs = np.asarray(probs) out_ar = np.hstack((np.asarray(time_samples.values()).T, probs)) columns = copy.copy(time_samples.keys()) columns.append('P(z | attacker)') columns.append('P(data | z, attacker)') out = pandas.DataFrame(out_ar, columns=columns) mcmc_results = Results(out, data[-1], prob_no_attacker, prob_true_value, data, metropolis=True) return mcmc_results