def calc_full_log_likelihood(count_matrix, node_membership, duration, bp_lambda, num_classes, add_com_assig_log_prob=True): """ Calculates the full log likelihood of the Poisson baseline model. :param count_matrix: n_classes x n_classes where entry ij is denotes the number of events in block-pair ij :param node_membership: (list) membership of every node to one of K classes :param duration: (int) duration of the network :param bp_lambda: n_classes x n_classes where entry ij is the lambda of the block pair ij :param num_classes: (int) number of blocks / classes :param add_com_assig_log_prob: if True, adds the likelihood the community assignment to the total log-likelihood. :return: log-likelihood of the Poisson baseline model """ log_likelihood = 0 bp_size = utils.calc_block_pair_size(node_membership, num_classes) bp_ll = count_matrix * np.log(bp_lambda) - (bp_lambda * duration * bp_size) log_likelihood += np.sum(bp_ll) if add_com_assig_log_prob: # Adding the log probability of the community assignments to the full log likelihood n_nodes = len(node_membership) _, block_count = np.unique(node_membership, return_counts=True) class_prob_mle = block_count / sum(block_count) rv_multi = multinomial(n_nodes, class_prob_mle) log_prob_community_assignment = rv_multi.logpmf(block_count) log_likelihood += log_prob_community_assignment return log_likelihood
def compute_prediction_mean_and_variance_for_block_pair_event_count( train_bp_mu, train_bp_alpha_beta_ratio, test_block_pair_events, train_node_membership, n_classes, train_duration, test_duration): """ Computes sample mean and variance of block pair event counts :param test_block_pair_events: (list) n_classes x n_classes where entry ij is a list of event lists between nodes in block i to nodes in block j of the test dataset. :param train_node_membership: (list) membership of every node to one of K classes in the train dataset. :param n_classes: (int) number of blocks / classes :param train_duration: duration of the train dataset :param test_duration: duration of the test dataset :return: """ train_bp_size = utils.calc_block_pair_size(train_node_membership, n_classes) sample_mean = (train_bp_mu * train_duration) / (1 - train_bp_alpha_beta_ratio) sample_mean = (sample_mean / train_duration) * test_duration * train_bp_size sample_var = (train_bp_mu * train_duration) / ( (1 - train_bp_alpha_beta_ratio)**3) sample_var = (sample_var / train_duration) * test_duration * train_bp_size test_block_pair_event_count = compute_block_pair_total_event_count( test_block_pair_events, n_classes) return sample_mean, sample_var, test_block_pair_event_count
def compute_block_pair_event_count_empirical_mean_and_variance( block_pair_events, node_membership, n_classes): """ Computes the mean and variance of block pair event counts :param block_pair_events: (list) n_classes x n_classes where entry ij is a list of event lists between nodes in block i to nodes in block j. :param node_membership: (list) membership of every node to one of K classes. :param n_classes: (int) number of blocks / classes :return: a tuple of two matrices of KxK for mean and variance of block pair event counts """ bp_size = utils.calc_block_pair_size(node_membership, n_classes).astype(int) block_pair_events_counts_mean = np.zeros((n_classes, n_classes)) block_pair_events_counts_variance = np.zeros((n_classes, n_classes)) for i in range(n_classes): for j in range(n_classes): temp_counts = [ len(event_list) for event_list in block_pair_events[i][j] ] # actual counts temp_counts.extend( [0] * (bp_size[i, j] - len(temp_counts))) # add 0's for node-pairs with no events block_pair_events_counts_mean[i, j] = np.mean(temp_counts) block_pair_events_counts_variance[i, j] = np.std(temp_counts)**2 return block_pair_events_counts_mean, block_pair_events_counts_variance
def compute_mu_pairwise_difference_confidence_interval( event_dict, node_membership, num_classes, mu, duration, block_pair_tuple_list, z_alpha): """ Computes the pairwise difference if mu along with its confidence interval :param event_dict: Edge dictionary of events between all node pair. :param node_membership: (list) membership of every node to one of K classes. :param num_classes: (int) number of blocks / classes :param mu: KxK matrix of mu values for each block pair :param duration: the duration of the network :param block_pair_tuple_list: (list) of tuples for pairwise difference [(1, 1, 1, 2), (1, 1, 2, 1)] :param z_alpha: significance level (resulting in (1 - z_alpha) * 100 % CI) :return: dict with passed tuples as keys and a tuple of (difference, CI) as value """ num_nodes = len(node_membership) agg_adj = utils.event_dict_to_aggregated_adjacency(num_nodes, event_dict) sample_mean, sample_var = estimate_utils.compute_sample_mean_and_variance( agg_adj, node_membership) bp_size = utils.calc_block_pair_size(node_membership, num_classes) z = 1 - (z_alpha / (4 * (num_classes - 1) * num_classes)) ci_percentile = norm.ppf(1 - ((1 - z) / 2)) pairwise_res_dict = {} for a, b, x, y in block_pair_tuple_list: diff = mu[a, b] - mu[x, y] sqroot = np.sqrt((9 / 4) * ((sample_mean[a, b] / bp_size[a, b]) + (sample_mean[x, y] / bp_size[x, y]))) ci = ci_percentile * (1 / duration) * sqroot pairwise_res_dict[(a, b, x, y)] = (diff, ci) return pairwise_res_dict
def compute_mu_and_m_confidence_interval(event_dict, node_membership, num_classes, z_alpha, duration): """ Computes the confidence interval for mu and m (alpha to beta ratio) :param event_dict: Edge dictionary of events between all node pair. :param node_membership: (list) membership of every node to one of K classes. :param num_classes: (int) number of blocks / classes :param z_alpha: significance level (resulting in (1 - z_alpha) * 100 % CI) :param duration: the duration of the network :return: matrix of KxK confidence interval for mu and m """ num_nodes = len(node_membership) agg_adj = utils.event_dict_to_aggregated_adjacency(num_nodes, event_dict) sample_mean, sample_var = estimate_utils.compute_sample_mean_and_variance( agg_adj, node_membership) bp_size = utils.calc_block_pair_size(node_membership, num_classes) z = 1 - (z_alpha / (2 * (num_classes**2))) ci_percentile = norm.ppf(1 - ((1 - z) / 2)) mu_ci = ci_percentile * np.sqrt((9 * sample_mean) / (4 * bp_size)) mu_ci /= duration m_ci = ci_percentile * np.sqrt(1 / (4 * bp_size * sample_mean)) return mu_ci, m_ci
def calc_full_log_likelihood(block_pair_events, node_membership, bp_mu, bp_alpha, bp_beta, duration, num_classes, add_com_assig_log_prob=True): """ Calculates the full log likelihood of the CHIP model. :param block_pair_events: (list) n_classes x n_classes where entry ij is a list of event lists between nodes in block i to nodes in block j. :param node_membership: (list) membership of every node to one of K classes. :param bp_mu: n_classes x n_classes where entry ij is the mu of the block pair ij :param bp_alpha: n_classes x n_classes where entry ij is the alpha of the block pair ij :param bp_beta: n_classes x n_classes where entry ij is the beta of the block pair ij :param duration: (int) duration of the network :param num_classes: (int) number of blocks / classes :param add_com_assig_log_prob: if True, adds the likelihood the community assignment to the total log-likelihood. :return: log-likelihood of the CHIP model """ log_likelihood = 0 bp_size = utils.calc_block_pair_size(node_membership, num_classes) for b_i in range(num_classes): for b_j in range(num_classes): log_likelihood += estimate_utils.block_pair_full_hawkes_log_likelihood( block_pair_events[b_i][b_j], bp_mu[b_i, b_j], bp_alpha[b_i, b_j], bp_beta[b_i, b_j], duration, block_pair_size=bp_size[b_i, b_j]) if add_com_assig_log_prob: # Adding the log probability of the community assignments to the full log likelihood n_nodes = len(node_membership) _, block_count = np.unique(node_membership, return_counts=True) class_prob_mle = block_count / sum(block_count) rv_multi = multinomial(n_nodes, class_prob_mle) log_prob_community_assignment = rv_multi.logpmf(block_count) log_likelihood += log_prob_community_assignment return log_likelihood
def estimate_bp_hawkes_params(event_dict, node_membership, duration, num_classes, agg_adj=None, return_block_pair_events=False): """ Estimate CHIP Hawkes parameters. :param event_dict: Edge dictionary of events between all node pair. :param node_membership: (list) membership of every node to one of K classes. :param duration: (int) duration of the network :param num_classes: (int) number of blocks / classes :param agg_adj: (optional) np array (num_nodes x num_nodes) Adjacency matrix where element ij denotes the number of events between nodes i an j. If None, this will be calculated. :param return_block_pair_events: (bool) If True, returns the return_block_pair_events :return: parameters of the CHIP model -> mu, alpha, beta, m """ if agg_adj is None: num_nodes = len(node_membership) agg_adj = utils.event_dict_to_aggregated_adjacency( num_nodes, event_dict) bp_mu, bp_alpha_beta_ratio = estimate_utils.estimate_hawkes_from_counts( agg_adj, node_membership, duration, 1e-10 / duration) bp_beta = np.zeros((num_classes, num_classes), dtype=np.float) block_pair_events = utils.event_dict_to_block_pair_events( event_dict, node_membership, num_classes) bp_size = utils.calc_block_pair_size(node_membership, num_classes) for b_i in range(num_classes): for b_j in range(num_classes): bp_beta[b_i, b_j], _ = estimate_utils.estimate_beta_from_events( block_pair_events[b_i][b_j], bp_mu[b_i, b_j], bp_alpha_beta_ratio[b_i, b_j], duration, bp_size[b_i, b_j]) bp_alpha = bp_alpha_beta_ratio * bp_beta if return_block_pair_events: return bp_mu, bp_alpha, bp_beta, bp_alpha_beta_ratio, block_pair_events return bp_mu, bp_alpha, bp_beta, bp_alpha_beta_ratio
def estimate_poisson_lambda(count_matrix, node_membership, duration, num_classes, default_lambda=1e-10): """ Estimate lambda for all block pairs. :param count_matrix: n_classes x n_classes where entry ij is denotes the number of events in block-pair ij :param node_membership: (list) membership of every node to one of K classes. :param duration: (int) duration of the network :param default_lambda: default value for lambda if there are no events in a block pair to estimate lambda :return: n_classes x n_classes where entry ij is the lambda of the block pair ij """ bp_size = utils.calc_block_pair_size(node_membership, num_classes) # if a block only has 1 node in it, its own bp_size will be 0. # But since count_matrix will be zero setting to 1 won't change the outcome. bp_size[bp_size == 0] = 1 bp_lambda = count_matrix / (duration * bp_size) bp_lambda[bp_lambda == 0] = default_lambda return bp_lambda