def test_entropy(sigma: float, sample: np.ndarray, base: float = np.e): log_fun = _select_vectorized_log_fun_for_base(base) assert np.isclose(entropy_from_samples(sample, base=base, discrete=False), entropy_of_normal_distribution(sigma, log_fun=log_fun), rtol=1e-2, atol=1e-2)
def discrete_joint_entropy(sample_x: np.ndarray, sample_y: np.ndarray, base: float = np.e) -> float: """ Approximate the joint entropy of x and y H(X, Y) = - E_{p_{x, y}} \left[ \log p_{x, y} (x, y) \right] from a sample of both distributions. Parameters ---------- sample_x: a NumPy array of draws of variable x sample_y: a NumPy array of draws of variable y base: the base of the logarithm used to control the units of measurement for the result Returns ------- The joint entropy between of x and y """ log_fun = _select_vectorized_log_fun_for_base(base) sample_x, sample_y, n = _check_dimensions_of_two_variable_sample( sample_x, sample_y) unique_combinations_xy, counts_xy = \ _construct_unique_combinations_and_counts_from_two_samples(sample_x, sample_y) joint_frequency = (1.0 / n) * counts_xy return -np.sum(joint_frequency * log_fun(joint_frequency))
def test_relative_entropy(base: float = np.e): log_fun = _select_vectorized_log_fun_for_base(base) assert np.isclose(relative_entropy_from_samples(sample_p, sample_q, base=base, discrete=False), relative_entropy_between_two_normal_distributions( mu_p, sigma_p, mu_q, sigma_q, log_fun=log_fun), rtol=1e-1, atol=1e-1)
def test_compare_slow_and_fast_implementations_of_relative_entropy( sample_p: np.ndarray, sample_q: np.ndarray, base: float): log_fun = _select_vectorized_log_fun_for_base(base) relative_entropy_from_slow_calculation = \ _discrete_relative_entropy_slow(sample_p=sample_p, sample_q=sample_q, log_fun=log_fun) relative_entropy_from_fast_calculation = \ discrete_relative_entropy(sample_p=sample_p, sample_q=sample_q, base=base) assert np.isclose(relative_entropy_from_slow_calculation, relative_entropy_from_fast_calculation)
def discrete_relative_entropy(sample_p: np.ndarray, sample_q: np.ndarray, base: float = np.e): """ Approximate the relative entropy of the discrete distribution q relative to the discrete distribution p D_KL(p||q) = E_p [log(p/q)] from samples of these distributions. Parameters ---------- sample_p: sample from the distribution p sample_q: sample from the distribution q base: the base of the logarithm used to control the units of measurement for the result Returns ------- The relative entropy of the distribution q relative to the distribution p. """ log_fun = _select_vectorized_log_fun_for_base(base) combined_sample = np.hstack((sample_p, sample_q)) unique_combined = np.unique(combined_sample) unique_q, counts_q = np.unique(sample_q, return_counts=True) frequencies_q = counts_q / len(sample_q) unique_p, counts_p = np.unique(sample_p, return_counts=True) frequencies_p = counts_p / len(sample_p) combined_frequencies_p, combined_frequencies_q = \ _construct_frequencies_for_two_samples(sorted_p_realizations=unique_p, sorted_q_realizations=unique_q, sorted_q_frequencies=frequencies_q, sorted_p_frequencies=frequencies_p, sorted_combined_realizations=unique_combined) return np.sum(combined_frequencies_p * log_fun(combined_frequencies_p / combined_frequencies_q))
def discrete_entropy(sample: np.ndarray, base: float = np.e) -> float: """ Approximate the entropy of a discrete distribution H(p) = - E_p[log(p)] from a sample. Parameters ---------- sample: a sample from the discrete distribution base: the base of the logarithm used to control the units of measurement for the result Returns ------- An approximation of the entropy of the discrete distribution from which the sample is drawn. """ log_fun = _select_vectorized_log_fun_for_base(base) frequencies = _construct_frequencies_for_one_sample(sample) return -np.sum(frequencies * log_fun(frequencies))
def test_entropy(sample: np.ndarray, base: float): log_fun = _select_vectorized_log_fun_for_base(base) entropy_from_divergence = discrete_entropy(sample=sample, base=base) entropy_from_scipy = discrete_entropy_scipy(sample=sample, log_fun=log_fun) assert np.isclose(entropy_from_divergence, entropy_from_scipy)