def test_posterior_maximum(xp): """ Test calculation of posterior maximum """ # # 1D predictions # quantiles = arange(xp, 0.1, 0.91, 0.1) y_pred = [ arange(xp, 2.0, 4.1, 1.0), arange(xp, 4.4, 4.51, 0.1), arange(xp, 5.0, 8.1, 1.0) ] y_pred = concatenate(xp, y_pred, 0) pm = posterior_maximum(y_pred, quantiles) assert np.isclose(pm, 4.45) # # 2D predictions # quantiles = arange(xp, 0.1, 0.91, 0.1) y_pred = eo.repeat(y_pred, 'q -> h q', h=10) pm = posterior_maximum(y_pred, quantiles) assert np.isclose(pm[0], 4.45) # # 3D predictions # quantiles = arange(xp, 0.1, 0.91, 0.1) y_pred = [ arange(xp, 2.0, 4.1, 1.0), arange(xp, 4.4, 4.51, 0.1), arange(xp, 5.0, 8.1, 1.0) ] y_pred = concatenate(xp, y_pred, 0) y_pred = eo.repeat(y_pred, 'q -> h q w', h=10, w=10) pm = posterior_maximum(y_pred, quantiles) assert np.isclose(pm[0, 0], 4.45) quantiles = arange(xp, 0.1, 0.91, 0.1) y_pred = [ arange(xp, 2.0, 4.1, 1.0), arange(xp, 4.4, 4.51, 0.1), arange(xp, 5.0, 8.1, 1.0) ] y_pred = concatenate(xp, y_pred, 0) y_pred = eo.repeat(y_pred, 'q -> h w q', h=10, w=10) pm = posterior_maximum(y_pred, quantiles, quantile_axis=-1) assert np.isclose(pm[0, 0], 4.45)
def posterior_quantiles(y_pred, bins, quantiles, bin_axis=1): if len(y_pred.shape) == 1: bin_axis = 0 n_y = y_pred.shape[bin_axis] n_b = len(bins) _check_dimensions(n_y, n_b) xp = get_array_module(y_pred) y_cdf = posterior_cdf(y_pred, bins, bin_axis=bin_axis) n = len(y_pred.shape) dx = bins[1:] - bins[:-1] x_shape = [1] * n x_shape[bin_axis] = numel(bins) dx = pad_zeros_left(xp, dx, 1, 0) dx = reshape(xp, dx, x_shape) y_qs = [] for q in quantiles: mask = as_type(xp, y_cdf <= q, y_cdf) y_q = bins[0] + xp.sum(mask * dx, bin_axis) y_q = expand_dims(xp, y_q, bin_axis) y_qs.append(y_q) y_q = concatenate(xp, y_qs, bin_axis) return y_q
def test_add(xp): """ Ensure that the maximum of the posterior is computed correctly. """ # # 1D predictions # bins = arange(xp, 0.0, 10.001, 1.0) bins_out = arange(xp, 0.5, 10.001, 1.0) y_pred = [ 0.5 * xp.ones(2), xp.zeros(8) ] y_pred = concatenate(xp, y_pred, 0) y_s = add(y_pred, bins, y_pred, bins, bins_out) assert np.isclose(y_s[0], 0.25) assert np.isclose(y_s[1], 0.5) assert np.isclose(y_s[2], 0.25) # # 2D predictions # y_pred = eo.repeat(y_pred, 'q -> h q', h=10) y_s = add(y_pred, bins, y_pred, bins, bins_out, bin_axis=1) assert np.isclose(y_s[0, 0], 0.25) assert np.isclose(y_s[0, 1], 0.5) assert np.isclose(y_s[0, 2], 0.25)
def test_concatenate(backend): """ Ensures that concatenation of array yields tensor with the expected size. """ array_1 = backend.ones((10, 1)) array_2 = backend.ones((10, 2)) result = concatenate(backend, [array_1, array_2], 1) assert numel(result) == 30
def posterior_quantiles(y_pred, quantiles, new_quantiles, quantile_axis=1): r""" Computes the median of the posterior distribution defined by an array of predicted quantiles. Args: y_pred: A rank-k tensor of predicted quantiles with the quantiles located along the axis given by ``quantile_axis``. quantiles: The quantile fractions corresponding to the quantiles located along the quantile axis. quantile_axis: The axis along which the quantiles are located. Returns: Rank k-1 tensor containing the posterior median for the provided inputs. """ if len(y_pred.shape) == 1: quantile_axis = 0 xp = get_array_module(y_pred) n = len(y_pred.shape) indices = arange(xp, 0, len(quantiles), 1.0) selection = [slice(0, None)] * n y_qs = [] for q in new_quantiles: mask = (quantiles[1:] > q) * (quantiles[:-1] <= q) index = indices[:-1][mask] if len(index) == 0: if quantiles[0] < q: selection[quantile_axis] = 0 selection_l = tuple(selection) return y_pred[selection_l] else: selection[quantile_axis] = -1 selection_r = tuple(selection) return y_pred[selection_r] index = int(index[0]) d = quantiles[index + 1] - quantiles[index] w_l = (quantiles[index + 1] - q) / d w_r = (q - quantiles[index]) / d selection = [slice(0, None)] * n selection[quantile_axis] = index selection_l = tuple(selection) selection[quantile_axis] = index + 1 selection_r = tuple(selection) y_q = w_l * y_pred[selection_l] + w_r * y_pred[selection_r] y_q = expand_dims(xp, y_q, quantile_axis) y_qs.append(y_q) return concatenate(xp, y_qs, quantile_axis)
def __call__(self, x, dist_axis=1): """ Evaluate the a priori. Args: x: Tensor containing the values at which to evaluate the a priori. dist_axis: The axis along which the tensor x is sorted. Returns; Tensor with the same size as 'x' containing the values of the a priori at 'x' obtained by linear interpolation. """ if len(x.shape) == 1: dist_axis = 0 xp = get_array_module(x) n_dims = len(x.shape) n = x.shape[dist_axis] x_index = [slice(0, None)] * n_dims x_index[dist_axis] = 0 selection_l = [slice(0, None)] * n_dims selection_l[dist_axis] = slice(0, -1) selection_l = tuple(selection_l) selection_r = [slice(0, None)] * n_dims selection_r[dist_axis] = slice(1, None) selection_r = tuple(selection_r) r_shape = [1] * n_dims r_shape[dist_axis] = -1 r_x = self.x.reshape(r_shape) r_y = self.y.reshape(r_shape) r_x_l = r_x[selection_l] r_x_r = r_x[selection_r] r_y_l = r_y[selection_l] r_y_r = r_y[selection_r] rs = [] for i in range(0, n): x_index[dist_axis] = slice(i, i + 1) index = tuple(x_index) x_i = x[index] mask = as_type(xp, (r_x_l < x_i) * (r_x_r >= x_i), x_i) r = r_y_l * (r_x_r - x_i) * mask r += r_y_r * (x_i - r_x_l) * mask r /= mask * (r_x_r - r_x_l) + (1.0 - mask) r = expand_dims(xp, r.sum(dist_axis), dist_axis) rs.append(r) r = concatenate(xp, rs, dist_axis) return r
def posterior_quantiles(y_pdf, bins, quantiles, bin_axis=1): """ Calculate posterior quantiles from predicted PDFs. Args: y_pdf: Tensor containing the predicted PDFs. bins: The bin-boundaries corresponding to the predictions. quantiles: List containing the quantiles fractions of the quantiles to compute. bin_axis: The index of the tensor axis which contains the predictions for each bin. Return: Tensor with same rank as ``y_pdf`` but with the values the values along ``bin_axis`` replaced with the quantiles of the predicted distributions. """ if len(y_pdf.shape) == 1: bin_axis = 0 n_y = y_pdf.shape[bin_axis] n_b = len(bins) n_dims = len(y_pdf.shape) _check_dimensions(n_y, n_b) xp = get_array_module(y_pdf) y_cdf = posterior_cdf(y_pdf, bins, bin_axis=bin_axis) n = len(y_pdf.shape) dx = bins[1:] - bins[:-1] x_shape = [1] * n x_shape[bin_axis] = numel(bins) dx = pad_zeros_left(xp, dx, 1, 0) dx = reshape(xp, dx, x_shape) selection = [slice(0, None)] * n_dims selection[bin_axis] = slice(0, -1) selection_l = tuple(selection) selection[bin_axis] = slice(1, None) selection_r = tuple(selection) cdf_l = y_cdf[selection_l] cdf_r = y_cdf[selection_r] d_cdf = cdf_r - cdf_l shape = [1] * n_dims shape[bin_axis] = -1 bins_l = bins.reshape(shape)[selection_l] bins_r = bins.reshape(shape)[selection_r] y_qs = [] for q in quantiles: mask_l = as_type(xp, cdf_l <= q, cdf_l) mask_r = as_type(xp, cdf_r > q, cdf_l) mask = mask_l * mask_r d_q = q - expand_dims(xp, (cdf_l * mask).sum(bin_axis), bin_axis) result = (d_q * bins_r + (d_cdf - d_q) * bins_l) * mask result = result / (d_cdf + as_type(xp, d_cdf < 1e-6, d_cdf)) result = result.sum(bin_axis) result = result + bins[-1] * (1.0 - as_type(xp, mask_r.sum(bin_axis) > 0, mask)) result = result + bins[0] * (1.0 - as_type(xp, mask_l.sum(bin_axis) > 0, mask)) result = expand_dims(xp, result, bin_axis) y_qs.append(result) y_q = concatenate(xp, y_qs, bin_axis) return y_q
def test_posterior_maximum(xp): """ Ensure that the maximum of the posterior is computed correctly. """ # # 1D predictions # bins = arange(xp, 0.0, 10.001, 1.0) y_pred = [ xp.ones(4), to_array(xp, [2.0]), xp.ones(4) ] y_pred = concatenate(xp, y_pred, 0) pm = posterior_maximum(y_pred, bins) assert np.isclose(pm, 4.5) # # 2D predictions # bins = arange(xp, 0.0, 10.001, 1.0) y_pred = [ xp.ones(4), to_array(xp, [2.0]), xp.ones(4) ] y_pred = concatenate(xp, y_pred, 0) y_pred = eo.repeat(y_pred, 'q -> h q', h=10) pm = posterior_maximum(y_pred, bins) assert np.isclose(pm[0], 4.5) # # 3D predictions # bins = arange(xp, 0.0, 10.001, 1.0) y_pred = [ xp.ones(4), to_array(xp, [2.0]), xp.ones(4) ] y_pred = concatenate(xp, y_pred, 0) y_pred = eo.repeat(y_pred, 'q -> h q w', h=10, w=10) pm = posterior_maximum(y_pred, bins) assert np.isclose(pm[0, 0], 4.5) bins = arange(xp, 0.0, 10.001, 1.0) y_pred = [ xp.ones(4), to_array(xp, [2.0]), xp.ones(4) ] y_pred = concatenate(xp, y_pred, 0) y_pred = eo.repeat(y_pred, 'q -> h w q', h=10, w=10) pm = posterior_maximum(y_pred, bins, -1) assert np.isclose(pm[0, 0], 4.5)
def pdf(y_pred, quantiles, quantile_axis=1): """ Calculate probability density function (PDF) of the posterior distribution defined by predicted quantiles. The PDF is approximated by computing the derivative of the cumulative distribution function (CDF), which is obtained from by fitting a piece-wise function to the predicted quantiles and corresponding quantile fractions. Args: y_pred: Tensor containing the predicted quantiles along the quantile axis. quantiles: The quantile fractions corresponding to the predicted quantiles in y_pred. quantile_axis: The axis of y_pred along which the predicted quantiles are located. Returns: Tuple ``(x_pdf, y_pdf)`` consisting of two arrays. ``x_pdf`` corresponds to the x-values of the PDF. ``y_pdf`` corresponds to the y-values of the PDF. """ if len(y_pred.shape) == 1: quantile_axis = 0 xp = get_array_module(y_pred) x_cdf, y_cdf = cdf(y_pred, quantiles, quantile_axis=quantile_axis) output_shape = list(x_cdf.shape) output_shape[quantile_axis] += 1 n_dims = len(output_shape) # # Assemble x-tensor # selection_l = [slice(0, None)] * n_dims selection_l[quantile_axis] = slice(0, -1) selection_l = tuple(selection_l) selection_r = [slice(0, None)] * n_dims selection_r[quantile_axis] = slice(1, None) selection_r = tuple(selection_r) x_pdf = 0.5 * (x_cdf[selection_l] + x_cdf[selection_r]) selection = [slice(0, None)] * n_dims selection[quantile_axis] = 0 x_pdf_l = x_cdf[tuple(selection)] x_pdf_l = expand_dims(xp, x_pdf_l, quantile_axis) selection[quantile_axis] = -1 x_pdf_r = x_cdf[tuple(selection)] x_pdf_r = expand_dims(xp, x_pdf_r, quantile_axis) x_pdf = concatenate(xp, [x_pdf_l, x_pdf, x_pdf_r], quantile_axis) # # Assemble y-tensor # y_pdf = 1.0 / (x_cdf[selection_r] - x_cdf[selection_l]) y_pdf = y_pdf * (y_cdf[1:] - y_cdf[:-1]) y_pdf = pad_zeros(xp, y_pdf, 1, quantile_axis) return x_pdf, y_pdf
def cdf(y_pred, quantiles, quantile_axis=1): """ Calculates the cumulative distribution function (CDF) from predicted quantiles. Args: y_pred: Array containing a range of predicted quantiles. The array is expected to contain the quantiles along the axis given by ``quantile_axis.`` quantiles: Array containing quantile fraction corresponding to the the predicted quantiles. quantile_axis: The index of the axis f the ``y_pred`` array, along which the quantiles are found. Returns: Tuple ``(x_cdf, y_cdf)`` of x and corresponding y-values of the CDF corresponding to quantiles given by ``y_pred``. Raises: InvalidArrayTypeException: When the data is provided neither as numpy array nor as torch tensor. InvalidDimensionException: When the provided predicted quantiles do not match the provided number of quantiles. """ if len(y_pred.shape) == 1: quantile_axis = 0 if y_pred.shape[quantile_axis] != len(quantiles): raise InvalidDimensionException( "Dimensions of the provided array 'y_pred' do not match the" "provided number of quantiles.") output_shape = list(y_pred.shape) xp = get_array_module(y_pred) y_cdf = quantiles y_cdf = concatenate(xp, [xp.zeros(1), y_cdf, xp.ones(1)], 0) selection = [slice(0, None)] * len(y_pred.shape) selection_c = copy(selection) selection_c[quantile_axis] = 0 selection_c = tuple(selection_c) selection_r = copy(selection) selection_r[quantile_axis] = 1 selection_r = tuple(selection_r) dy = (y_pred[selection_r] - y_pred[selection_c]) dy /= (quantiles[1] - quantiles[0]) x_cdf_l = y_pred[selection_c] - 2.0 * quantiles[0] * dy x_cdf_l = expand_dims(xp, x_cdf_l, quantile_axis) selection_l = copy(selection) selection_l[quantile_axis] = -2 selection_l = tuple(selection_l) selection_c = copy(selection) selection_c[quantile_axis] = -1 selection_c = tuple(selection_c) dy = (y_pred[selection_c] - y_pred[selection_l]) dy /= (quantiles[-1] - quantiles[-2]) x_cdf_r = y_pred[selection_c] + 2.0 * (1.0 - quantiles[-1]) * dy x_cdf_r = expand_dims(xp, x_cdf_r, quantile_axis) x_cdf = concatenate(xp, [x_cdf_l, y_pred, x_cdf_r], quantile_axis) return x_cdf, y_cdf
def pdf_binned(y_pred, quantiles, bins, quantile_axis=1): """ Calculate binned representation of the posterior probability density function (PDF). The binned PDF is simple calculated by linearly interpolating the piece-wise linear PDF computed using the :py:meth`pdf` method. Args: y_pred: Rank-k Tensor containing the predicted quantiles along the quantile axis. quantiles: The quantile fractions corresponding to the predicted quantiles in y_pred. bins: Rank-1 tensor containing the ``n_bins`` boundaries for the bins to use to bin the PDF. quantile_axis: The axis of y_pred along which the predicted quantiles are located. Returns: Rank-k tensor with ``n_bins - 1`` elements along ``quantile_axis`` containing the probability of the result to fall between the corresponding bin edges. """ if len(y_pred.shape) == 1: quantile_axis = 0 xp = get_array_module(y_pred) n = len(y_pred.shape) x_cdf, y_cdf = cdf(y_pred, quantiles, quantile_axis=quantile_axis) y_cdf_shape = [1] * n y_cdf_shape[quantile_axis] = -1 y_cdf = reshape(xp, y_cdf, y_cdf_shape) selection_l = [slice(0, None)] * n selection_l[quantile_axis] = slice(0, -1) selection_l = tuple(selection_l) selection_r = [slice(0, None)] * n selection_r[quantile_axis] = slice(1, None) selection_r = tuple(selection_r) selection_le = [slice(0, None)] * n selection_le[quantile_axis] = 0 selection_le = tuple(selection_le) selection_re = [slice(0, None)] * n selection_re[quantile_axis] = -1 selection_re = tuple(selection_re) y_pdf_binned = [] # # Interpolate CDF for leftmost bin boundary. # b_l = bins[0] mask_r = as_type(xp, (x_cdf[selection_r] >= b_l), y_cdf) mask_l = as_type(xp, (x_cdf[selection_l] < b_l), y_cdf) mask = mask_l * mask_r mask_xr = as_type(xp, xp.sum(mask_r, quantile_axis) == 0.0, mask_r) mask_xl = as_type(xp, xp.sum(mask_l, quantile_axis) == 0.0, mask_l) x_cdf_l = xp.sum(x_cdf[selection_l] * mask, quantile_axis) x_cdf_r = xp.sum(x_cdf[selection_r] * mask, quantile_axis) d = (x_cdf_r - x_cdf_l) + (1.0 - xp.sum(mask, quantile_axis)) w_cdf_l = (x_cdf_r - b_l) / d w_cdf_r = (b_l - x_cdf_l) / d y_cdf_l = ( xp.sum(mask * y_cdf[selection_l] * mask, quantile_axis) * w_cdf_l + xp.sum(mask * y_cdf[selection_r] * mask, quantile_axis) * w_cdf_r + mask_xl * y_cdf[selection_le] + mask_xr * y_cdf[selection_re]) for i in range(len(bins) - 1): b_r = bins[i + 1] # # Interpolate CDF for right bin boundary. # mask_r = as_type(xp, (x_cdf[selection_r] >= b_r), y_cdf) mask_l = as_type(xp, (x_cdf[selection_l] < b_r), y_cdf) mask = mask_l * mask_r mask_xr = as_type(xp, xp.sum(mask_r, quantile_axis) == 0.0, mask_r) mask_xl = as_type(xp, xp.sum(mask_l, quantile_axis) == 0.0, mask_l) x_cdf_l = xp.sum(x_cdf[selection_l] * mask, quantile_axis) x_cdf_r = xp.sum(x_cdf[selection_r] * mask, quantile_axis) d = (x_cdf_r - x_cdf_l) + (1.0 - xp.sum(mask, quantile_axis)) w_cdf_l = (x_cdf_r - b_r) / d w_cdf_r = (b_r - x_cdf_l) / d y_cdf_r = ( xp.sum(mask * y_cdf[selection_l] * mask, quantile_axis) * w_cdf_l + xp.sum(mask * y_cdf[selection_r] * mask, quantile_axis) * w_cdf_r + mask_xl * y_cdf[selection_le] + mask_xr * y_cdf[selection_re]) dy_cdf = expand_dims(xp, y_cdf_r - y_cdf_l, quantile_axis) y_pdf_binned.append(dy_cdf / (b_r - b_l)) y_cdf_l = y_cdf_r b_l = b_r return concatenate(xp, y_pdf_binned, quantile_axis)
def correct_a_priori(y_pred, quantiles, r, quantile_axis=1): """ Correct predicted quantiles for a priori. Args: y_pred: Rank-k tensor containing the predicted quantiles along the axis given by 'quantile_axis'. quantiles: Rank-1 tensor containing the quantile fractions that correspond to the predicted quantiles. r: A priori density ratio to use to correct the observations. quantile_axis: The axis along which the quantile are oriented in 'y_pred'. """ if len(y_pred.shape) == 1: quantile_axis = 0 xp = get_array_module(y_pred) n_dims = len(y_pred.shape) x_pdf, y_pdf = pdf(y_pred, quantiles, quantile_axis=quantile_axis) selection = [slice(0, None)] * len(y_pred.shape) selection_c = copy(selection) selection_c[quantile_axis] = 0 selection_c = tuple(selection_c) selection_r = copy(selection) selection_r[quantile_axis] = 1 selection_r = tuple(selection_r) dy = y_pred[selection_r] - y_pred[selection_c] dy /= quantiles[1] - quantiles[0] x_cdf_l = y_pred[selection_c] - 2.0 * quantiles[0] * dy x_cdf_l = expand_dims(xp, x_cdf_l, quantile_axis) selection_l = copy(selection) selection_l[quantile_axis] = -2 selection_l = tuple(selection_l) selection_c = copy(selection) selection_c[quantile_axis] = -1 selection_c = tuple(selection_c) dy = y_pred[selection_c] - y_pred[selection_l] dy /= quantiles[-1] - quantiles[-2] x_cdf_r = y_pred[selection_c] + 2.0 * (1.0 - quantiles[-1]) * dy x_cdf_r = expand_dims(xp, x_cdf_r, quantile_axis) x_cdf = concatenate(xp, [x_cdf_l, y_pred, x_cdf_r], quantile_axis) selection_l = [slice(0, None)] * n_dims selection_l[quantile_axis] = slice(0, -1) selection_l = tuple(selection_l) selection_r = [slice(0, None)] * n_dims selection_r[quantile_axis] = slice(1, None) selection_r = tuple(selection_r) x_index = [slice(0, None)] * n_dims x_index[quantile_axis] = 0 y_pdf_new = r(x_pdf, dist_axis=quantile_axis) * y_pdf selection = [slice(0, None)] * n_dims selection[quantile_axis] = slice(1, -1) selection = tuple(selection) y_cdf_new = cumtrapz(xp, y_pdf_new[selection], x_cdf, quantile_axis) selection = [slice(0, None)] * n_dims selection[quantile_axis] = slice(-1, None) selection = tuple(selection) y_cdf_new = y_cdf_new / y_cdf_new[selection] x_cdf_l = x_cdf[selection_l] x_cdf_r = x_cdf[selection_r] y_cdf_new_l = y_cdf_new[selection_l] y_cdf_new_r = y_cdf_new[selection_r] y_pred_new = [] for i in range(0, len(quantiles)): q = quantiles[i] mask = as_type(xp, (y_cdf_new_l < q) * (y_cdf_new_r >= q), x_cdf_l) y_new = x_cdf_l * (y_cdf_new_r - q) * mask y_new += x_cdf_r * (q - y_cdf_new_l) * mask y_new /= mask * (y_cdf_new_r - y_cdf_new_l) + (1.0 - mask) y_new = expand_dims(xp, y_new.sum(quantile_axis), quantile_axis) y_pred_new.append(y_new) y_pred_new = concatenate(xp, y_pred_new, quantile_axis) return y_pred_new