def process_batch(self, key, y_pred, y, cache=None): _check_input_dimensions(y_pred, y) if hasattr(self.model, "_post_process_prediction"): y_pred = self.model._post_process_prediction(y_pred, key=key) if self.tensor_backend is None: self.tensor_backend = get_tensor_backend(y_pred) xp = self.tensor_backend # Get deviation from mean from cache # if not already computed. if cache is not None and "y_mean" in cache: y_mean = cache["y_mean"] else: y_mean = self.model.posterior_mean(y_pred=y_pred, key=key) if cache is not None: cache["y_mean"] = y_mean y_mean = xp.to_numpy(y_mean) y = xp.to_numpy(y) if self.mask is not None: if hasattr(self.model, "quantile_axis"): dist_axis = self.model.quantile_axis else: dist_axis = self.model.bin_axis if len(y.shape) > len(y_mean.shape): y = y.squeeze(dist_axis) y_mean = y_mean[y > self.mask] y = y[y > self.mask] self.y_pred.setdefault(key, []).append(y_mean.ravel()) self.y.setdefault(key, []).append(y.ravel())
def process_batch(self, key, y_pred, y, cache=None): _check_input_dimensions(y_pred, y) if hasattr(self.model, "_post_process_prediction"): y_pred = self.model._post_process_prediction(y_pred, key=key) self.keys.add(key) if self.tensor_backend is None: self.tensor_backend = get_tensor_backend(y_pred) xp = self.tensor_backend crps = self.model.crps(y_pred=y_pred, y_true=y, key=key) if crps is None: return None crps_batches = self.crps.setdefault(key, []) crps = xp.to_numpy(crps) y = xp.to_numpy(y) if self.mask is not None: if hasattr(self.model, "quantile_axis"): dist_axis = self.model.quantile_axis else: dist_axis = self.model.bin_axis if len(y.shape) > len(crps.shape): y = y.squeeze(dist_axis) crps = crps[y > self.mask] crps_batches.append(crps.ravel())
def to_tensor(cls, tensor, like=None): """ Convert a tensor to a tensor of the given tensor backend. If the tensor is from another tensor backend, it will be converted to the tensor backend corresponding to this class by converting it to a numpy array and from that to a tensor of this backend. If the tensor is already a tensor of this backend, it will directly return the tensor, or, if ``like`` is given, convert it to a tensor that is compatible with the tensor ``like``. """ from quantnn.backends import get_tensor_backend if type(tensor).__module__ in ["numpy", "numpy.ma"]: return cls.from_numpy(tensor, like) backend = get_tensor_backend(tensor) if backend == cls: if like is not None: return cls.as_type(tensor, like) else: return tensor array = backend.to_numpy(tensor) return cls.from_numpy(array, like)
def __call__(self, x): """ Transform tensor. Args: x: Tensor containing the values to transform. Return: Tensor containing the transformed values. """ if self.xp is None: xp = get_tensor_backend(x) self.xp = xp else: xp = self.xp return xp.log(x.double()).float()
def invert(self, y): """ Transform transformed values back to original space. Args: y: Tensor containing the transformed values to transform back. Returns: Tensor containing the original values. """ if self.xp is None: xp = get_tensor_backend(y) self.xp = xp else: xp = self.xp return xp.exp(np.log(10) * y.double()).float()
def invert(self, y): """ Transform transformed values back to original space. Args: y: Tensor containing the transformed values to transform back. Returns: Tensor containing the original values. """ if self.xp is None: xp = get_tensor_backend(y) self.xp = xp else: xp = self.xp return xp.where(y > 10, y, xp.log(xp.exp(y) + 1.0))
def process_batch(self, key, y_pred, y, cache=None): _check_input_dimensions(y_pred, y) if hasattr(self.model, "_post_process_prediction"): y_pred = self.model._post_process_prediction(y_pred, key=key) if hasattr(self._model, "quantiles"): quantiles = self._model.quantiles else: quantiles = self.quantiles if quantiles is None: quantiles = np.linspace(0.05, 0.95, 10) y_pred = self.model.posterior_quantiles( y_pred=y_pred, quantiles=quantiles, key=key ) if y_pred is None: return None if self.tensor_backend is None: self.tensor_backend = get_tensor_backend(y_pred) xp = self.tensor_backend # Get deviation from mean from cache # if not already computed. axes = list(range(len(y.shape))) if hasattr(self.model, "quantile_axis"): q_axis = self.model.quantile_axis else: q_axis = self.model.bin_axis del axes[q_axis] if self.mask is not None: valid_pixels = xp.as_type(y > self.mask, y) valid_predictions = valid_pixels * xp.as_type(y <= y_pred, y_pred) c = self.calibration.get(key, xp.zeros(len(quantiles), y)) self.calibration[key] = c + valid_predictions.sum(axes) n = self.n_samples.get(key, xp.zeros(len(quantiles), y)) self.n_samples[key] = n + valid_pixels.sum() else: valid_predictions = xp.as_type(y <= y_pred, y_pred) c = self.calibration.get(key, xp.zeros(len(quantiles), y)) self.calibration[key] = c + valid_predictions.sum(axes) n = self.n_samples.get(key, xp.zeros(len(quantiles), y)) self.n_samples[key] = n + xp.size(y)
def __call__(self, x): """ Transform tensor. Args: x: Tensor containing the values to transform. Return: Tensor containing the transformed values. """ if self.xp is None: xp = get_tensor_backend(x) self.xp = xp else: xp = self.xp return xp.where(x > 1, x - 1.0, xp.log(x))
def process_batch(self, key, y_pred, y, cache=None): _check_input_dimensions(y_pred, y) if hasattr(self.model, "_post_process_prediction"): y_pred = self.model._post_process_prediction(y_pred, key=key) self.keys.add(key) if self.tensor_backend is None: self.tensor_backend = get_tensor_backend(y_pred) xp = self.tensor_backend # Get deviation from mean from cache # if not already computed. if cache is not None and "y_mean" in cache: y_mean = cache["y_mean"] else: y_mean = self.model.posterior_mean(y_pred=y_pred, key=key) if cache is not None: cache["y_mean"] = y_mean if len(y.shape) > len(y_mean.shape): if hasattr(self.model, "quantile_axis"): dist_axis = self.model.quantile_axis else: dist_axis = self.model.bin_axis y = y.squeeze(dist_axis) dy = y_mean - y # Calculate the squared error. if self.mask is not None: mask = xp.as_type(y > self.mask, y) se = self.squared_error.get(key, 0.0) self.squared_error[key] = se + ((mask * dy) ** 2).sum() n = self.n_samples.get(key, 0.0) self.n_samples[key] = n + mask.sum() else: se = self.squared_error.get(key, 0.0) self.squared_error[key] = se + (dy ** 2).sum() n = self.n_samples.get(key, 0.0) self.n_samples[key] = n + xp.size(y)
def aggregate_batches(self, batches): """ Aggregate list of batches. Args: batches: List of batches to aggregate. Return: Tuple ``(x, y)`` containing the aggregated inputs and outputs in 'batches'. """ xs = [] ys = None # Collect batches. for x, y in batches: xs.append(x) if isinstance(y, dict): if ys is None: ys = {} for k, y in y.items(): ys.setdefault(k, []).append(y) else: if ys is None: ys = [] ys.append(y) if self.backend is None: self.backend = get_tensor_backend(xs[0]) x = self.backend.concatenate(xs, 0) y = utils.apply(lambda y: self.backend.concatenate(y, 0), ys) if self.shuffle: indices = self._rng.permutation(x.shape[0]) f = lambda x: x[indices] x = f(x) y = utils.apply(f, y) return x, y
def process_batch(self, key, y_pred, y, cache=None): _check_input_dimensions(y_pred, y) if hasattr(self.model, "_post_process_prediction"): y_pred = self.model._post_process_prediction(y_pred, key=key) qf = self.model.quantile_function(y_pred=y_pred, y=y, key=key) if self.tensor_backend is None: self.tensor_backend = get_tensor_backend(y_pred) xp = self.tensor_backend qf = xp.to_numpy(qf) if self.mask is not None: y = xp.to_numpy(y) if hasattr(self.model, "quantile_axis"): dist_axis = self.model.quantile_axis else: dist_axis = self.model.bin_axis if len(y.shape) > len(qf.shape): y = y.squeeze(dist_axis) self.qfs.setdefault(key, []).append(qf[y > self.mask]) else: self.qfs.setdefault(key, []).append(qf.ravel())