def calc_stc(cum, gpu=False): """ Calculate STC (spatio-temporal consistensy; Hanssen et al., 2008, Terrafirma) of time series of displacement. Note that isolated pixels (which have no surrounding pixel) have nan of STC. Input: cum : Cumulative displacement (n_im, length, width) gpu : GPU flag Return: stc : STC (length, width) """ if gpu: import cupy as xp cum = xp.asarray(cum) else: xp = np n_im, length, width = cum.shape ### Add 1 pixel margin to cum data filled with nan cum1 = xp.ones((n_im, length + 2, width + 2), dtype=xp.float32) * xp.nan cum1[:, 1:length + 1, 1:width + 1] = cum ### Calc STC for surrounding 8 pixels _stc = xp.ones((length, width, 8), dtype=xp.float32) * xp.nan pixels = [[0, 0], [0, 1], [0, 2], [1, 0], [1, 2], [2, 0], [2, 1], [2, 2]] ## Left Top = [0, 0], Rigth Bottmon = [2, 2], Center = [1, 1] for i, pixel in enumerate(pixels): ### Spatial difference (surrounding pixel-center) d_cum = cum1[:, pixel[0]:length + pixel[0], pixel[1]:width + pixel[1]] - cum1[:, 1:length + 1, 1:width + 1] ### Temporal difference (double difference) dd_cum = d_cum[:-1, :, :] - d_cum[1:, :, :] ### STC (i.e., RMS of DD) sumsq_dd_cum = xp.nansum(dd_cum**2, axis=0) n_dd_cum = (xp.sum(~xp.isnan(dd_cum), axis=0)).astype(xp.float32) #nof non-nan n_dd_cum[n_dd_cum == 0] = xp.nan #to avoid 0 division _stc[:, :, i] = xp.sqrt(sumsq_dd_cum / n_dd_cum) ### Strange but some adjacent pixels can have identical time series, ### resulting in 0 of stc. To avoid this, replace 0 with nan. _stc[_stc == 0] = xp.nan ### Identify minimum value as final STC with warnings.catch_warnings(): ## To silence warning by All-Nan slice warnings.simplefilter('ignore', RuntimeWarning) stc = xp.nanmin(_stc, axis=2) if gpu: stc = xp.asnumpy(stc) del cum, cum1, _stc, d_cum, dd_cum, sumsq_dd_cum, n_dd_cum return stc
def _masked_column_mean(arr, masked_value): """Compute the mean of each column in the 2D array arr, ignoring any instances of masked_value""" mask = _get_mask(arr, masked_value) count_missing_values = mask.sum(axis=0) n_elems = arr.shape[0] - count_missing_values mean = cp.nansum(arr, axis=0) if not cp.isnan(masked_value): mean -= (count_missing_values * masked_value) mean /= n_elems return mean
def test_nansum_axis_float16(self): # Note that the above test example overflows in float16. We use a # smaller array instead, return True if array is too large. if (numpy.prod(self.shape) > 24): return True a = testing.shaped_arange(self.shape, dtype='e') a[:, 1] = cupy.nan sa = cupy.nansum(a, axis=1) b = testing.shaped_arange(self.shape, numpy, dtype='f') b[:, 1] = numpy.nan sb = numpy.nansum(b, axis=1) testing.assert_allclose(sa, sb.astype('e'))
def _dense_fit(self, X, strategy, missing_values, fill_value): """Fit the transformer on dense data.""" mask = _get_mask(X, missing_values) # Mean if strategy == "mean": count_missing_values = mask.sum(axis=0) n_elems = X.shape[0] - count_missing_values mean = np.nansum(X, axis=0) mean -= (count_missing_values * missing_values) mean /= n_elems return mean # Median elif strategy == "median": count_missing_values = mask.sum(axis=0) n_elems = X.shape[0] - count_missing_values middle, is_odd = np.divmod(n_elems, 2) is_odd = is_odd.astype(np.bool) middle += count_missing_values X_sorted = X.copy() X_sorted[mask] = np.nan X_sorted = np.sort(X, axis=0) median = np.empty(X.shape[1], dtype=X.dtype) wis_odd = np.argwhere(is_odd).squeeze() wnot_odd = np.argwhere(~is_odd).squeeze() median[wis_odd] = X_sorted[middle[wis_odd], wis_odd] elm1 = X_sorted[middle[wnot_odd] - 1, wnot_odd] elm2 = X_sorted[middle[wnot_odd], wnot_odd] median[wnot_odd] = (elm1 + elm2) / 2. return median # Most frequent elif strategy == "most_frequent": n_features = X.shape[1] most_frequent = cpu_np.empty(n_features, dtype=X.dtype) for i in range(n_features): feature_mask_idxs = np.where(~mask[:, i])[0] values, counts = np.unique(X[feature_mask_idxs, i], return_counts=True) count_max = counts.max() if count_max > 0: value = values[counts == count_max].min() else: value = np.nan most_frequent[i] = value return np.array(most_frequent) # Constant elif strategy == "constant": return np.full(X.shape[1], fill_value, dtype=X.dtype)
def _csc_mean_variance_axis0(X): """Compute mean, variance and nans count on the axis 0 of a CSC matrix Parameters ---------- X : sparse CSC matrix Input array Returns ------- mean, variance, nans count """ n_samples, n_features = X.shape means = cp.empty(n_features) variances = cp.empty(n_features) counts_nan = cp.empty(n_features) start = X.indptr[0] for i, end in enumerate(X.indptr[1:]): col = X.data[start:end] _count_zeros = n_samples - col.size _count_nans = (col != col).sum() _mean = cp.nansum(col) / (n_samples - _count_nans) _variance = cp.nansum((col - _mean)**2) _variance += _count_zeros * (_mean**2) _variance /= (n_samples - _count_nans) means[i] = _mean variances[i] = _variance counts_nan[i] = _count_nans start = end return means, variances, counts_nan
def test_nansum_out_wrong_shape(self): a = testing.shaped_arange(self.shape) a[:, 1] = cupy.nan b = cupy.empty((2, 3)) with self.assertRaises(ValueError): cupy.nansum(a, axis=1, out=b)
def _acausal_classifier_gpu(filter_posterior, movement_state_transition, discrete_state_transition, observed_position_bin, uniform): ''' Parameters ---------- filter_posterior : ndarray, shape (n_time, 2, n_position_bins) movement_state_transition : ndarray, shape (n_position_bins, n_position_bins) discrete_state_transition : ndarray, shape (n_time, 2) discrete_state_transition[k, 0] = Pr(I_{k} = 1 | I_{k-1} = 0, v_{k}) discrete_state_transition[k, 1] = Pr(I_{k} = 1 | I_{k-1} = 1, v_{k}) observed_position_bin : ndarray, shape (n_time,) Which position bin is the animal in. position_bin_size : float Returns ------- smoother_posterior : ndarray, shape (n_time, 2, n_position_bins) p(x_{k + 1}, I_{k + 1} \vert H_{1:T}) smoother_probability : ndarray, shape (n_time, 2) smoother_probability[:, 0] = Pr(I_{1:T} = 0) smoother_probability[:, 1] = Pr(I_{1:T} = 1) smoother_prior : ndarray, shape (n_time, 2, n_position_bins) p(x_{k + 1}, I_{k + 1} \vert H_{1:k}) weights : ndarray, shape (n_time, 2, n_position_bins) \sum_{I_{k+1}} \int \Big[ \frac{p(x_{k+1} \mid x_{k}, I_{k}, I_{k+1}) * Pr(I_{k + 1} \mid I_{k}, v_{k}) * p(x_{k+1}, I_{k+1} \mid H_{1:T})} {p(x_{k + 1}, I_{k + 1} \mid H_{1:k})} \Big] dx_{k+1} ''' # noqa filter_posterior = cp.asarray(filter_posterior, dtype=cp.float32) movement_state_transition = cp.asarray( movement_state_transition, dtype=cp.float32) discrete_state_transition = cp.asarray( discrete_state_transition, dtype=cp.float32) observed_position_bin = cp.asarray(observed_position_bin) uniform = cp.asarray(uniform, dtype=cp.float32) EPS = cp.asarray(np.spacing(1), dtype=cp.float32) filter_probability = cp.sum(filter_posterior, axis=2) smoother_posterior = cp.zeros_like(filter_posterior) n_time, _, n_position_bins = filter_posterior.shape smoother_posterior[-1] = filter_posterior[-1].copy() for k in cp.arange(n_time - 2, -1, -1): smoother_prior = cp.zeros((2, n_position_bins), dtype=cp.float32) weights = cp.zeros((2, n_position_bins), dtype=cp.float32) position_ind = observed_position_bin[k + 1] # Predict p(x_{k + 1}, I_{k + 1} \vert H_{1:k}) # I_{k} = 0, I_{k + 1} = 0 smoother_prior[0, position_ind] = ( (1 - discrete_state_transition[k + 1, 0]) * filter_probability[k, 0]) # I_{k} = 1, I_{k + 1} = 0 smoother_prior[0, position_ind] += ( (1 - discrete_state_transition[k + 1, 1]) * filter_probability[k, 1]) # I_{k} = 0, I_{k + 1} = 1 smoother_prior[1] = ( discrete_state_transition[k + 1, 0] * uniform * filter_probability[k, 0]) # I_{k} = 1, I_{k + 1} = 1 smoother_prior[1] += ( discrete_state_transition[k + 1, 1] * (movement_state_transition.T @ filter_posterior[k, 1])) # Update p(x_{k}, I_{k} \vert H_{1:k}) ratio = cp.exp( cp.log(smoother_posterior[k + 1]) - cp.log(smoother_prior + EPS)) integrated_ratio = cp.sum(ratio, axis=1) # I_{k} = 0, I_{k + 1} = 0 weights[0] = ( (1 - discrete_state_transition[k + 1, 0]) * ratio[0, position_ind]) # I_{k} = 0, I_{k + 1} = 1 weights[0] += ( uniform * discrete_state_transition[k + 1, 0] * integrated_ratio[1]) # I_{k} = 1, I_{k + 1} = 0 weights[1] = ( (1 - discrete_state_transition[k + 1, 1]) * ratio[0, position_ind]) # I_{k} = 1, I_{k + 1} = 1 weights[1] += ( discrete_state_transition[k + 1, 1] * ratio[1] @ movement_state_transition) smoother_posterior[k] = weights * filter_posterior[k] smoother_posterior[k] /= cp.nansum(smoother_posterior[k]) smoother_probability = cp.sum(smoother_posterior, axis=2) return (cp.asnumpy(smoother_posterior), cp.asnumpy(smoother_probability))
def _causal_classifier_gpu(likelihood, movement_state_transition, discrete_state_transition, observed_position_bin, uniform): ''' Parameters ---------- likelihood : ndarray, shape (n_time, ...) movement_state_transition : ndarray, shape (n_position_bins, n_position_bins) discrete_state_transition : ndarray, shape (n_time, 2) discrete_state_transition[k, 0] = Pr(I_{k} = 1 | I_{k-1} = 0, v_{k}) discrete_state_transition[k, 1] = Pr(I_{k} = 1 | I_{k-1} = 1, v_{k}) observed_position_bin : ndarray, shape (n_time,) Which position bin is the animal in. position_bin_size : float Returns ------- posterior : ndarray, shape (n_time, 2, n_position_bins) state_probability : ndarray, shape (n_time, 2) state_probability[:, 0] = Pr(I_{1:T} = 0) state_probability[:, 1] = Pr(I_{1:T} = 1) prior : ndarray, shape (n_time, 2, n_position_bins) ''' likelihood = cp.asarray(likelihood, dtype=cp.float32) movement_state_transition = cp.asarray( movement_state_transition, dtype=cp.float32) discrete_state_transition = cp.asarray( discrete_state_transition, dtype=cp.float32) observed_position_bin = cp.asarray(observed_position_bin) uniform = cp.asarray(uniform, dtype=cp.float32) n_position_bins = movement_state_transition.shape[0] n_time = likelihood.shape[0] n_states = 2 posterior = cp.zeros( (n_time, n_states, n_position_bins), dtype=cp.float32) state_probability = cp.zeros((n_time, n_states), dtype=cp.float32) # Initial Conditions posterior[0, 0, observed_position_bin[0]] = likelihood[0, 0, 0] norm = cp.nansum(posterior[0]) data_log_likelihood = cp.log(norm) posterior[0] /= norm state_probability[0] = cp.sum(posterior[0], axis=1) for k in np.arange(1, n_time): prior = cp.zeros((n_states, n_position_bins), dtype=cp.float32) position_ind = observed_position_bin[k] # I_{k - 1} = 0, I_{k} = 0 prior[0, position_ind] = ( (1 - discrete_state_transition[k, 0]) * state_probability[k - 1, 0]) # I_{k - 1} = 1, I_{k} = 0 prior[0, position_ind] += ( (1 - discrete_state_transition[k, 1]) * state_probability[k - 1, 1]) # I_{k - 1} = 0, I_{k} = 1 prior[1] = (discrete_state_transition[k, 0] * uniform * state_probability[k - 1, 0]) # I_{k - 1} = 1, I_{k} = 1 prior[1] += ( discrete_state_transition[k, 1] * (movement_state_transition.T @ posterior[k - 1, 1])) posterior[k] = prior * likelihood[k] norm = cp.nansum(posterior[k]) data_log_likelihood += cp.log(norm) posterior[k] /= norm state_probability[k] = cp.sum(posterior[k], axis=1) return (cp.asnumpy(posterior), cp.asnumpy(state_probability), data_log_likelihood)
def _cov_pairwise(x1, x2, factor): return cupy.nansum(x1 * x2, axis=1, keepdims=True) * cupy.true_divide( 1, factor)