def test(self):
shape = (3, 5, 10)
a = biggus.ConstantArray(shape, dtype=np.float32)
b = biggus.ConstantArray(shape, dtype=np.float64)
with set_chunk_size(32//8*10-1):
result = biggus.sum(a * b, axis=0).ndarray()
self.assertEqual(result.shape, shape[1:])
def test(self):
shape = (3, 5, 10)
a = biggus.ConstantArray(shape, dtype=np.float32)
b = biggus.ConstantArray(shape, dtype=np.float64)
with set_chunk_size(32 // 8 * 10 - 1):
result = biggus.sum(a * b, axis=0).ndarray()
self.assertEqual(result.shape, shape[1:])
def _biggus_filter(self, data, weights):
# Filter a data array (time, <other dimensions>) using information in
# weights dictionary.
#
# Args:
#
# * data:
# biggus array of the data to be filtered
# * weights:
# dictionary of absolute record offset : weight
# Build filter_matrix (time to time' mapping).
shape = data.shape
# Build filter matrix as a numpy array and then populate.
filter_matrix_np = np.zeros((shape[0], shape[0])).astype(self.dtype)
for offset, value in weights.items():
filter_matrix_np += np.diag([value] * (shape[0] - offset),
k=offset)
if offset > 0:
filter_matrix_np += np.diag([value] * (shape[0] - offset),
k=-offset)
# Create biggus array for filter matrix, adding in other dimensions.
for _ in shape[1:]:
filter_matrix_np = filter_matrix_np[..., np.newaxis]
filter_matrix_bg_single = biggus.NumpyArrayAdapter(filter_matrix_np)
# Broadcast to correct shape (time, time', lat, lon).
filter_matrix_bg = biggus.BroadcastArray(
filter_matrix_bg_single,
{i + 2: j
for i, j in enumerate(shape[1:])})
# Broadcast filter to same shape.
biggus_data_for_filter = biggus.BroadcastArray(data[np.newaxis, ...],
{0: shape[0]})
# Multiply two arrays together and sum over second time dimension.
filtered_data = biggus.sum(biggus_data_for_filter * filter_matrix_bg,
axis=1)
# Cut off records at start and end of output array where the filter
# cannot be fully applied.
filter_halfwidth = len(weights) - 1
filtered_data = filtered_data[filter_halfwidth:-filter_halfwidth]
return filtered_data
def _biggus_filter(self, data, weights):
# Filter a data array (time, <other dimensions>) using information in
# weights dictionary.
#
# Args:
#
# * data:
# biggus array of the data to be filtered
# * weights:
# dictionary of absolute record offset : weight
# Build filter_matrix (time to time' mapping).
shape = data.shape
# Build filter matrix as a numpy array and then populate.
filter_matrix_np = np.zeros((shape[0], shape[0])).astype(self.dtype)
for offset, value in weights.items():
filter_matrix_np += np.diag([value] * (shape[0] - offset),
k=offset)
if offset > 0:
filter_matrix_np += np.diag([value] * (shape[0] - offset),
k=-offset)
# Create biggus array for filter matrix, adding in other dimensions.
for _ in shape[1:]:
filter_matrix_np = filter_matrix_np[..., np.newaxis]
filter_matrix_bg_single = biggus.NumpyArrayAdapter(filter_matrix_np)
# Broadcast to correct shape (time, time', lat, lon).
filter_matrix_bg = biggus.BroadcastArray(
filter_matrix_bg_single, {i+2: j for i, j in enumerate(shape[1:])})
# Broadcast filter to same shape.
biggus_data_for_filter = biggus.BroadcastArray(data[np.newaxis, ...],
{0: shape[0]})
# Multiply two arrays together and sum over second time dimension.
filtered_data = biggus.sum(biggus_data_for_filter * filter_matrix_bg,
axis=1)
# Cut off records at start and end of output array where the filter
# cannot be fully applied.
filter_halfwidth = len(weights) - 1
filtered_data = filtered_data[filter_halfwidth:-filter_halfwidth]
return filtered_data
def get_seasonal_means_with_ttest_stats(
self,
season_to_monthperiod=None,
start_year=None,
end_year=None,
convert_monthly_accumulators_to_daily=False):
"""
:param season_to_monthperiod:
:param start_year:
:param end_year:
:param convert_monthly_accumulators_to_daily: if true converts monthly accumulators to daily,
:return dict(season: [mean, std, nobs])
"""
if True:
raise NotImplementedError(
"Biggus way of calculation is not implemented, use the dask version of the method"
)
# select the interval of interest
timesel = [
i for i, d in enumerate(self.time)
if start_year <= d.year <= end_year
]
data = self.data[timesel, :, :]
times = [self.time[i] for i in timesel]
if convert_monthly_accumulators_to_daily:
ndays = np.array(
[calendar.monthrange(d.year, d.month)[1] for d in times])
data = biggus.divide(data, ndays[:, np.newaxis, np.newaxis])
else:
data = self.data
year_month_to_index_arr = defaultdict(list)
for i, t in enumerate(times):
year_month_to_index_arr[t.year, t.month].append(i)
# calculate monthly means
monthly_data = {}
for y in range(start_year, end_year + 1):
for m in range(1, 13):
aslice = slice(year_month_to_index_arr[y, m][0],
year_month_to_index_arr[y, m][-1] + 1)
monthly_data[y, m] = biggus.mean(
data[aslice.start:aslice.stop, :, :], axis=0)
result = {}
for season, month_period in season_to_monthperiod.items():
assert isinstance(month_period, MonthPeriod)
seasonal_means = []
ndays_per_season = []
for p in month_period.get_season_periods(start_year=start_year,
end_year=end_year):
lmos = biggus.ArrayStack([
monthly_data[start.year, start.month]
for start in p.range("months")
])
ndays_per_month = np.array([
calendar.monthrange(start.year, start.month)[1]
for start in p.range("months")
])
seasonal_mean = biggus.sum(biggus.multiply(
lmos, ndays_per_month[:, np.newaxis, np.newaxis]),
axis=0)
seasonal_mean = biggus.divide(seasonal_mean,
ndays_per_month.sum())
seasonal_means.append(seasonal_mean)
ndays_per_season.append(ndays_per_month.sum())
seasonal_means = biggus.ArrayStack(seasonal_means)
ndays_per_season = np.array(ndays_per_season)
print(seasonal_means.shape, ndays_per_season.shape)
assert seasonal_means.shape[0] == ndays_per_season.shape[0]
clim_mean = biggus.sum(biggus.multiply(
seasonal_means, ndays_per_season[:, np.newaxis, np.newaxis]),
axis=0) / ndays_per_season.sum()
diff = biggus.subtract(seasonal_means,
clim_mean.masked_array()[np.newaxis, :, :])
sq_mean = biggus.sum(biggus.multiply(
diff**2, ndays_per_season[:, np.newaxis, np.newaxis]),
axis=0) / ndays_per_season.sum()
clim_std = biggus.power(sq_mean, 0.5)
clim_mean = clim_mean.masked_array()
print("calculated mean")
clim_std = clim_std.masked_array()
print("calculated std")
result[season] = [clim_mean, clim_std, ndays_per_season.shape[0]]
return result
