def _check_array(self, X):
if isinstance(X, pd.DataFrame):
X = X.values
if isinstance(X, dd.DataFrame):
X = X.to_dask_array(lengths=True)
X = check_array(
X,
accept_dask_dataframe=False,
accept_unknown_chunks=False,
accept_sparse=False,
remove_zero_chunks=True,
)
if X.dtype == "int32":
X = X.astype("float32")
elif X.dtype == "int64":
X = X.astype("float64")
if isinstance(X, np.ndarray):
X = da.from_array(X,
chunks=(max(1,
len(X) // cpu_count()), X.shape[-1]))
bad = (da.isnull(X).any(), da.isinf(X).any())
if any(*compute(bad)):
msg = ("Input contains NaN, infinity or a value too large for "
"dtype('float64').")
raise ValueError(msg)
return X
def _check_array(self, X):
t0 = tic()
if isinstance(X, pd.DataFrame):
X = X.values
elif isinstance(X, dd.DataFrame):
raise TypeError("Cannot fit on dask.dataframe due to unknown "
"partition lengths.")
if X.dtype == 'int32':
X = X.astype('float32')
elif X.dtype == 'int64':
X = X.astype('float64')
X = check_array(X,
accept_dask_dataframe=False,
accept_unknown_chunks=False,
accept_sparse=False)
if isinstance(X, np.ndarray):
X = da.from_array(X,
chunks=(max(1,
len(X) // cpu_count()), X.shape[-1]))
bad = (da.isnull(X).any(), da.isinf(X).any())
if any(*compute(bad)):
msg = ("Input contains NaN, infinity or a value too large for "
"dtype('float64').")
raise ValueError(msg)
t1 = tic()
logger.info("Finished check_array in %0.2f s", t1 - t0)
return X
def _apply_sort(self, df: dd.DataFrame, sort_columns: List[str],
sort_ascending: List[bool]) -> dd.DataFrame:
# Split the first column. We need to handle this one with set_index
first_sort_column = sort_columns[0]
first_sort_ascending = sort_ascending[0]
# Sort the first column with set_index. Currently, we can only handle ascending sort
if not first_sort_ascending:
raise NotImplementedError(
"The first column needs to be sorted ascending (yet)")
# We can only sort if there are no NaNs or infs.
# Therefore we need to do a single pass over the dataframe
# to warn the user
# We shall also treat inf as na
col = df[first_sort_column]
isnan = col.isna().any()
numeric = pd.api.types.is_numeric_dtype(col.dtype)
# Important to evaluate the isinf late, as it only works with numeric-type columns
if isnan.compute() or (numeric and da.isinf(col).any().compute()):
raise ValueError("Can not sort a column with NaNs")
df = df.set_index(first_sort_column, drop=False).reset_index(drop=True)
# sort the remaining columns if given
if len(sort_columns) > 1:
sort_partition_func = lambda x: x.reset_index(
drop=True).sort_values(sort_columns, ascending=sort_ascending)
df = df.map_partitions(sort_partition_func, meta=df._meta)
return df
def redistribute(pop, target, default):
# Redistribute population proportionally at the target locations
redistribution = target / target.sum()
# If there are no target locations, all redistribution values will be inf.
# In this case, use the default
return da.where(
da.isinf(redistribution) | da.isnan(redistribution), default,
pop * redistribution)
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2,))
b = da.from_array(y, chunks=(2,))
c = da.from_array(z, chunks=(2,))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a ** b, x ** y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a ** 2, x ** 2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2 ** b, 2 ** y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b/10), np.arcsin(y/10))
assert eq(da.arccos(b/10), np.arccos(y/10))
assert eq(da.arctan(b/10), np.arctan(y/10))
assert eq(da.arctan2(b*10, a), np.arctan2(y*10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b*10), np.arcsinh(y*10))
assert eq(da.arccosh(b*10), np.arccosh(y*10))
assert eq(da.arctanh(b/10), np.arctanh(y/10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
def model_summary(self):
"""
Summarize totals of each species and their parameters in the domain. This is used primarily to service the
data requirements of an excel output in ``logger.write_xlsx``
:return: dict of species and their parameters
"""
def add_reduction(data, key, to_compute):
data[key] = to_compute
model_times = self.model_times
for species in self.summary.keys():
lcl_cmp = {} # Custom compute tree
post_cmp = {} # Compute tree for after
# Collect the species name
try:
species_name = [
name for name in self.domain.species_names
if species == pd.name_key(name)
][0]
except IndexError:
raise pd.PopdynError(
"Unable to gather the species name from the key {}".format(
species))
sp_log = self.summary[species]
# Carrying Capacity NOTE: This should be summarized by group in the future
ds = []
change_ds = []
first_cc = None
cc_mean_zero = []
cc_mean_nonzero = []
for time in model_times:
self.total_carrying_capacity(species, time)
cc_a = self.to_compute[-1]
total_cc = cc_a.sum()
cc_a_mean = (cc_a.mean() / (self.domain.csx * self.domain.csy)
) * 1e6 # Assuming metres
cc_mean_zero.append(cc_a_mean)
cc_mean_nonzero.append(
(cc_a[cc_a != 0].mean() /
(self.domain.csx * self.domain.csy)) * 1e6)
if first_cc is None:
first_cc = (cc_a.mean() /
(self.domain.csx * self.domain.csy)) * 1e6
change_ds.append(
da.where(first_cc > 0, cc_a_mean / first_cc, 0.0))
ds.append(total_cc)
key = "Habitat/{}/Total n".format(species_name)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
key = "Habitat/{}/Relative Change".format(species_name)
change_ds[0] = np.nan
add_reduction(lcl_cmp, key, change_ds)
key = "Habitat/{}/Mean [including zeros] (n per km. sq.)".format(
species_name)
cc_mean_zero[0] = np.nan
add_reduction(lcl_cmp, key, cc_mean_zero)
key = "Habitat/{}/Mean [excluding zeros] (n per km. sq.)".format(
species_name)
cc_mean_nonzero[0] = np.nan
add_reduction(lcl_cmp, key, cc_mean_nonzero)
# Collect average ages
ave_ages = []
for time in model_times:
self.average_age(species, time)
m = self.to_compute[-1]
not_inf = ~da.isinf(m)
ave_ages.append(m[not_inf].mean())
key = "Population/{}/NA/Average Age".format(species_name)
ave_ages[0] = np.nan
add_reduction(lcl_cmp, key, ave_ages)
# Add total population and lambda population for species
total_pop = []
lambda_pop = []
prv_pop = None
for time in model_times:
self.total_population(species, time)
pop_sum = self.to_compute[-1].sum()
total_pop.append(pop_sum)
if prv_pop is not None:
lambda_pop.append(pop_sum / prv_pop)
else:
lambda_pop.append(1)
prv_pop = pop_sum
key = "Population/{}/NA/Total Population".format(species_name)
add_reduction(lcl_cmp, key, total_pop)
key = "Population/{}/NA/Total Population Lambda".format(
species_name)
add_reduction(lcl_cmp, key, lambda_pop)
# Collect all offspring
tot_new_off = []
for time in model_times:
self.total_offspring(species, time)
tot_new_off.append(self.to_compute[-1].sum())
key = "Natality/{}/NA/Total new offspring".format(species_name)
tot_new_off[0] = np.nan
add_reduction(lcl_cmp, key, tot_new_off)
# Collect all deaths
all_deaths = []
for time in model_times:
self.total_mortality(species, time)
all_deaths.append(self.to_compute[-1].sum())
key = "Mortality/{}/NA/All deaths".format(species_name)
all_deaths[0] = np.nan
add_reduction(lcl_cmp, key, all_deaths)
# Collect deaths by type
mort_types = self.list_mortality_types(species, None)
for mort_type in mort_types:
if not ("Density Dependent Rate" in mort_type
or "Converted to " in mort_type):
ds = []
for time in model_times:
self.total_mortality(species,
time,
mortality_name=mort_type)
ds.append(self.to_compute[-1].sum())
key = "Mortality/{}/NA/Total deaths from {}".format(
species_name, mort_type)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
# Iterate groups and populate data
for sex in ["male", "female"]:
sp_log["Domain"]["Age Groups"][sex] = []
sp_log["Domain"]["Age Ranges"][sex] = []
sex_str = sex
# Collect total population by sex
sex_pop = []
for time in model_times:
self.total_population(species, time, sex)
sex_pop.append(self.to_compute[-1].sum())
key = "Population/{}/NA/Total {}s".format(
species_name, sex_str[0].upper() + sex_str[1:])
add_reduction(lcl_cmp, key, sex_pop)
if sex == "female":
# Collect offspring / female
key = "Natality/{}/NA/Total offspring per female".format(
species_name)
def topf(lcl_cmp, key):
species_name = key.split("/")[1]
return np.concatenate([
np.array([np.nan]),
np.where(
np.asarray(
lcl_cmp["Population/{}/NA/Total Females".
format(species_name)])[:-1] > 0,
np.asarray(lcl_cmp[
"Natality/{}/NA/Total new offspring".
format(species_name)])[1:] /
np.asarray(
lcl_cmp["Population/{}/NA/Total Females".
format(species_name)])[:-1],
np.inf,
),
]).tolist()
post_cmp[key] = topf
# Calculate the F:M ratio if both sexes present
if ("Population/{}/NA/Total Males".format(species_name)
in lcl_cmp.keys() and
"Population/{}/NA/Total Females".format(species_name)
in lcl_cmp.keys()):
key = "Natality/{}/NA/F:M Ratio".format(species_name)
def fmr(lcl_cmp, key):
species_name = key.split("/")[1]
return np.where(
np.asarray(lcl_cmp["Population/{}/NA/Total Males".
format(species_name)]) > 0,
np.asarray(lcl_cmp["Population/{}/NA/Total Females"
.format(species_name)]) /
np.asarray(lcl_cmp["Population/{}/NA/Total Males".
format(species_name)]),
np.inf,
).tolist()
post_cmp[key] = fmr
# Collect average ages by sex
ave_ages = []
for time in model_times:
self.average_age(species, time, sex)
m = self.to_compute[-1]
ave_ages.append(m[~da.isinf(m)].mean())
key = "Population/{}/NA/Average {} Age".format(
species_name, sex_str[0].upper() + sex_str[1:])
ave_ages[0] = np.nan
add_reduction(lcl_cmp, key, ave_ages)
# Offspring by sex
for __sex in ["male", "female"]:
offspring_sex = __sex[0].upper() + __sex[1:] + " Offspring"
ds = []
for time in model_times:
self.total_offspring(species,
time,
sex,
offspring_sex=offspring_sex)
ds.append(self.to_compute[-1].sum())
key = "Natality/{}/NA/{} offspring from {}s".format(
species_name,
__sex[0].upper() + __sex[1:],
sex_str[0].upper() + sex_str[1:],
)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
# Collect deaths by sex
sex_death = []
for time in model_times:
self.total_mortality(species, time, sex)
sex_death.append(self.to_compute[-1].sum())
key = "Mortality/{}/NA/Total {} deaths".format(
species_name, sex)
sex_death[0] = np.nan
add_reduction(lcl_cmp, key, sex_death)
# Collect deaths by type/sex
mort_types = self.list_mortality_types(species, None, sex)
for mort_type in mort_types:
if not ("Density Dependent Rate" in mort_type
or "Converted to " in mort_type):
ds = []
for time in model_times:
self.total_mortality(species,
time,
sex,
mortality_name=mort_type)
ds.append(self.to_compute[-1].sum())
key = "Mortality/{}/NA/{} deaths from {}".format(
species_name, sex_str[0].upper() + sex_str[1:],
mort_type)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
for gp in self.domain._group_keys(species):
if gp is None:
continue
try:
group_name = [
name for name in self.domain.group_names(species)
if gp == pd.name_key(name)
][0]
except IndexError:
raise pd.PopdynError(
"Unable to gather the group name from the key {}".
format(gp))
if sex is not None:
sp_log["Domain"]["Age Groups"][sex].append(group_name)
age_range = self.domain.age_from_group(
species, sex, gp)
sp_log["Domain"]["Age Ranges"][sex].append(age_range)
# Collect the total population of the group, which is only needed once
if ("Population/{}/{}/Total".format(
species_name, group_name) not in lcl_cmp.keys()):
gp_pop = []
lambda_pop = []
prv_pop = None
for time in model_times:
self.total_population(species, time, group=gp)
pop_sum = self.to_compute[-1].sum()
gp_pop.append(pop_sum)
if prv_pop is None:
lambda_pop.append(1.0)
else:
lambda_pop.append(pop_sum / prv_pop)
prv_pop = pop_sum.copy()
key = "Population/{}/{}/Total".format(
species_name, group_name)
add_reduction(lcl_cmp, key, gp_pop)
key = "Population/{}/{}/Lambda".format(
species_name, group_name)
add_reduction(lcl_cmp, key, lambda_pop)
# Collect average ages by gp
ave_ages = []
for time in model_times:
self.average_age(species, time, sex, gp)
m = self.to_compute[-1]
ave_ages.append(m[~da.isinf(m)].mean())
key = "Population/{}/{}/Average {} Age".format(
species_name, gp, sex_str[0].upper() + sex_str[1:])
ave_ages[0] = np.nan
add_reduction(lcl_cmp, key, ave_ages)
# Collect the population of this group and sex
gp_sex_pop = []
for time in model_times:
self.total_population(species, time, sex, gp)
gp_sex_pop.append(self.to_compute[-1].sum())
key = "Population/{}/{}/{}s".format(
species_name, group_name,
sex_str[0].upper() + sex_str[1:])
add_reduction(lcl_cmp, key, gp_sex_pop)
# Calculate the F:M ratio if both sexes present
if ("Population/{}/{}/Males".format(
species_name, group_name) in lcl_cmp.keys()
and "Population/{}/{}/Females".format(
species_name, group_name) in lcl_cmp.keys()):
key = "Natality/{}/{}/F:M Ratio".format(
species_name, group_name)
def fmr_gp(lcl_cmp, key):
species_name = key.split("/")[1]
group_name = key.split("/")[2]
return np.where(
np.asarray(
lcl_cmp["Population/{}/{}/Males".format(
species_name, group_name)]) > 0,
np.asarray(
lcl_cmp["Population/{}/{}/Females".format(
species_name, group_name)]) /
np.asarray(
lcl_cmp["Population/{}/{}/Males".format(
species_name, group_name)]),
np.inf,
).tolist()
post_cmp[key] = fmr_gp
# Natality
# Offspring
ds = []
for time in model_times:
self.total_offspring(species, time, sex, gp)
ds.append(self.to_compute[-1].sum())
key = "Natality/{}/{}/Total offspring".format(
species_name, group_name)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
for __sex in ["male", "female"]:
offspring_sex = __sex[0].upper(
) + __sex[1:] + " Offspring"
ds = []
for time in model_times:
self.total_offspring(species,
time,
sex,
gp,
offspring_sex=offspring_sex)
ds.append(self.to_compute[-1].sum())
key = "Natality/{}/{}/{} offspring from {}s".format(
species_name,
group_name,
__sex[0].upper() + __sex[1:],
sex_str[0].upper() + sex_str[1:],
)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
if sex == "female":
# Density coefficient
dd_fec_ds = []
for time in model_times:
self.fecundity(species, time, sex, gp, coeff=True)
dd_fec_ds.append(self.to_compute[-1].mean())
key = "Natality/{}/{}/Density-Based Fecundity Reduction Rate".format(
species_name, group_name)
dd_fec_ds[0] = np.nan
add_reduction(lcl_cmp, key, dd_fec_ds)
# Fecundity rate
ds = []
for time in model_times:
self.fecundity(species, time, sex, gp)
ds.append(self.to_compute[-1].mean())
key = "Natality/{}/{}/{} mean fecundity".format(
species_name, group_name, sex_str)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
# Offspring per female
key = "Natality/{}/{}/offspring per female".format(
species_name, group_name)
def opf_gp(lcl_cmp, key):
species_name = key.split("/")[1]
group_name = key.split("/")[2]
return np.concatenate([
np.array([np.nan]),
np.where(
np.asarray(lcl_cmp[
"Population/{}/{}/Females".format(
species_name,
group_name)])[:-1] > 0,
np.asarray(lcl_cmp[
"Natality/{}/{}/Total offspring".
format(species_name, group_name)])[1:]
/ np.asarray(lcl_cmp[
"Population/{}/{}/Females".format(
species_name, group_name)])[:-1],
np.inf,
),
]).tolist()
post_cmp[key] = opf_gp
# Mortality
# Male/Female by group
mort_ds = []
for time in model_times:
self.total_mortality(species, time, sex, gp)
mort_ds.append(self.to_compute[-1].sum())
key = "Mortality/{}/{}/{} deaths".format(
species_name, group_name,
sex_str[0].upper() + sex_str[1:])
mort_ds[0] = np.nan
add_reduction(lcl_cmp, key, mort_ds)
# All for group
ds = []
for time in model_times:
self.total_mortality(species, time, None, gp)
ds.append(self.to_compute[-1].sum())
key = "Mortality/{}/{}/Total deaths".format(
species_name, group_name)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
mort_types = self.list_mortality_types(
species, None, sex, gp)
for mort_type in mort_types:
if mort_type != "Density Dependent Rate":
ds = []
for time in model_times:
self.total_mortality(species, time, sex, gp,
mort_type)
ds.append(self.to_compute[-1].sum())
if "Converted to " in mort_type:
mort_str = "{} {}".format(sex_str, mort_type)
else:
mort_str = "{} {} deaths".format(
sex_str, mort_type)
key = "Mortality/{}/{}/{}".format(
species_name, group_name, mort_str)
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
# Skip the implicit mortality types, as they will not be included in the params
if (mort_type in ["Old Age", "Density Dependent"]
or "Converted to " in mort_type):
continue
# Collect the parameter
ds = []
for time in model_times:
self.total_mortality(species, time, sex, gp,
mort_type, False)
ds.append(self.to_compute[-1].mean())
key = "Mortality/{}/{}/{} mean {} rate".format(
species_name, group_name, sex_str, mort_type)
if "Density Dependent Rate" in key:
key = key[:-4]
ds[0] = np.nan
add_reduction(lcl_cmp, key, ds)
# Compute and populate the species log
print("Computing initial values")
keys = lcl_cmp.keys()
reduced_values = {key: [] for key in keys}
for i in range(len(lcl_cmp[keys[0]])):
for key, val in zip(
keys,
da.compute([lcl_cmp[key][i] for key in keys])[0]):
reduced_values[key].append(val)
print("Computing derived values")
for key, val in post_cmp.items():
reduced_values[key] = val(reduced_values, key)
for key, values in reduced_values.items():
_keys = key.split("/")
try:
d = sp_log[_keys[0]]
except KeyError:
sp_log[_keys[0]] = {}
d = sp_log[_keys[0]]
for key in _keys[1:-1]:
try:
d = d[key]
except KeyError:
d[key] = {}
d = d[key]
d[_keys[-1]] = values
def test_arithmetic():
x = np.arange(5).astype('f4') + 2
y = np.arange(5).astype('i8') + 2
z = np.arange(5).astype('i4') + 2
a = da.from_array(x, chunks=(2, ))
b = da.from_array(y, chunks=(2, ))
c = da.from_array(z, chunks=(2, ))
assert eq(a + b, x + y)
assert eq(a * b, x * y)
assert eq(a - b, x - y)
assert eq(a / b, x / y)
assert eq(b & b, y & y)
assert eq(b | b, y | y)
assert eq(b ^ b, y ^ y)
assert eq(a // b, x // y)
assert eq(a**b, x**y)
assert eq(a % b, x % y)
assert eq(a > b, x > y)
assert eq(a < b, x < y)
assert eq(a >= b, x >= y)
assert eq(a <= b, x <= y)
assert eq(a == b, x == y)
assert eq(a != b, x != y)
assert eq(a + 2, x + 2)
assert eq(a * 2, x * 2)
assert eq(a - 2, x - 2)
assert eq(a / 2, x / 2)
assert eq(b & True, y & True)
assert eq(b | True, y | True)
assert eq(b ^ True, y ^ True)
assert eq(a // 2, x // 2)
assert eq(a**2, x**2)
assert eq(a % 2, x % 2)
assert eq(a > 2, x > 2)
assert eq(a < 2, x < 2)
assert eq(a >= 2, x >= 2)
assert eq(a <= 2, x <= 2)
assert eq(a == 2, x == 2)
assert eq(a != 2, x != 2)
assert eq(2 + b, 2 + y)
assert eq(2 * b, 2 * y)
assert eq(2 - b, 2 - y)
assert eq(2 / b, 2 / y)
assert eq(True & b, True & y)
assert eq(True | b, True | y)
assert eq(True ^ b, True ^ y)
assert eq(2 // b, 2 // y)
assert eq(2**b, 2**y)
assert eq(2 % b, 2 % y)
assert eq(2 > b, 2 > y)
assert eq(2 < b, 2 < y)
assert eq(2 >= b, 2 >= y)
assert eq(2 <= b, 2 <= y)
assert eq(2 == b, 2 == y)
assert eq(2 != b, 2 != y)
assert eq(-a, -x)
assert eq(abs(a), abs(x))
assert eq(~(a == b), ~(x == y))
assert eq(~(a == b), ~(x == y))
assert eq(da.logaddexp(a, b), np.logaddexp(x, y))
assert eq(da.logaddexp2(a, b), np.logaddexp2(x, y))
assert eq(da.exp(b), np.exp(y))
assert eq(da.log(a), np.log(x))
assert eq(da.log10(a), np.log10(x))
assert eq(da.log1p(a), np.log1p(x))
assert eq(da.expm1(b), np.expm1(y))
assert eq(da.sqrt(a), np.sqrt(x))
assert eq(da.square(a), np.square(x))
assert eq(da.sin(a), np.sin(x))
assert eq(da.cos(b), np.cos(y))
assert eq(da.tan(a), np.tan(x))
assert eq(da.arcsin(b / 10), np.arcsin(y / 10))
assert eq(da.arccos(b / 10), np.arccos(y / 10))
assert eq(da.arctan(b / 10), np.arctan(y / 10))
assert eq(da.arctan2(b * 10, a), np.arctan2(y * 10, x))
assert eq(da.hypot(b, a), np.hypot(y, x))
assert eq(da.sinh(a), np.sinh(x))
assert eq(da.cosh(b), np.cosh(y))
assert eq(da.tanh(a), np.tanh(x))
assert eq(da.arcsinh(b * 10), np.arcsinh(y * 10))
assert eq(da.arccosh(b * 10), np.arccosh(y * 10))
assert eq(da.arctanh(b / 10), np.arctanh(y / 10))
assert eq(da.deg2rad(a), np.deg2rad(x))
assert eq(da.rad2deg(a), np.rad2deg(x))
assert eq(da.logical_and(a < 1, b < 4), np.logical_and(x < 1, y < 4))
assert eq(da.logical_or(a < 1, b < 4), np.logical_or(x < 1, y < 4))
assert eq(da.logical_xor(a < 1, b < 4), np.logical_xor(x < 1, y < 4))
assert eq(da.logical_not(a < 1), np.logical_not(x < 1))
assert eq(da.maximum(a, 5 - a), np.maximum(a, 5 - a))
assert eq(da.minimum(a, 5 - a), np.minimum(a, 5 - a))
assert eq(da.fmax(a, 5 - a), np.fmax(a, 5 - a))
assert eq(da.fmin(a, 5 - a), np.fmin(a, 5 - a))
assert eq(da.isreal(a + 1j * b), np.isreal(x + 1j * y))
assert eq(da.iscomplex(a + 1j * b), np.iscomplex(x + 1j * y))
assert eq(da.isfinite(a), np.isfinite(x))
assert eq(da.isinf(a), np.isinf(x))
assert eq(da.isnan(a), np.isnan(x))
assert eq(da.signbit(a - 3), np.signbit(x - 3))
assert eq(da.copysign(a - 3, b), np.copysign(x - 3, y))
assert eq(da.nextafter(a - 3, b), np.nextafter(x - 3, y))
assert eq(da.ldexp(c, c), np.ldexp(z, z))
assert eq(da.fmod(a * 12, b), np.fmod(x * 12, y))
assert eq(da.floor(a * 0.5), np.floor(x * 0.5))
assert eq(da.ceil(a), np.ceil(x))
assert eq(da.trunc(a / 2), np.trunc(x / 2))
assert eq(da.degrees(b), np.degrees(y))
assert eq(da.radians(a), np.radians(x))
assert eq(da.rint(a + 0.3), np.rint(x + 0.3))
assert eq(da.fix(a - 2.5), np.fix(x - 2.5))
assert eq(da.angle(a + 1j), np.angle(x + 1j))
assert eq(da.real(a + 1j), np.real(x + 1j))
assert eq((a + 1j).real, np.real(x + 1j))
assert eq(da.imag(a + 1j), np.imag(x + 1j))
assert eq((a + 1j).imag, np.imag(x + 1j))
assert eq(da.conj(a + 1j * b), np.conj(x + 1j * y))
assert eq((a + 1j * b).conj(), (x + 1j * y).conj())
assert eq(da.clip(b, 1, 4), np.clip(y, 1, 4))
assert eq(da.fabs(b), np.fabs(y))
assert eq(da.sign(b - 2), np.sign(y - 2))
l1, l2 = da.frexp(a)
r1, r2 = np.frexp(x)
assert eq(l1, r1)
assert eq(l2, r2)
l1, l2 = da.modf(a)
r1, r2 = np.modf(x)
assert eq(l1, r1)
assert eq(l2, r2)
assert eq(da.around(a, -1), np.around(x, -1))
