def get_scalar_dtype_strategy(exclude=None):
"""
A `hypothesis` strategy yielding
"""
possible_strategies = {
"datetime": hyp_np.datetime64_dtypes(max_period="ms", min_period="ns"),
"uint": hyp_np.unsigned_integer_dtypes(),
"int": hyp_np.integer_dtypes(),
"float": hyp_np.floating_dtypes(),
"byte": hyp_np.byte_string_dtypes(),
"unicode": hyp_np.unicode_string_dtypes(),
}
if exclude is None:
exclude = {}
elif not isinstance(exclude, list):
exclude = [exclude]
for ex in exclude:
if ex in possible_strategies:
del possible_strategies[ex]
else:
raise ValueError(
"Strategy {} unknown. Possible values are {}".format(
ex, possible_strategies.keys()
)
)
return hyp_st.one_of(*list(possible_strategies.values()))
def ints_floats_complex_or_booleans(draw):
dtypes = (
unsigned_integer_dtypes(endianness="="),
integer_dtypes(endianness="="),
floating_dtypes(endianness="="),
# complex_number_dtypes(endianness="="),
boolean_dtypes(),
)
return draw(one_of(dtypes))
def ints_floats_datetimes_and_timedeltas(draw):
dtypes = (
unsigned_integer_dtypes(endianness="="),
integer_dtypes(endianness="="),
floating_dtypes(endianness="="),
datetime64_dtypes(endianness="="),
timedelta64_dtypes(endianness="="),
)
return draw(one_of(dtypes))
def ints_floats_or_complex_dtypes(draw):
# Endianness needs to be specified for now, otherwise the byte-order may get flipped
# https://jira/browse/SOQTEST-6478
dtypes = (
unsigned_integer_dtypes(endianness="="),
integer_dtypes(endianness="="),
floating_dtypes(endianness="="),
# complex_number_dtypes(endianness="="),
)
return draw(one_of(dtypes))
def ints_or_floats_dtypes(draw):
# Endianness needs to be specified for now, otherwise the byte-order may get flipped
# https://jira/browse/SOQTEST-6478
# Half floats are not supported.
dtypes = (
unsigned_integer_dtypes(endianness="="),
integer_dtypes(endianness="="),
floating_dtypes(endianness="=", sizes=(32, 64)),
)
return draw(one_of(dtypes))
def one_of_supported_dtypes(draw):
# A strategy that selects a dtype that riptable is known to handle.
# dtype size 16-bit is not supported
# little endian is not supported
return one_of(
boolean_dtypes(),
integer_dtypes(endianness="=", sizes=(8, 32, 64)),
unsigned_integer_dtypes(endianness="=", sizes=(8, 32, 64)),
floating_dtypes(endianness="=", sizes=(32, 64)),
byte_string_dtypes(endianness="="),
unicode_string_dtypes(endianness="="),
# the following dtypes are not supported
# complex_number_dtypes(),
# datetime64_dtypes(),
# timedelta64_dtypes(),
)
def numpy_number(draw, min_val, max_val):
dtype = draw(st.one_of(nps.integer_dtypes(),
nps.unsigned_integer_dtypes(),
nps.floating_dtypes()))
if 'f' in dtype.str:
if min_val < np.finfo(dtype).min:
min_val = np.finfo(dtype).min
if max_val > np.finfo(dtype).max:
max_val = np.finfo(dtype).max
number = draw(st.floats(min_val, max_val, allow_nan=False,
allow_infinity=False))
else:
if min_val < np.iinfo(dtype).min:
min_val = np.iinfo(dtype).min
if max_val > np.iinfo(dtype).max:
max_val = np.iinfo(dtype).max
number = draw(st.integers(min_val, max_val))
return np.array([number], dtype)[0]
def numpy_number(draw, min_val, max_val):
dtype = draw(
st.one_of(nps.integer_dtypes(), nps.unsigned_integer_dtypes(),
nps.floating_dtypes()))
if 'f' in dtype.str:
if min_val < np.finfo(dtype).min:
min_val = np.finfo(dtype).min
if max_val > np.finfo(dtype).max:
max_val = np.finfo(dtype).max
number = draw(
st.floats(min_val, max_val, allow_nan=False, allow_infinity=False))
else:
min_val, max_val = np.ceil(min_val), np.floor(max_val)
if min_val < np.iinfo(dtype).min:
min_val = np.iinfo(dtype).min
if max_val > np.iinfo(dtype).max:
max_val = np.iinfo(dtype).max
number = draw(st.integers(min_val, max_val))
return np.array([number], dtype)[0]
from __future__ import absolute_import, division, print_function
import hypothesis.extra.numpy as npst
import hypothesis.strategies as st
from hypothesis import given, settings
import xarray as xr
# Run for a while - arrays are a bigger search space than usual
settings.register_profile("ci", deadline=None)
settings.load_profile("ci")
an_array = npst.arrays(
dtype=st.one_of(
npst.unsigned_integer_dtypes(),
npst.integer_dtypes(),
npst.floating_dtypes(),
),
shape=npst.array_shapes(max_side=3), # max_side specified for performance
)
@given(st.data(), an_array)
def test_CFMask_coder_roundtrip(data, arr):
names = data.draw(st.lists(st.text(), min_size=arr.ndim,
max_size=arr.ndim, unique=True).map(tuple))
original = xr.Variable(names, arr)
coder = xr.coding.variables.CFMaskCoder()
roundtripped = coder.decode(coder.encode(original))
xr.testing.assert_identical(original, roundtripped)
with Tiff(filename, "w") as handle:
handle.write(img, method="tile")
with Tiff(filename) as handle:
data = handle[:]
assert np.all(img == data[:, :, :3])
with Tiff(filename, "w") as handle:
handle.write(img, method="scanline")
with Tiff(filename) as handle:
data = handle[:]
assert np.all(img == data[:, :, :3])
# scanline integer tests
@settings(buffer_size=11000000)
@given(data=hnp.arrays(dtype=st.one_of(hnp.integer_dtypes(endianness="="), hnp.unsigned_integer_dtypes(endianness="=")),
shape=hnp.array_shapes(min_dims=2, max_dims=2, min_side=10, max_side=50)))
def test_write_int_scanline(data, tmpdir_factory):
filename = str(tmpdir_factory.mktemp("write").join("int_img.tif"))
with Tiff(filename, "w") as handle:
handle.write(data, method="scanline")
with tifffile.TiffFile(filename) as handle:
img = handle.asarray()
assert data.dtype == img.dtype
assert np.all(data == img)
# tile integer tests
@settings(buffer_size=11000000)
@given(data=hnp.arrays(dtype=st.one_of(hnp.integer_dtypes(endianness="="), hnp.unsigned_integer_dtypes(endianness="=")),
from hypothesis.strategies import (
booleans,
composite,
floats,
integers,
lists,
one_of,
text,
tuples,
)
from superintendent.distributed.dbqueue import DatabaseQueue
guaranteed_dtypes = one_of(
np_strategies.scalar_dtypes(),
np_strategies.unsigned_integer_dtypes(),
np_strategies.datetime64_dtypes(),
np_strategies.floating_dtypes(),
np_strategies.integer_dtypes(),
)
@composite
def dataframe(draw):
n_cols = draw(integers(min_value=1, max_value=20))
dtypes = draw(
lists(
one_of(
np_strategies.floating_dtypes(),
np_strategies.integer_dtypes(),
np_strategies.unicode_string_dtypes(),
pytest.importorskip("hypothesis")
import hypothesis.extra.numpy as npst
import hypothesis.strategies as st
from hypothesis import given, settings
import xarray as xr
# Run for a while - arrays are a bigger search space than usual
settings.register_profile("ci", deadline=None)
settings.load_profile("ci")
an_array = npst.arrays(
dtype=st.one_of(
npst.unsigned_integer_dtypes(), npst.integer_dtypes(), npst.floating_dtypes()
),
shape=npst.array_shapes(max_side=3), # max_side specified for performance
)
@pytest.mark.slow
@given(st.data(), an_array)
def test_CFMask_coder_roundtrip(data, arr):
names = data.draw(
st.lists(st.text(), min_size=arr.ndim, max_size=arr.ndim, unique=True).map(
tuple
)
)
original = xr.Variable(names, arr)
coder = xr.coding.variables.CFMaskCoder()
@given(
hnp.arrays(
dtype=hnp.scalar_dtypes(),
shape=hypothesis_two_dimensional_array_shape(
ensure_even_second_dimension=True),
))
def test_separate_real_imag_of_mc_samples_second_half(array):
assert_equal(
separate_real_imag_of_mc_samples(array)[1], array[:,
len(array[0]) // 2:])
@given(
hnp.arrays(
dtype=hst.one_of(hnp.unsigned_integer_dtypes(), hnp.integer_dtypes(),
hnp.floating_dtypes()),
shape=hypothesis_two_dimensional_array_shape(
ensure_even_second_dimension=True),
))
def test_real_imag_2_complex_array_shape(array):
assert_equal(
real_imag_2_complex(array).shape, (len(array), len(array[0]) // 2))
@given(
hnp.arrays(
dtype=hst.one_of(hnp.unsigned_integer_dtypes(), hnp.integer_dtypes(),
hnp.floating_dtypes()),
shape=hypothesis_even_dimension(),
))
@pytest.mark.skip(
reason=
"Categorical generator needs to be rewritten for better performance before re-enabling this test to run in TeamCity builds."
)
@hypothesis.settings(suppress_health_check=[HealthCheck.too_slow])
@given(data())
@pytest.mark.parametrize(
"datatype, elements",
[
pytest.param(
integer_dtypes(endianness="=", sizes=(64, )),
integers(min_value=1, max_value=np.iinfo(np.int64).max),
id="integer_dtype",
),
pytest.param(
unsigned_integer_dtypes(endianness="=", sizes=(64, )),
integers(min_value=1, max_value=np.iinfo(np.int64).max),
id="unsigned_integer_dtype",
),
pytest.param(
byte_string_dtypes(endianness="="), None, id="byte_string_dtype"),
pytest.param(
datetime64_dtypes(endianness="="),
None,
id="datetime64_dtype",
marks=[
pytest.mark.xfail(
reason="RIP-375 - Categorical unsupported dtypes"),
pytest.mark.skip,
],
),
These ones pass, just as you'd hope!
"""
import hypothesis.extra.numpy as npst
import hypothesis.strategies as st
from hypothesis import given, settings
import xarray as xr
# Run for a while - arrays are a bigger search space than usual
settings.register_profile("ci", deadline=None)
settings.load_profile("ci")
an_array = npst.arrays(
dtype=st.one_of(
npst.unsigned_integer_dtypes(),
npst.integer_dtypes(),
npst.floating_dtypes(),
),
shape=npst.array_shapes(max_side=3), # max_side specified for performance
)
@given(st.data(), an_array)
def test_CFMask_coder_roundtrip(data, arr):
names = data.draw(
st.lists(st.text(), min_size=arr.ndim, max_size=arr.ndim,
unique=True).map(tuple))
original = xr.Variable(names, arr)
coder = xr.coding.variables.CFMaskCoder()
roundtripped = coder.decode(coder.encode(original))
# as typing.*IO from 3.6 onwards and mypy 0.730 errors on the compat form.
typing.io.BinaryIO: st.builds(io.BytesIO, st.binary()), # type: ignore
typing.io.TextIO: st.builds(io.StringIO, st.text()), # type: ignore
}
)
try:
# These aren't present in the typing module backport.
_global_type_lookup[typing.SupportsBytes] = st.binary()
_global_type_lookup[typing.SupportsRound] = st.complex_numbers()
except AttributeError: # pragma: no cover
pass
try:
strat = st.integers() | st.booleans()
if np is not None: # pragma: no branch
strat |= (unsigned_integer_dtypes() | integer_dtypes()).flatmap(from_dtype)
_global_type_lookup[typing.SupportsIndex] = strat # type: ignore
except AttributeError: # pragma: no cover
pass
def register(type_, fallback=None):
if isinstance(type_, str):
# Use the name of generic types which are not available on all
# versions, and the function just won't be added to the registry
type_ = getattr(typing, type_, None)
if type_ is None: # pragma: no cover
return lambda f: f
def inner(func):
if fallback is None:
_global_type_lookup[type_] = func
from typing import Optional
import hypothesis.extra.numpy as hnp
import hypothesis.strategies as st
import numpy as np
import pytest
from hypothesis import given, settings
from numpy.testing import assert_array_equal
from mygrad import Tensor
real_types = (hnp.integer_dtypes() | hnp.unsigned_integer_dtypes()
| hnp.floating_dtypes())
@given(
tensor=st.tuples(
hnp.arrays(shape=hnp.array_shapes(), dtype=real_types),
st.booleans(),
).map(lambda x: Tensor(x[0], constant=x[1])),
dest_type=real_types,
constant=st.booleans() | st.none(),
)
def test_astype(tensor: Tensor, dest_type: type, constant: Optional[bool]):
tensor = tensor * 1 # give tensor a creator
new_tensor = tensor.astype(dest_type, constant=constant)
assert new_tensor.constant is (tensor.constant
if constant is None else constant)
assert tensor.creator is not None
assert new_tensor.creator is None
from hypothesis import given
import hypothesis.strategies as st
import hypothesis.extra.numpy as hnp
import numpy as np
import pytest
from mygrad import Tensor
from mygrad.nnet.initializers import normal
@given(dtype=hnp.unsigned_integer_dtypes() | hnp.integer_dtypes()
| hnp.complex_number_dtypes())
def test_normal_dtype_validation(dtype):
with pytest.raises(ValueError):
normal(1, dtype=dtype)
@given(std=st.floats(-1000, 0, exclude_max=True))
def test_normal_std_validation(std):
with pytest.raises(ValueError):
normal(1, std=std)
_array_shapes = ((1000000, ), (1000, 100, 10), (10, 10, 10, 10, 10, 10))
@given(
shape=st.sampled_from(_array_shapes),
mean=st.floats(-100, 100),
std=st.floats(0, 5),
)
"""Make a covariance matrix from its Cholesky decomposition"""
lower = np.tril(cholesky)
# in a future version, np.diagonal will return a writable view
diag = np.diagonal(lower).copy()
diag[np.abs(diag) < 1e-3] = 1
lower[np.diag_indices_from(lower)] = diag
return lower @ lower.T
# Pre-defined strategies
some_shape = hn.array_shapes(max_dims=4, max_side=20)
simple_shape = hn.array_shapes(max_dims=2, max_side=10)
some_dtype = st.one_of(
# hn.boolean_dtypes(),
hn.integer_dtypes(),
hn.unsigned_integer_dtypes(),
hn.floating_dtypes(),
hn.complex_number_dtypes(),
# hn.datetime64_dtypes(),
# hn.timedelta64_dtypes(),
)
real_dtype = st.one_of(
# hn.boolean_dtypes(),
hn.integer_dtypes(),
hn.unsigned_integer_dtypes(),
hn.floating_dtypes(),
# hn.complex_number_dtypes(),
# hn.datetime64_dtypes(),
# hn.timedelta64_dtypes(),
)
some_array = hn.arrays(dtype=some_dtype, shape=some_shape)
from hypothesis import assume, given
import hypothesis.strategies as st
import hypothesis.extra.numpy as hnp
import numpy as np
from mygrad import Tensor
from mygrad.nnet.initializers import constant as constant_initializer
@given(
array=hnp.arrays(
dtype=hnp.integer_dtypes() | hnp.unsigned_integer_dtypes() | hnp.floating_dtypes(),
shape=hnp.array_shapes(),
),
constant=st.booleans(),
)
def test_constant_initializer(array, constant):
value = array.reshape(-1)[0] # let hypothesis do all the heavy lifting picking an acceptable value for the dtype
assume(not np.isnan(value))
tensor = constant_initializer(array.shape, value=value, dtype=array.dtype, constant=constant)
assert isinstance(tensor, Tensor)
assert np.allclose(tensor.data, value, equal_nan=True)
assert tensor.data.shape == array.shape
assert tensor.data.dtype == array.dtype
typing.io.BinaryIO:
st.builds(io.BytesIO, st.binary()), # type: ignore
typing.io.TextIO:
st.builds(io.StringIO, st.text()), # type: ignore
})
try:
# These aren't present in the typing module backport.
_global_type_lookup[typing.SupportsBytes] = st.binary()
_global_type_lookup[typing.SupportsRound] = st.complex_numbers()
except AttributeError: # pragma: no cover
pass
try:
strat = st.integers() | st.booleans()
if np is not None: # pragma: no branch
strat |= (unsigned_integer_dtypes()
| integer_dtypes()).flatmap(from_dtype)
_global_type_lookup[typing.SupportsIndex] = strat # type: ignore
except AttributeError: # pragma: no cover
pass
def register(type_, fallback=None):
if isinstance(type_, str):
# Use the name of generic types which are not available on all
# versions, and the function just won't be added to the registry
type_ = getattr(typing, type_, None)
if type_ is None: # pragma: no cover
return lambda f: f
def inner(func):
if fallback is None:
"""
from functools import partial
import numpy as np
import pandas as pd
import pytest
import xarray as xr
pytest.importorskip("hypothesis")
import hypothesis.extra.numpy as npst # isort:skip
import hypothesis.extra.pandas as pdst # isort:skip
import hypothesis.strategies as st # isort:skip
from hypothesis import given # isort:skip
numeric_dtypes = st.one_of(npst.unsigned_integer_dtypes(),
npst.integer_dtypes(), npst.floating_dtypes())
numeric_series = numeric_dtypes.flatmap(lambda dt: pdst.series(dtype=dt))
an_array = npst.arrays(
dtype=numeric_dtypes,
shape=npst.array_shapes(max_dims=2), # can only convert 1D/2D to pandas
)
@st.composite
def datasets_1d_vars(draw) -> xr.Dataset:
"""Generate datasets with only 1D variables
Suitable for converting to pandas dataframes.
assert np.all(img == data[:, :, :3])
with Tiff(filename, "w") as handle:
handle.write(img, method="scanline")
with Tiff(filename) as handle:
data = handle[:]
assert np.all(img == data[:, :, :3])
# scanline integer tests
@settings(buffer_size=11000000)
@given(data=hnp.arrays(dtype=st.one_of(
hnp.integer_dtypes(endianness="="),
hnp.unsigned_integer_dtypes(endianness="=")),
shape=hnp.array_shapes(min_dims=2,
max_dims=2,
min_side=10,
max_side=50)))
def test_write_int_scanline(data, tmpdir_factory):
filename = str(tmpdir_factory.mktemp("write").join("int_img.tif"))
with Tiff(filename, "w") as handle:
handle.write(data, method="scanline")
with tifffile.TiffFile(filename) as handle:
img = handle.asarray()
np.testing.assert_array_equal(data, img)
@settings(buffer_size=11000000)
nps.mutually_broadcastable_shapes(
num_shapes=2,
base_shape=base_shape,
min_side=0,
max_side=3,
min_dims=2,
max_dims=2,
),
f,
)
@settings(deadline=None)
@given(
shape=nps.array_shapes(min_dims=1, min_side=1),
dtype=st.one_of(nps.unsigned_integer_dtypes(), nps.integer_dtypes()),
data=st.data(),
)
def test_advanced_integer_index_is_valid_with_default_result_shape(shape, dtype, data):
index = data.draw(nps.integer_array_indices(shape, dtype=dtype))
x = np.zeros(shape)
out = x[index] # raises if the index is invalid
assert not np.shares_memory(x, out) # advanced indexing should not return a view
assert all(dtype == x.dtype for x in index)
@settings(deadline=None)
@given(
shape=nps.array_shapes(min_dims=1, min_side=1),
min_dims=st.integers(0, 3),
min_side=st.integers(0, 3),
with Tiff(filename, "w") as handle:
handle.write(img, method="tile")
with Tiff(filename) as handle:
data = handle[:]
assert np.all(img == data[:, :, :3])
with Tiff(filename, "w") as handle:
handle.write(img, method="scanline")
with Tiff(filename) as handle:
data = handle[:]
assert np.all(img == data[:, :, :3])
# scanline integer tests
@settings(buffer_size=11000000)
@given(data=hnp.arrays(dtype=st.one_of(hnp.integer_dtypes(endianness="="), hnp.unsigned_integer_dtypes(endianness="=")),
shape=hnp.array_shapes(min_dims=2, max_dims=2, min_side=10, max_side=50)))
def test_write_int_scanline(data, tmpdir_factory):
filename = str(tmpdir_factory.mktemp("write").join("int_img.tif"))
with Tiff(filename, "w") as handle:
handle.write(data, method="scanline")
with tifffile.TiffFile(filename) as handle:
img = handle.asarray()
np.testing.assert_array_equal(data, img)
@settings(buffer_size=11000000)
@given(data=hnp.arrays(dtype=st.one_of(hnp.integer_dtypes(endianness="="), hnp.unsigned_integer_dtypes(endianness="=")),
shape=hnp.array_shapes(min_dims=2, max_dims=2, min_side=10, max_side=50)))
def test_write_int_scanline_set_rows_per_strip(data, tmpdir_factory):
filename = str(tmpdir_factory.mktemp("write").join("int_img.tif"))
