def s3_base(worker_id):
"""
Fixture for mocking S3 interaction.
Sets up moto server in separate process locally
Return url for motoserver/moto CI service
"""
pytest.importorskip("s3fs")
pytest.importorskip("boto3")
with tm.ensure_safe_environment_variables():
# temporary workaround as moto fails for botocore >= 1.11 otherwise,
# see https://github.com/spulec/moto/issues/1924 & 1952
os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
if os.environ.get("PANDAS_CI", "0") == "1":
if is_platform_arm() or is_platform_mac() or is_platform_windows():
# NOT RUN on Windows/MacOS/ARM, only Ubuntu
# - subprocess in CI can cause timeouts
# - Azure pipelines/Github Actions do not support
# container services for the above OSs
# - CircleCI will probably hit the Docker rate pull limit
pytest.skip("S3 tests do not have a corresponding service in "
"Windows, MacOS or ARM platforms")
else:
yield "http://localhost:5000"
else:
requests = pytest.importorskip("requests")
pytest.importorskip("moto", minversion="1.3.14")
pytest.importorskip("flask") # server mode needs flask too
# Launching moto in server mode, i.e., as a separate process
# with an S3 endpoint on localhost
worker_id = "5" if worker_id == "master" else worker_id.lstrip(
"gw")
endpoint_port = f"555{worker_id}"
endpoint_uri = f"http://127.0.0.1:{endpoint_port}/"
# pipe to null to avoid logging in terminal
with subprocess.Popen(
shlex.split(f"moto_server s3 -p {endpoint_port}"),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
) as proc:
timeout = 5
while timeout > 0:
try:
# OK to go once server is accepting connections
r = requests.get(endpoint_uri)
if r.ok:
break
except Exception:
pass
timeout -= 0.1
time.sleep(0.1)
yield endpoint_uri
proc.terminate()
class TestFloatSubtype(AstypeTests):
"""Tests specific to IntervalIndex with float subtype"""
indexes = [
interval_range(-10.0, 10.0, closed="neither"),
IntervalIndex.from_arrays([-1.5, np.nan, 0.0, 0.0, 1.5],
[-0.5, np.nan, 1.0, 1.0, 3.0],
closed="both"),
]
@pytest.fixture(params=indexes)
def index(self, request):
return request.param
@pytest.mark.parametrize("subtype", ["int64", "uint64"])
def test_subtype_integer(self, subtype):
index = interval_range(0.0, 10.0)
dtype = IntervalDtype(subtype, "right")
result = index.astype(dtype)
expected = IntervalIndex.from_arrays(index.left.astype(subtype),
index.right.astype(subtype),
closed=index.closed)
tm.assert_index_equal(result, expected)
# raises with NA
msg = r"Cannot convert non-finite values \(NA or inf\) to integer"
with pytest.raises(ValueError, match=msg):
index.insert(0, np.nan).astype(dtype)
@pytest.mark.parametrize("subtype", ["int64", "uint64"])
def test_subtype_integer_with_non_integer_borders(self, subtype):
index = interval_range(0.0, 3.0, freq=0.25)
dtype = IntervalDtype(subtype, "right")
result = index.astype(dtype)
expected = IntervalIndex.from_arrays(index.left.astype(subtype),
index.right.astype(subtype),
closed=index.closed)
tm.assert_index_equal(result, expected)
@pytest.mark.xfail(is_platform_arm(), reason="GH 41740")
def test_subtype_integer_errors(self):
# float64 -> uint64 fails with negative values
index = interval_range(-10.0, 10.0)
dtype = IntervalDtype("uint64", "right")
msg = re.escape(
"Cannot convert interval[float64, right] to interval[uint64, right]; "
"subtypes are incompatible")
with pytest.raises(TypeError, match=msg):
index.astype(dtype)
@pytest.mark.parametrize("subtype", ["datetime64[ns]", "timedelta64[ns]"])
def test_subtype_datetimelike(self, index, subtype):
dtype = IntervalDtype(subtype, "right")
msg = "Cannot convert .* to .*; subtypes are incompatible"
with pytest.raises(TypeError, match=msg):
index.astype(dtype)
([1.0, 1.1], "Float64", "integer", "Float64"),
([1, pd.NA], "Int64", "integer", "Int8"),
([450, 300], "Int64", "integer", "Int16"),
([1, 1], "Float64", "integer", "Int8"),
([np.iinfo(np.int64).max - 1, 1], "Int64", "integer", "Int64"),
([1, 1], "Int64", "signed", "Int8"),
([1.0, 1.0], "Float32", "signed", "Int8"),
([1.0, 1.1], "Float64", "signed", "Float64"),
([1, pd.NA], "Int64", "signed", "Int8"),
([450, -300], "Int64", "signed", "Int16"),
pytest.param(
[np.iinfo(np.uint64).max - 1, 1],
"UInt64",
"signed",
"UInt64",
marks=pytest.mark.xfail(not is_platform_arm(), reason="GH38798"),
),
([1, 1], "Int64", "unsigned", "UInt8"),
([1.0, 1.0], "Float32", "unsigned", "UInt8"),
([1.0, 1.1], "Float64", "unsigned", "Float64"),
([1, pd.NA], "Int64", "unsigned", "UInt8"),
([450, -300], "Int64", "unsigned", "Int64"),
([-1, -1], "Int32", "unsigned", "Int32"),
([1, 1], "Float64", "float", "Float32"),
([1, 1.1], "Float64", "float", "Float32"),
([1, 1], "Float32", "float", "Float32"),
([1, 1.1], "Float32", "float", "Float32"),
),
)
def test_downcast_nullable_numeric(data, input_dtype, downcast,
expected_dtype):
class TestUInt64Index(NumericInt):
_index_cls = UInt64Index
@pytest.fixture
def dtype(self):
return np.uint64
@pytest.fixture(
params=["int64", "float64", "category", "datetime64", "object"], )
def invalid_dtype(self, request):
return request.param
@pytest.fixture
def simple_index(self, dtype):
# compat with shared Int64/Float64 tests
return self._index_cls(np.arange(5, dtype=dtype))
@pytest.fixture(
params=[
[2**63, 2**63 + 10, 2**63 + 15, 2**63 + 20, 2**63 + 25],
[2**63 + 25, 2**63 + 20, 2**63 + 15, 2**63 + 10, 2**63],
],
ids=["index_inc", "index_dec"],
)
def index(self, request):
return self._index_cls(request.param)
def test_constructor(self, dtype):
index_cls = self._index_cls
idx = index_cls([1, 2, 3])
res = Index([1, 2, 3], dtype=dtype)
tm.assert_index_equal(res, idx)
idx = index_cls([1, 2**63])
res = Index([1, 2**63], dtype=dtype)
tm.assert_index_equal(res, idx)
idx = index_cls([1, 2**63])
res = Index([1, 2**63])
tm.assert_index_equal(res, idx)
idx = Index([-1, 2**63], dtype=object)
res = Index(np.array([-1, 2**63], dtype=object))
tm.assert_index_equal(res, idx)
# https://github.com/pandas-dev/pandas/issues/29526
idx = index_cls([1, 2**63 + 1], dtype=dtype)
res = Index([1, 2**63 + 1], dtype=dtype)
tm.assert_index_equal(res, idx)
@pytest.mark.xfail(
not is_platform_arm(),
reason="https://github.com/numpy/numpy/issues/19146",
)
def test_constructor_does_not_cast_to_float(self):
# https://github.com/numpy/numpy/issues/19146
values = [0, np.iinfo(np.uint64).max]
result = UInt64Index(values)
assert list(result) == values
result = getattr(ds.rolling(window, closed="left"), func)()
expected = Series(values, index=index)
tm.assert_series_equal(result, expected)
def test_rolling_sem(frame_or_series):
# GH: 26476
obj = frame_or_series([0, 1, 2])
result = obj.rolling(2, min_periods=1).sem()
if isinstance(result, DataFrame):
result = Series(result[0].values)
expected = Series([np.nan] + [0.7071067811865476] * 2)
tm.assert_series_equal(result, expected)
@pytest.mark.xfail(is_platform_arm() and not is_platform_mac(),
reason="GH 38921")
@pytest.mark.parametrize(
("func", "third_value", "values"),
[
("var", 1, [5e33, 0, 0.5, 0.5, 2, 0]),
("std", 1, [7.071068e16, 0, 0.7071068, 0.7071068, 1.414214, 0]),
("var", 2, [5e33, 0.5, 0, 0.5, 2, 0]),
("std", 2, [7.071068e16, 0.7071068, 0, 0.7071068, 1.414214, 0]),
],
)
def test_rolling_var_numerical_issues(func, third_value, values):
# GH: 37051
ds = Series([99999999999999999, 1, third_value, 2, 3, 1, 1])
result = getattr(ds.rolling(2), func)()
expected = Series([np.nan] + values)
result = getattr(ds.rolling(window, closed="left"), func)()
expected = Series(values, index=index)
tm.assert_series_equal(result, expected)
def test_rolling_sem(frame_or_series):
# GH: 26476
obj = frame_or_series([0, 1, 2])
result = obj.rolling(2, min_periods=1).sem()
if isinstance(result, DataFrame):
result = Series(result[0].values)
expected = Series([np.nan] + [0.707107] * 2)
tm.assert_series_equal(result, expected)
@pytest.mark.xfail(is_platform_arm(), reason="GH 41740")
@pytest.mark.parametrize(
("func", "third_value", "values"),
[
("var", 1, [5e33, 0, 0.5, 0.5, 2, 0]),
("std", 1, [7.071068e16, 0, 0.7071068, 0.7071068, 1.414214, 0]),
("var", 2, [5e33, 0.5, 0, 0.5, 2, 0]),
("std", 2, [7.071068e16, 0.7071068, 0, 0.7071068, 1.414214, 0]),
],
)
def test_rolling_var_numerical_issues(func, third_value, values):
# GH: 37051
ds = Series([99999999999999999, 1, third_value, 2, 3, 1, 1])
result = getattr(ds.rolling(2), func)()
expected = Series([np.nan] + values)
tm.assert_series_equal(result, expected)
result = getattr(ds.rolling(window, closed="left"), func)()
expected = Series(values, index=index)
tm.assert_series_equal(result, expected)
def test_rolling_sem(frame_or_series):
# GH: 26476
obj = frame_or_series([0, 1, 2])
result = obj.rolling(2, min_periods=1).sem()
if isinstance(result, DataFrame):
result = Series(result[0].values)
expected = Series([np.nan] + [0.7071067811865476] * 2)
tm.assert_series_equal(result, expected)
@pytest.mark.xfail(is_platform_arm() and not is_platform_mac(), reason="GH 38921")
@pytest.mark.parametrize(
("func", "third_value", "values"),
[
("var", 1, [5e33, 0, 0.5, 0.5, 2, 0]),
("std", 1, [7.071068e16, 0, 0.7071068, 0.7071068, 1.414214, 0]),
("var", 2, [5e33, 0.5, 0, 0.5, 2, 0]),
("std", 2, [7.071068e16, 0.7071068, 0, 0.7071068, 1.414214, 0]),
],
)
def test_rolling_var_numerical_issues(func, third_value, values):
# GH: 37051
ds = Series([99999999999999999, 1, third_value, 2, 3, 1, 1])
result = getattr(ds.rolling(2), func)()
expected = Series([np.nan] + values)
tm.assert_series_equal(result, expected)
