def create_gradient_acquisition_fixtures():
# Create list of tuples of parameters with (fixture, tolerance) for acquisitions that gave gradients only
parameters = []
for acquisition in acquisition_tests:
if acquisition.has_gradients:
acquisition_name = acquisition.name
lazy_fixture = pytest_lazyfixture.lazy_fixture(acquisition.name)
parameters.append(pytest.param(lazy_fixture, acquisition.rmse_gradient_tolerance, id=acquisition_name))
return parameters
import numpy as np
import pytest
from pytest_lazyfixture import lazy_fixture
import stk
from ...case_data import CaseData
@pytest.fixture(
params=(
lazy_fixture('center'),
lazy_fixture('head'),
lazy_fixture('tail'),
lazy_fixture('unaligning'),
), )
def case_data(request):
return request.param
@pytest.fixture
def center(id, position, flip):
return CaseData(
vertex=stk.polymer.linear.LinearVertex(id, position, flip),
id=id,
position=position,
cell=np.array([0, 0, 0]),
)
@pytest.fixture
def create_fixture_parameters():
return [
pytest.param(lazy_fixture(warping.name), id=warping.name)
for warping in warpings
]
def parametrize_global_domains(func: Any) -> Any:
fixtures = [
lazy_fixture(f"global_domain_{i}") for i in range(GLOBAL_DOMAIN_COUNT)
]
return pytest.mark.parametrize("domain", fixtures)(func)
def lf(fname):
return lazy_fixture(fname)
class TestModisL2:
"""Test MODIS L2 reader."""
def test_available_reader(self):
"""Test that MODIS L2 reader is available."""
assert 'modis_l2' in available_readers()
def test_scene_available_datasets(self, modis_l2_nasa_mod35_file):
"""Test that datasets are available."""
scene = Scene(reader='modis_l2', filenames=modis_l2_nasa_mod35_file)
available_datasets = scene.all_dataset_names()
assert len(available_datasets) > 0
assert 'cloud_mask' in available_datasets
assert 'latitude' in available_datasets
assert 'longitude' in available_datasets
@pytest.mark.parametrize(
('input_files', 'has_5km', 'has_500', 'has_250', 'default_res'), [
[
lazy_fixture('modis_l2_nasa_mod35_file'), True, False, False,
1000
],
])
def test_load_longitude_latitude(self, input_files, has_5km, has_500,
has_250, default_res):
"""Test that longitude and latitude datasets are loaded correctly."""
from .test_modis_l1b import _load_and_check_geolocation
scene = Scene(reader='modis_l2', filenames=input_files)
shape_5km = _shape_for_resolution(5000)
shape_500m = _shape_for_resolution(500)
shape_250m = _shape_for_resolution(250)
default_shape = _shape_for_resolution(default_res)
with dask.config.set(scheduler=CustomScheduler(
max_computes=1 + has_5km + has_500 + has_250)):
_load_and_check_geolocation(scene,
"*",
default_res,
default_shape,
True,
check_callback=_check_shared_metadata)
_load_and_check_geolocation(scene,
5000,
5000,
shape_5km,
has_5km,
check_callback=_check_shared_metadata)
_load_and_check_geolocation(scene,
500,
500,
shape_500m,
has_500,
check_callback=_check_shared_metadata)
_load_and_check_geolocation(scene,
250,
250,
shape_250m,
has_250,
check_callback=_check_shared_metadata)
def test_load_quality_assurance(self, modis_l2_nasa_mod35_file):
"""Test loading quality assurance."""
scene = Scene(reader='modis_l2', filenames=modis_l2_nasa_mod35_file)
dataset_name = 'quality_assurance'
scene.load([dataset_name])
quality_assurance_id = make_dataid(name=dataset_name, resolution=1000)
assert quality_assurance_id in scene
quality_assurance = scene[quality_assurance_id]
assert quality_assurance.shape == _shape_for_resolution(1000)
_check_shared_metadata(quality_assurance, expect_area=True)
@pytest.mark.parametrize(
('input_files', 'loadables', 'request_resolution', 'exp_resolution',
'exp_area'), [
[
lazy_fixture('modis_l2_nasa_mod35_mod03_files'),
["cloud_mask"], 1000, 1000, True
],
[
lazy_fixture('modis_l2_imapp_mask_byte1_geo_files'),
["cloud_mask", "land_sea_mask", "snow_ice_mask"], None, 1000,
True
],
])
def test_load_category_dataset(self, input_files, loadables,
request_resolution, exp_resolution,
exp_area):
"""Test loading category products."""
scene = Scene(reader='modis_l2', filenames=input_files)
kwargs = {
"resolution": request_resolution
} if request_resolution is not None else {}
scene.load(loadables, **kwargs)
for ds_name in loadables:
cat_id = make_dataid(name=ds_name, resolution=exp_resolution)
assert cat_id in scene
cat_data_arr = scene[cat_id]
assert isinstance(cat_data_arr.data, da.Array)
cat_data_arr = cat_data_arr.compute()
assert cat_data_arr.shape == _shape_for_resolution(exp_resolution)
assert cat_data_arr.values[0, 0] == 0.0
assert cat_data_arr.attrs.get('resolution') == exp_resolution
# mask variables should be integers
assert np.issubdtype(cat_data_arr.dtype, np.integer)
assert cat_data_arr.attrs.get('_FillValue') is not None
_check_shared_metadata(cat_data_arr, expect_area=exp_area)
@pytest.mark.parametrize(('input_files', 'exp_area'), [
[lazy_fixture('modis_l2_nasa_mod35_file'), False],
[lazy_fixture('modis_l2_nasa_mod35_mod03_files'), True],
])
def test_load_250m_cloud_mask_dataset(self, input_files, exp_area):
"""Test loading 250m cloud mask."""
scene = Scene(reader='modis_l2', filenames=input_files)
dataset_name = 'cloud_mask'
scene.load([dataset_name], resolution=250)
cloud_mask_id = make_dataid(name=dataset_name, resolution=250)
assert cloud_mask_id in scene
cloud_mask = scene[cloud_mask_id]
assert isinstance(cloud_mask.data, da.Array)
cloud_mask = cloud_mask.compute()
assert cloud_mask.shape == _shape_for_resolution(250)
assert cloud_mask.values[0, 0] == 0.0
# mask variables should be integers
assert np.issubdtype(cloud_mask.dtype, np.integer)
assert cloud_mask.attrs.get('_FillValue') is not None
_check_shared_metadata(cloud_mask, expect_area=exp_area)
@pytest.mark.parametrize(
('input_files', 'loadables', 'exp_resolution', 'exp_area',
'exp_value'),
[
[
lazy_fixture('modis_l2_nasa_mod06_file'), ["surface_pressure"],
5000, True, 4.0
],
# snow mask is considered a category product, factor/offset ignored
[
lazy_fixture('modis_l2_imapp_snowmask_file'), ["snow_mask"],
1000, False, 1.0
],
[
lazy_fixture('modis_l2_imapp_snowmask_geo_files'),
["snow_mask"], 1000, True, 1.0
],
])
def test_load_l2_dataset(self, input_files, loadables, exp_resolution,
exp_area, exp_value):
"""Load and check an L2 variable."""
scene = Scene(reader='modis_l2', filenames=input_files)
scene.load(loadables)
for ds_name in loadables:
assert ds_name in scene
data_arr = scene[ds_name]
assert isinstance(data_arr.data, da.Array)
data_arr = data_arr.compute()
assert data_arr.values[0, 0] == exp_value
assert data_arr.shape == _shape_for_resolution(exp_resolution)
assert data_arr.attrs.get('resolution') == exp_resolution
_check_shared_metadata(data_arr, expect_area=exp_area)
from contextlib import nullcontext as does_not_raise
from unittest.mock import patch
import pytest
import xarray as xr
# need to import this way (rather than use pytest.lazy_fixture) to make it work with dask
from pytest_lazyfixture import lazy_fixture
from pangeo_forge_recipes.recipes.base import BaseRecipe
all_recipes = [
lazy_fixture("netCDFtoZarr_sequential_recipe"),
lazy_fixture("netCDFtoZarr_sequential_multi_variable_recipe"),
]
def test_to_pipelines_warns(netCDFtoZarr_sequential_recipe):
RecipeClass, file_pattern, kwargs, ds_expected, target = netCDFtoZarr_sequential_recipe
rec = RecipeClass(file_pattern, **kwargs)
with pytest.warns(FutureWarning):
rec.to_pipelines()
@pytest.mark.parametrize("recipe_fixture", all_recipes)
def test_recipe(recipe_fixture, execute_recipe):
"""The basic recipe test. Use this as a template for other tests."""
RecipeClass, file_pattern, kwargs, ds_expected, target = recipe_fixture
rec = RecipeClass(file_pattern, **kwargs)
execute_recipe(rec)
class TestSystemClient(object):
def test_lazy_system_loading(self, client):
assert client._loaded is False
assert client._system is None
send_mock = Mock()
client.send_bg_request = send_mock
client.command_1()
assert client._loaded is True
assert client._system is not None
assert client._commands is not None
assert send_mock.called is True
def test_no_attribute(self, client):
with pytest.raises(AttributeError):
client.command_3()
@pytest.mark.parametrize(
"constraint,systems",
[
("1.0.0", lazy_fixture("system_1")),
("latest", lazy_fixture("system_1")),
(None, lazy_fixture("system_1")),
pytest.param(
"1.0.0", None, marks=pytest.mark.xfail(raises=FetchError)),
],
)
def test_load_bg_system(self, client, easy_client, system_1, constraint,
systems):
client._version_constraint = constraint
easy_client.find_unique_system.return_value = systems
client.load_bg_system()
assert client._loaded is True
assert client._system == system_1
def test_load_bg_system_no_constraint_failure(self, client, easy_client):
easy_client.find_systems.return_value = []
with pytest.raises(FetchError):
client.load_bg_system()
def test_load_bg_system_latest(self, client, easy_client, system_1,
system_2):
easy_client.find_systems.return_value = [system_1, system_2]
client.load_bg_system()
assert client._system == system_2
def test_create_request_no_context(self, client, easy_client,
mock_success):
easy_client.create_request.return_value = mock_success
client.command_1()
assert easy_client.create_request.call_args[0][0].parent is None
@pytest.mark.parametrize(
"context,expected",
[
(None, None),
(Mock(current_request=None), None),
(
Mock(current_request=Mock(id="1"),
bg_host="localhost",
bg_port=3000),
"1",
),
(
Mock(current_request=Mock(id="1"),
bg_host="OTHER_BG",
bg_port=3000),
None,
),
],
)
def test_create_request_context(self, monkeypatch, client, easy_client,
mock_success, context, expected):
easy_client.create_request.return_value = mock_success
monkeypatch.setattr(brewtils.rest.system_client, "request_context",
context)
client.command_1()
parent = easy_client.create_request.call_args[0][0].parent
if expected is None:
assert parent is None
else:
assert parent.id == expected
@pytest.mark.parametrize(
"kwargs",
[
({
"_system_name": "",
"_system_version": "",
"_instance_name": ""
}),
({
"_command": "",
"_system_version": "",
"_instance_name": ""
}),
({
"_command": "",
"_system_name": "",
"_instance_name": ""
}),
({
"_command": "",
"_system_name": "",
"_system_version": ""
}),
],
)
def test_create_request_missing_fields(self, client, kwargs):
with pytest.raises(ValidationError):
client._construct_bg_request(**kwargs)
@pytest.mark.usefixtures("sleep_patch")
def test_execute_command_1(self, client, easy_client, mock_success,
mock_in_progress):
easy_client.find_unique_request.return_value = mock_success
easy_client.create_request.return_value = mock_in_progress
request = client.command_1(_blocking=False).result()
easy_client.find_unique_request.assert_called_with(
id=mock_in_progress.id)
assert request.status == mock_success.status
assert request.output == mock_success.output
@pytest.mark.usefixtures("sleep_patch")
def test_execute_command_1_error_raise(self, client, easy_client,
mock_error):
easy_client.create_request.return_value = mock_error
with pytest.raises(RequestFailedError) as ex:
client.command_1(_raise_on_error=True)
assert ex.value.request.status == mock_error.status
assert ex.value.request.output == mock_error.output
@pytest.mark.usefixtures("sleep_patch")
def test_execute_command_1_error_raise_future(self, client, easy_client,
mock_error):
easy_client.create_request.return_value = mock_error
future = client.command_1(_blocking=False, _raise_on_error=True)
with pytest.raises(RequestFailedError) as ex:
future.result()
assert ex.value.request.status == mock_error.status
assert ex.value.request.output == mock_error.output
@pytest.mark.usefixtures("sleep_patch")
def test_execute_command_1_error(self, client, easy_client, mock_error):
easy_client.create_request.return_value = mock_error
request = client.command_1(_raise_on_error=False)
assert request.status == mock_error.status
assert request.output == mock_error.output
def test_execute_command_with_delays(self, client, easy_client,
sleep_patch, mock_success,
mock_in_progress):
easy_client.create_request.return_value = mock_in_progress
easy_client.find_unique_request.side_effect = [
mock_in_progress,
mock_in_progress,
mock_success,
]
client.command_1(_blocking=False).result()
sleep_patch.assert_has_calls([call(0.5), call(1.0), call(2.0)])
easy_client.find_unique_request.assert_called_with(
id=mock_in_progress.id)
def test_execute_with_max_delay(self, client, easy_client, mock_success,
mock_in_progress, sleep_patch):
easy_client.create_request.return_value = mock_in_progress
easy_client.find_unique_request.side_effect = [
mock_in_progress,
mock_in_progress,
mock_success,
]
client._max_delay = 1
client.command_1(_blocking=False).result()
sleep_patch.assert_has_calls([call(0.5), call(1.0), call(1.0)])
easy_client.find_unique_request.assert_called_with(
id=mock_in_progress.id)
@pytest.mark.usefixtures("sleep_patch")
def test_execute_with_timeout(self, client, easy_client, mock_in_progress):
easy_client.create_request.return_value = mock_in_progress
easy_client.find_unique_request.return_value = mock_in_progress
client._timeout = 1
future = client.command_1(_blocking=False)
with pytest.raises(TimeoutExceededError):
future.result()
easy_client.find_unique_request.assert_called_with(
id=mock_in_progress.id)
@pytest.mark.usefixtures("sleep_patch")
def test_execute_non_blocking_command_1(self, client, easy_client,
mock_success, mock_in_progress):
easy_client.find_unique_request.return_value = mock_success
easy_client.create_request.return_value = mock_in_progress
request = client.command_1(_blocking=False).result()
easy_client.find_unique_request.assert_called_with(
id=mock_in_progress.id)
assert request.status == mock_success.status
assert request.output == mock_success.output
@pytest.mark.usefixtures("sleep_patch")
def test_execute_non_blocking_multiple_commands(self, client, easy_client,
mock_success,
mock_in_progress):
easy_client.find_unique_request.return_value = mock_success
easy_client.create_request.return_value = mock_in_progress
futures = [client.command_1(_blocking=False) for _ in range(3)]
wait(futures)
easy_client.find_unique_request.assert_called_with(
id=mock_in_progress.id)
for future in futures:
result = future.result()
assert result.status == mock_success.status
assert result.output == mock_success.output
@pytest.mark.usefixtures("sleep_patch")
def test_execute_non_blocking_multiple_commands_timeout(
self, client, easy_client, mock_in_progress):
easy_client.find_unique_request.return_value = mock_in_progress
easy_client.create_request.return_value = mock_in_progress
client._timeout = 1
futures = [client.command_1(_blocking=False) for _ in range(3)]
wait(futures)
easy_client.find_unique_request.assert_called_with(
id=mock_in_progress.id)
for future in futures:
with pytest.raises(TimeoutExceededError):
future.result()
def test_always_update(self, client, easy_client, mock_success):
client._always_update = True
client.load_bg_system()
easy_client.create_request.return_value = mock_success
load_mock = Mock()
client.load_bg_system = load_mock
client.command_1()
assert load_mock.called is True
def test_retry_send_no_different_version(self, client, easy_client):
easy_client.create_request.side_effect = ValidationError
with pytest.raises(ValidationError):
client.command_1()
assert easy_client.create_request.call_count == 1
def test_retry_send_different_version(self, client, easy_client, system_2,
mock_success):
client.load_bg_system()
easy_client.find_systems.return_value = [system_2]
easy_client.create_request.side_effect = [
ValidationError, mock_success
]
client.command_1()
assert client._system.version == "2.0.0"
assert easy_client.create_request.call_count == 2
@pytest.mark.parametrize(
"latest,versions",
[
("1.0.0", ["1.0.0"]),
("2.0.0", ["1.0.0", "2.0.0"]),
("1.2.0", ["1.0.0", "1.2.0"]),
("1.0.0", ["1.0.0", "0.2.1rc1"]),
("1.0.0rc1", ["1.0.0rc1", "0.2.1"]),
("1.0.0rc1", ["1.0.0rc1", "0.2.1rc1"]),
("1.0", ["1.0", "0.2.1"]),
("1.0.0", ["1.0.0rc1", "1.0.0"]),
("b", ["a", "b"]),
("1.0.0", ["a", "b", "1.0.0"]),
],
)
def test_determine_latest(self, client, versions, latest):
systems = [Mock(version=version) for version in versions]
assert client._determine_latest(systems).version == latest
def create_acquisition_fixture_parameters():
return [pytest.param(pytest_lazyfixture.lazy_fixture(acq.name), id=acq.name) for acq in acquisition_tests]
from pytest import fixture, mark
from pytest_lazyfixture import lazy_fixture
from auditor.audit_tables import audit_tables, TallyTables, AuditTables
def test_successful_audit(opened_data: AuditTables,
commitment_data: TallyTables):
assert audit_tables(opened_data, commitment_data) is True
@mark.parametrize(
"invalid_opened_data",
(
lazy_fixture("invalid_opened_data_one_table_longer"),
lazy_fixture("invalid_opened_data_all_columns_opened"),
lazy_fixture("invalid_opened_data_wrong_keys"),
lazy_fixture("invalid_opened_data_missing_keys"),
),
)
def test_invalid_data_audit(invalid_opened_data: AuditTables,
commitment_data: TallyTables):
assert audit_tables(invalid_opened_data, commitment_data) is False
@mark.parametrize(
"invalid_commitment_data",
(
lazy_fixture("invalid_commitment_data_invalid_table"),
lazy_fixture("invalid_commitment_data_one_table_longer"),
lazy_fixture("invalid_commitment_data_wrong_keys"),
class TestNearestNeighborResampler:
"""Test the KDTreeNearestXarrayResampler class."""
def test_nearest_swath_1d_mask_to_grid_1n(self, swath_def_1d_xarray_dask,
data_1d_float32_xarray_dask,
coord_def_2d_float32_dask):
"""Test 1D swath definition to 2D grid definition; 1 neighbor."""
resampler = KDTreeNearestXarrayResampler(swath_def_1d_xarray_dask,
coord_def_2d_float32_dask)
res = resampler.resample(
data_1d_float32_xarray_dask,
mask_area=data_1d_float32_xarray_dask.isnull(),
radius_of_influence=100000)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
_check_common_metadata(
res, isinstance(coord_def_2d_float32_dask, AreaDefinition))
actual = res.values
expected = np.array([
[1., 2., 2.],
[1., 2., 2.],
[1., np.nan, 2.],
[1., 2., 2.],
])
np.testing.assert_allclose(actual, expected)
def test_nearest_type_preserve(self, swath_def_1d_xarray_dask,
coord_def_2d_float32_dask):
"""Test 1D swath definition to 2D grid definition; 1 neighbor."""
data = xr.DataArray(da.from_array(np.array([1, 2, 3]), chunks=5),
dims=('my_dim1', ))
resampler = KDTreeNearestXarrayResampler(swath_def_1d_xarray_dask,
coord_def_2d_float32_dask)
res = resampler.resample(data,
fill_value=255,
radius_of_influence=100000)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
_check_common_metadata(
res, isinstance(coord_def_2d_float32_dask, AreaDefinition))
actual = res.values
expected = np.array([
[1, 2, 2],
[1, 2, 2],
[1, 255, 2],
[1, 2, 2],
])
np.testing.assert_equal(actual, expected)
def test_nearest_swath_2d_mask_to_area_1n(self, swath_def_2d_xarray_dask,
data_2d_float32_xarray_dask,
area_def_stere_target):
"""Test 2D swath definition to 2D area definition; 1 neighbor."""
resampler = KDTreeNearestXarrayResampler(swath_def_2d_xarray_dask,
area_def_stere_target)
res = resampler.resample(data_2d_float32_xarray_dask,
radius_of_influence=50000)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
_check_common_metadata(
res, isinstance(area_def_stere_target, AreaDefinition))
res = res.values
cross_sum = float(np.nansum(res))
expected = 167913.0
assert cross_sum == expected
assert res.shape == resampler.target_geo_def.shape
def test_nearest_area_2d_to_area_1n(self, area_def_stere_source,
data_2d_float32_xarray_dask,
area_def_stere_target):
"""Test 2D area definition to 2D area definition; 1 neighbor."""
resampler = KDTreeNearestXarrayResampler(area_def_stere_source,
area_def_stere_target)
with assert_maximum_dask_computes(0):
resampler.precompute(radius_of_influence=50000)
with assert_maximum_dask_computes(0):
res = resampler.resample(data_2d_float32_xarray_dask,
radius_of_influence=50000)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
_check_common_metadata(
res, isinstance(area_def_stere_target, AreaDefinition))
res = res.values
cross_sum = float(np.nansum(res))
expected = 303048.0
assert cross_sum == expected
assert res.shape == resampler.target_geo_def.shape
def test_nearest_swath_2d_to_area_1n_pm180(
self, swath_def_2d_xarray_dask_antimeridian,
data_2d_float32_xarray_dask, area_def_lonlat_pm180_target):
"""Test 2D swath definition to 2D area definition; 1 neighbor; output prime meridian at 180 degrees."""
resampler = KDTreeNearestXarrayResampler(
swath_def_2d_xarray_dask_antimeridian,
area_def_lonlat_pm180_target)
res = resampler.resample(data_2d_float32_xarray_dask,
radius_of_influence=50000)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
_check_common_metadata(
res, isinstance(area_def_lonlat_pm180_target, AreaDefinition))
res = res.values
cross_sum = float(np.nansum(res))
expected = 115591.0
assert cross_sum == expected
assert res.shape == resampler.target_geo_def.shape
def test_nearest_area_2d_to_area_1n_no_roi(self, area_def_stere_source,
data_2d_float32_xarray_dask,
area_def_stere_target):
"""Test 2D area definition to 2D area definition; 1 neighbor, no radius of influence."""
resampler = KDTreeNearestXarrayResampler(area_def_stere_source,
area_def_stere_target)
resampler.precompute()
res = resampler.resample(data_2d_float32_xarray_dask)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
_check_common_metadata(
res, isinstance(area_def_stere_target, AreaDefinition))
res = res.values
cross_sum = float(np.nansum(res))
expected = 952386.0
assert cross_sum == expected
assert res.shape == resampler.target_geo_def.shape
def test_nearest_area_2d_to_area_1n_no_roi_no_geocentric(
self, area_def_stere_source, data_2d_float32_xarray_dask,
area_def_stere_target):
# pretend the resolutions can't be determined
with mock.patch.object(area_def_stere_source, 'geocentric_resolution') as sgr, \
mock.patch.object(area_def_stere_target, 'geocentric_resolution') as dgr:
sgr.side_effect = RuntimeError
dgr.side_effect = RuntimeError
resampler = KDTreeNearestXarrayResampler(area_def_stere_source,
area_def_stere_target)
res = resampler.resample(data_2d_float32_xarray_dask)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
_check_common_metadata(
res, isinstance(area_def_stere_target, AreaDefinition))
res = res.values
cross_sum = np.nansum(res)
expected = 20666.0
assert cross_sum == expected
assert res.shape == resampler.target_geo_def.shape
@pytest.mark.parametrize("input_data", [
lazy_fixture("data_2d_float32_numpy"),
lazy_fixture("data_2d_float32_dask"),
lazy_fixture("data_2d_float32_xarray_numpy"),
])
def test_object_type_with_warnings(self, area_def_stere_source,
area_def_stere_target, input_data):
"""Test that providing certain input data causes a warning."""
resampler = KDTreeNearestXarrayResampler(area_def_stere_source,
area_def_stere_target)
with catch_warnings(
PerformanceWarning) as w, assert_maximum_dask_computes(1):
res = resampler.resample(input_data)
assert type(res) is type(input_data)
_check_common_metadata(
res, isinstance(area_def_stere_target, AreaDefinition))
is_data_arr_dask = isinstance(input_data, xr.DataArray) and isinstance(
input_data.data, da.Array)
is_dask_based = isinstance(input_data, da.Array) or is_data_arr_dask
if is_dask_based:
assert not w
else:
assert_warnings_contain(w, "will be converted")
res = np.array(res)
cross_sum = np.nansum(res)
expected = 952386.0 # same as 'test_nearest_area_2d_to_area_1n_no_roi'
assert cross_sum == expected
assert res.shape == resampler.target_geo_def.shape
def test_nearest_area_2d_to_area_1n_3d_data(self, area_def_stere_source,
data_3d_float32_xarray_dask,
area_def_stere_target):
"""Test 2D area definition to 2D area definition; 1 neighbor, 3d data."""
resampler = KDTreeNearestXarrayResampler(area_def_stere_source,
area_def_stere_target)
resampler.precompute(radius_of_influence=50000)
res = resampler.resample(data_3d_float32_xarray_dask,
radius_of_influence=50000)
assert isinstance(res, xr.DataArray)
assert isinstance(res.data, da.Array)
assert list(res.coords['bands']) == ['r', 'g', 'b']
_check_common_metadata(
res, isinstance(area_def_stere_target, AreaDefinition))
res = res.values
cross_sum = float(np.nansum(res))
expected = 909144.0
assert cross_sum == expected
assert res.shape[:2] == resampler.target_geo_def.shape
class TestInvalidUsageNearestNeighborResampler:
"""Test the resampler being given input that should raise an error.
If a case here is removed because functionality is added to the resampler
then a working case should be added above.
"""
@pytest.mark.parametrize("input_data", [
lazy_fixture("data_2d_float32_xarray_dask"),
lazy_fixture("data_3d_float32_xarray_dask"),
])
def test_mismatch_geo_data_dims(
self,
area_def_stere_source,
area_def_stere_target,
input_data,
):
resampler = KDTreeNearestXarrayResampler(area_def_stere_source,
area_def_stere_target)
data = input_data.rename({'y': 'my_dim_y', 'x': 'my_dim_x'})
with pytest.raises(ValueError, match='.*dimensions do not match.*'):
resampler.resample(data)
def test_mismatch_geo_data_dims_swath(self, swath_def_2d_xarray_dask,
area_def_stere_target,
data_2d_float32_xarray_dask):
new_swath_def = SwathDefinition(
swath_def_2d_xarray_dask.lons.rename({
'y': 'my_dim_y',
'x': 'my_dim_x'
}),
swath_def_2d_xarray_dask.lats.rename({
'y': 'my_dim_y',
'x': 'my_dim_x'
}))
resampler = KDTreeNearestXarrayResampler(new_swath_def,
area_def_stere_target)
with pytest.raises(ValueError, match='.*dimensions do not match.*'):
resampler.resample(data_2d_float32_xarray_dask)
@pytest.mark.parametrize("src_geom", [
lazy_fixture("area_def_stere_source"),
lazy_fixture("swath_def_2d_xarray_dask")
])
@pytest.mark.parametrize(("match", "call_precompute"), [
(".*data.*shape.*", False),
(".*'mask'.*shape.*", True),
])
def test_inconsistent_input_shapes(self, src_geom, match, call_precompute,
area_def_stere_target,
data_2d_float32_xarray_dask):
"""Test that geometry and data of the same size but different size still error."""
# transpose the source geometries
if isinstance(src_geom, AreaDefinition):
src_geom = AreaDefinition(
src_geom.area_id,
src_geom.description,
src_geom.proj_id,
src_geom.crs,
src_geom.height,
src_geom.width,
src_geom.area_extent,
)
else:
src_geom = SwathDefinition(
src_geom.lons.T.rename({
'y': 'x',
'x': 'y'
}),
src_geom.lats.T.rename({
'y': 'x',
'x': 'y'
}),
)
resampler = KDTreeNearestXarrayResampler(src_geom,
area_def_stere_target)
with pytest.raises(ValueError, match=match):
if call_precompute:
resampler.precompute(
mask=data_2d_float32_xarray_dask.notnull())
else:
resampler.resample(data_2d_float32_xarray_dask)
"""
return request.param
@pytest.fixture(
params=(1, 2, 9), )
def bond_order(request):
"""
The bond order of a bond created by a :class:`.Reaction`.
"""
return request.param
@pytest.fixture(
params=(
lazy_fixture('one_one_reaction'),
lazy_fixture('one_two_reaction'),
lazy_fixture('two_two_reaction'),
lazy_fixture('dative_reaction'),
), )
def case_data(request):
"""
A :class:`.CaseData` instance.
"""
return request.param
@pytest.fixture
def wrapper_matern52_1(dim2, gpy_prodmatern52):
return get_wrapper_dict(dim2, measure_lebesgue, gpy_prodmatern52,
ProductMatern52GPy,
QuadratureProductMatern52LebesgueMeasure)
@pytest.fixture
def wrapper_matern52_2(dim1, gpy_matern52):
return get_wrapper_dict(dim1, measure_lebesgue, gpy_matern52,
ProductMatern52GPy,
QuadratureProductMatern52LebesgueMeasure)
gpy_test_list = [
lazy_fixture("wrapper_rbf_1"),
lazy_fixture("wrapper_rbf_2"),
lazy_fixture("wrapper_brownian_1"),
lazy_fixture("wrapper_brownian_2"),
lazy_fixture("wrapper_matern32_1"),
lazy_fixture("wrapper_matern32_2"),
lazy_fixture("wrapper_matern52_1"),
lazy_fixture("wrapper_matern52_2"),
]
@pytest.mark.parametrize("wrapper", gpy_test_list)
def test_create_emukit_model_from_gpy_model_types(wrapper):
gpy_model = GPy.models.GPRegression(kernel=wrapper["gpy_kernel"],
X=wrapper["data"][0],
def create_acquisition_fixture_parameters():
return [pytest.param(pytest_lazyfixture.lazy_fixture(acq.name), id=acq.name) for acq in acquisition_tests]
[-10, 20, -30],
[0.5, 10, -0.921],
])
def origin(request):
return np.array(request.param)
@pytest.fixture
def get_origin_0(origin):
return lambda molecule: origin
@pytest.fixture(
params=[
lambda molecule: molecule.get_centroid(),
lazy_fixture('get_origin_0'),
], )
def get_origin(request):
"""
Return an origin parameter for a molecule.
Parameters
----------
molecule : :class:`.Molecule`
The molecule which needs an origin parameter for some method.
Returns
-------
:class:`numpy.ndarray`
The origin parameter to use.
bucket = storage_client.create_bucket(bucket)
bucket.add_lifecycle_delete_rule(
age=14) # delete buckets automatically after 14 days
bucket.patch()
bucket.blob("registry.db")
return FeatureStore(config=RepoConfig(
registry=f"gs://{bucket_name}/registry.db",
project="default",
provider="gcp",
))
@pytest.mark.parametrize(
"test_feature_store",
[lazy_fixture("feature_store_with_local_registry")],
)
def test_apply_entity_success(test_feature_store):
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
# Register Entity with Core
test_feature_store.apply([entity])
entities = test_feature_store.list_entities()
entity = entities[0]
class TestClient:
@pytest.fixture
def secure_mock_client(self):
client = Client(
core_url=CORE_URL,
serving_url=SERVING_URL,
core_enable_ssl=True,
serving_enable_ssl=True,
)
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
@pytest.fixture
def mock_client(self):
client = Client(core_url=CORE_URL, serving_url=SERVING_URL)
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
@pytest.fixture
def mock_client_with_auth(self):
client = Client(
core_url=CORE_URL,
serving_url=SERVING_URL,
enable_auth=True,
auth_token=_FAKE_JWT_TOKEN,
)
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
@pytest.fixture
def secure_mock_client_with_auth(self):
client = Client(
core_url=CORE_URL,
serving_url=SERVING_URL,
core_enable_ssl=True,
serving_enable_ssl=True,
enable_auth=True,
auth_token=_FAKE_JWT_TOKEN,
)
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
@pytest.fixture
def server_credentials(self):
private_key = pkgutil.get_data(__name__, _PRIVATE_KEY_RESOURCE_PATH)
certificate_chain = pkgutil.get_data(__name__,
_CERTIFICATE_CHAIN_RESOURCE_PATH)
return grpc.ssl_server_credentials(
((private_key, certificate_chain), ))
@pytest.fixture
def core_server(self):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
port = find_free_port()
server.add_insecure_port(f"[::]:{port}")
server.start()
yield port
server.stop(0)
@pytest.fixture
def serving_server(self):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
Serving.add_ServingServiceServicer_to_server(ServingServicer(), server)
port = find_free_port()
server.add_insecure_port(f"[::]:{port}")
server.start()
yield port
server.stop(0)
@pytest.fixture
def secure_core_server(self, server_credentials):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
port = find_free_port()
server.add_secure_port(f"[::]:{port}", server_credentials)
server.start()
yield port
server.stop(0)
@pytest.fixture
def secure_serving_server(self, server_credentials):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
Serving.add_ServingServiceServicer_to_server(ServingServicer(), server)
port = find_free_port()
server.add_secure_port(f"[::]:{port}", server_credentials)
server.start()
yield port
server.stop(0)
@pytest.fixture
def secure_core_server_with_auth(self, server_credentials):
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10),
interceptors=(AllowAuthInterceptor(), ),
)
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
port = find_free_port()
server.add_secure_port(f"[::]:{port}", server_credentials)
server.start()
yield port
server.stop(0)
@pytest.fixture
def insecure_core_server_with_auth(self, server_credentials):
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10),
interceptors=(AllowAuthInterceptor(), ),
)
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
port = find_free_port()
server.add_insecure_port(f"[::]:{port}")
server.start()
yield port
server.stop(0)
@pytest.fixture
def insecure_core_server_that_blocks_auth(self, server_credentials):
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10),
interceptors=(DisallowAuthInterceptor(), ),
)
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
port = find_free_port()
server.add_insecure_port(f"[::]:{port}")
server.start()
yield port
server.stop(0)
@pytest.fixture
def secure_client(self, secure_core_server, secure_serving_server):
root_certificate_credentials = pkgutil.get_data(
__name__, _ROOT_CERTIFICATE_RESOURCE_PATH)
ssl_channel_credentials = grpc.ssl_channel_credentials(
root_certificates=root_certificate_credentials)
with mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value=ssl_channel_credentials),
):
yield Client(
core_url=f"localhost:{secure_core_server}",
serving_url=f"localhost:{secure_serving_server}",
core_enable_ssl=True,
serving_enable_ssl=True,
)
@pytest.fixture
def secure_core_client_with_auth(self, secure_core_server_with_auth):
root_certificate_credentials = pkgutil.get_data(
__name__, _ROOT_CERTIFICATE_RESOURCE_PATH)
ssl_channel_credentials = grpc.ssl_channel_credentials(
root_certificates=root_certificate_credentials)
with mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value=ssl_channel_credentials),
):
yield Client(
core_url=f"localhost:{secure_core_server_with_auth}",
core_enable_ssl=True,
enable_auth=True,
auth_token=_FAKE_JWT_TOKEN,
)
@pytest.fixture
def client(self, core_server, serving_server):
return Client(
core_url=f"localhost:{core_server}",
serving_url=f"localhost:{serving_server}",
)
@pytest.fixture
def partitioned_df(self):
# Partitioned DataFrame
N_ROWS = 100
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
final_offset = ([time_offset] * 33 +
[time_offset - timedelta(days=1)] * 33 +
[time_offset - timedelta(days=2)] * 34)
final_part_offset = ([time_offset - timedelta(days=99)] * 33 +
[time_offset - timedelta(days=100)] * 33 +
[time_offset - timedelta(days=101)] * 34)
return pd.DataFrame({
"datetime":
final_offset,
"datetime_col":
final_part_offset,
"dev_feature_float": [np.float(row) for row in range(N_ROWS)],
"dev_feature_string":
["feat_" + str(row) for row in range(N_ROWS)],
})
@pytest.fixture
def non_partitioned_df(self):
# Non-Partitioned DataFrame
N_ROWS = 100
time_offset = datetime.utcnow().replace(tzinfo=pytz.utc)
return pd.DataFrame({
"datetime": [time_offset] * N_ROWS,
"dev_feature_float": [np.float(row) for row in range(N_ROWS)],
"dev_feature_string":
["feat_" + str(row) for row in range(N_ROWS)],
})
@pytest.fixture
def get_online_features_fields_statuses(self):
ROW_COUNT = 100
fields_statuses_tuple_list = []
for row_number in range(0, ROW_COUNT):
fields_statuses_tuple_list.append((
{
"driver_id": ValueProto.Value(int64_val=row_number),
"driver:age": ValueProto.Value(int64_val=1),
"driver:rating": ValueProto.Value(string_val="9"),
"driver:null_value": ValueProto.Value(),
},
{
"driver_id":
GetOnlineFeaturesResponse.FieldStatus.PRESENT,
"driver:age":
GetOnlineFeaturesResponse.FieldStatus.PRESENT,
"driver:rating":
GetOnlineFeaturesResponse.FieldStatus.PRESENT,
"driver:null_value":
GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE,
},
))
return fields_statuses_tuple_list
@pytest.mark.parametrize(
"mocked_client",
[lazy_fixture("mock_client"),
lazy_fixture("secure_mock_client")],
)
def test_version(self, mocked_client, mocker):
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeastCoreVersion",
return_value=GetFeastCoreVersionResponse(version="0.3.2"),
)
mocker.patch.object(
mocked_client._serving_service_stub,
"GetFeastServingInfo",
return_value=GetFeastServingInfoResponse(version="0.3.2"),
)
status = mocked_client.version()
assert (status["core"]["url"] == CORE_URL
and status["core"]["version"] == "0.3.2"
and status["serving"]["url"] == SERVING_URL
and status["serving"]["version"] == "0.3.2")
@pytest.mark.parametrize(
"mocked_client",
[
lazy_fixture("mock_client"),
lazy_fixture("mock_client_with_auth"),
lazy_fixture("secure_mock_client"),
lazy_fixture("secure_mock_client_with_auth"),
],
)
def test_get_historical_features(self, mocked_client, mocker):
assert 1 == 1
@pytest.mark.parametrize(
"test_client",
[lazy_fixture("client"),
lazy_fixture("secure_client")],
)
def test_apply_entity_success(self, test_client):
test_client.set_project("project1")
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
# Register Entity with Core
test_client.apply_entity(entity)
entities = test_client.list_entities()
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(
ValueProto.ValueType.STRING).name
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
@pytest.mark.parametrize(
"test_client",
[lazy_fixture("client"),
lazy_fixture("secure_client")],
)
def test_apply_feature_table_success(self, test_client):
test_client.set_project("project1")
# Create Feature Tables
batch_source = FileSource(
file_format=ParquetFormat(),
file_url="file://feast/*",
event_timestamp_column="ts_col",
created_timestamp_column="timestamp",
date_partition_column="date_partition_col",
)
stream_source = KafkaSource(
bootstrap_servers="localhost:9094",
message_format=ProtoFormat("class.path"),
topic="test_topic",
event_timestamp_column="ts_col",
)
ft1 = FeatureTable(
name="my-feature-table-1",
features=[
Feature(name="fs1-my-feature-1", dtype=ValueType.INT64),
Feature(name="fs1-my-feature-2", dtype=ValueType.STRING),
Feature(name="fs1-my-feature-3", dtype=ValueType.STRING_LIST),
Feature(name="fs1-my-feature-4", dtype=ValueType.BYTES_LIST),
],
entities=["fs1-my-entity-1"],
labels={"team": "matchmaking"},
batch_source=batch_source,
stream_source=stream_source,
)
# Register Feature Table with Core
test_client.apply_feature_table(ft1)
feature_tables = test_client.list_feature_tables()
# List Feature Tables
assert (len(feature_tables) == 1
and feature_tables[0].name == "my-feature-table-1"
and feature_tables[0].features[0].name == "fs1-my-feature-1"
and feature_tables[0].features[0].dtype == ValueType.INT64
and feature_tables[0].features[1].name == "fs1-my-feature-2"
and feature_tables[0].features[1].dtype == ValueType.STRING
and feature_tables[0].features[2].name == "fs1-my-feature-3"
and feature_tables[0].features[2].dtype
== ValueType.STRING_LIST
and feature_tables[0].features[3].name == "fs1-my-feature-4"
and feature_tables[0].features[3].dtype == ValueType.BYTES_LIST
and feature_tables[0].entities[0] == "fs1-my-entity-1")
@pytest.mark.parametrize(
"test_client", [lazy_fixture("client"),
lazy_fixture("secure_client")])
def test_list_features(self, test_client, mocker):
mocker.patch.object(
test_client,
"_core_service_stub",
return_value=Core.CoreServiceStub(grpc.insecure_channel("")),
)
feature1_proto = FeatureSpecProto(
name="feature_1", value_type=ValueProto.ValueType.FLOAT)
feature2_proto = FeatureSpecProto(
name="feature_2", value_type=ValueProto.ValueType.STRING)
mocker.patch.object(
test_client._core_service_stub,
"ListFeatures",
return_value=ListFeaturesResponse(
features={
"driver_car:feature_1": feature1_proto,
"driver_car:feature_2": feature2_proto,
}),
)
features = test_client.list_features_by_ref(project="test")
assert len(features) == 2
native_feature_list = []
for _, feature_proto in features.items():
native_feature_list.append(feature_proto)
assert sorted(native_feature_list) == sorted([
Feature.from_proto(feature1_proto),
Feature.from_proto(feature2_proto)
])
@pytest.mark.parametrize(
"mocked_client",
[lazy_fixture("mock_client")],
)
def test_ingest_dataframe_partition(self, mocked_client, mocker,
partitioned_df, tmp_path):
"""
Test ingestion with local FileSource, using DataFrame.
Partition column stated but not provided in Dataset.
"""
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeatureTable",
return_value=_ingest_test_getfeaturetable_mocked_resp(
f"file://{tmp_path}", "date"),
)
mocked_client.set_project("my_project")
ft = mocked_client.get_feature_table("ingest_featuretable")
mocked_client.ingest(ft, partitioned_df, timeout=600)
pq_df = pq.read_table(tmp_path).to_pandas().drop(columns=["date"])
partitioned_df, pq_df = _ingest_test_format_dataframes(
partitioned_df, pq_df, True)
assert_frame_equal(partitioned_df, pq_df)
@pytest.mark.parametrize(
"mocked_client",
[lazy_fixture("mock_client")],
)
def test_ingest_dataframe_no_partition(self, mocked_client, mocker,
non_partitioned_df, tmp_path):
"""
Test ingestion with local FileSource, using DataFrame.
Partition column not stated.
"""
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeatureTable",
return_value=_ingest_test_getfeaturetable_mocked_resp(
f"file://{tmp_path}"),
)
mocked_client.set_project("my_project")
ft = mocked_client.get_feature_table("ingest_featuretable")
mocked_client.ingest(ft, non_partitioned_df, timeout=600)
# Since not partitioning, we're only looking for single file
single_file = [
f for f in os.listdir(tmp_path)
if os.path.isfile(os.path.join(tmp_path, f))
][0]
pq_df = pq.read_table(tmp_path / single_file).to_pandas()
non_partitioned_df, pq_df = _ingest_test_format_dataframes(
non_partitioned_df, pq_df)
assert_frame_equal(non_partitioned_df, pq_df)
@pytest.mark.parametrize(
"mocked_client",
[lazy_fixture("mock_client")],
)
def test_ingest_csv(self, mocked_client, mocker, tmp_path):
"""
Test ingestion with local FileSource, using CSV file.
Partition column is provided.
"""
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeatureTable",
return_value=_ingest_test_getfeaturetable_mocked_resp(
f"file://{tmp_path}", "date"),
)
partitioned_df = pd.read_csv(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"./data/dev_featuretable.csv",
),
parse_dates=["datetime"],
)
mocked_client.set_project("my_project")
ft = mocked_client.get_feature_table("ingest_featuretable")
mocked_client.ingest(ft, partitioned_df, timeout=600)
pq_df = pq.read_table(tmp_path).to_pandas().drop(columns=["date"])
partitioned_df, pq_df = _ingest_test_format_dataframes(
partitioned_df, pq_df, True)
assert_frame_equal(partitioned_df, pq_df)
@pytest.mark.parametrize(
"mocked_client,auth_metadata",
[
(lazy_fixture("mock_client"), ()),
(lazy_fixture("mock_client_with_auth"), (AUTH_METADATA)),
(lazy_fixture("secure_mock_client"), ()),
(lazy_fixture("secure_mock_client_with_auth"), (AUTH_METADATA)),
],
ids=[
"mock_client_without_auth",
"mock_client_with_auth",
"secure_mock_client_without_auth",
"secure_mock_client_with_auth",
],
)
def test_get_online_features(self, mocked_client, auth_metadata, mocker,
get_online_features_fields_statuses):
ROW_COUNT = 100
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
request = GetOnlineFeaturesRequestV2(project="driver_project")
request.features.extend([
FeatureRefProto(feature_table="driver", name="age"),
FeatureRefProto(feature_table="driver", name="rating"),
FeatureRefProto(feature_table="driver", name="null_value"),
])
receive_response = GetOnlineFeaturesResponse()
entity_rows = []
for row_number in range(0, ROW_COUNT):
fields = get_online_features_fields_statuses[row_number][0]
statuses = get_online_features_fields_statuses[row_number][1]
request.entity_rows.append(
GetOnlineFeaturesRequestV2.EntityRow(
fields={
"driver_id": ValueProto.Value(int64_val=row_number)
}))
entity_rows.append(
{"driver_id": ValueProto.Value(int64_val=row_number)})
receive_response.field_values.append(
GetOnlineFeaturesResponse.FieldValues(fields=fields,
statuses=statuses))
mocker.patch.object(
mocked_client._serving_service_stub,
"GetOnlineFeaturesV2",
return_value=receive_response,
)
got_response = mocked_client.get_online_features(
entity_rows=entity_rows,
feature_refs=["driver:age", "driver:rating", "driver:null_value"],
project="driver_project",
) # type: GetOnlineFeaturesResponse
mocked_client._serving_service_stub.GetOnlineFeaturesV2.assert_called_with(
request, metadata=auth_metadata, timeout=10)
got_fields = got_response.field_values[1].fields
got_statuses = got_response.field_values[1].statuses
assert (got_fields["driver_id"] == ValueProto.Value(int64_val=1)
and got_statuses["driver_id"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver:age"] == ValueProto.Value(int64_val=1)
and got_statuses["driver:age"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT and
got_fields["driver:rating"] == ValueProto.Value(string_val="9")
and got_statuses["driver:rating"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver:null_value"] == ValueProto.Value()
and got_statuses["driver:null_value"]
== GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE)
@pytest.mark.parametrize(
"mocked_client,auth_metadata",
[
(lazy_fixture("mock_client"), ()),
(lazy_fixture("mock_client_with_auth"), (AUTH_METADATA)),
(lazy_fixture("secure_mock_client"), ()),
(lazy_fixture("secure_mock_client_with_auth"), (AUTH_METADATA)),
],
ids=[
"mock_client_without_auth",
"mock_client_with_auth",
"secure_mock_client_without_auth",
"secure_mock_client_with_auth",
],
)
def test_get_online_features_multi_entities(
self, mocked_client, auth_metadata, mocker,
get_online_features_fields_statuses):
ROW_COUNT = 100
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
request = GetOnlineFeaturesRequestV2(project="driver_project")
request.features.extend([
FeatureRefProto(feature_table="driver", name="age"),
FeatureRefProto(feature_table="driver", name="rating"),
FeatureRefProto(feature_table="driver", name="null_value"),
])
receive_response = GetOnlineFeaturesResponse()
entity_rows = []
for row_number in range(0, ROW_COUNT):
fields = get_online_features_fields_statuses[row_number][0]
fields["driver_id2"] = ValueProto.Value(int64_val=1)
statuses = get_online_features_fields_statuses[row_number][1]
statuses[
"driver_id2"] = GetOnlineFeaturesResponse.FieldStatus.PRESENT
request.entity_rows.append(
GetOnlineFeaturesRequestV2.EntityRow(
fields={
"driver_id": ValueProto.Value(int64_val=row_number),
"driver_id2": ValueProto.Value(int64_val=row_number),
}))
entity_rows.append({
"driver_id":
ValueProto.Value(int64_val=row_number),
"driver_id2":
ValueProto.Value(int64_val=row_number),
})
receive_response.field_values.append(
GetOnlineFeaturesResponse.FieldValues(fields=fields,
statuses=statuses))
mocker.patch.object(
mocked_client._serving_service_stub,
"GetOnlineFeaturesV2",
return_value=receive_response,
)
got_response = mocked_client.get_online_features(
entity_rows=entity_rows,
feature_refs=["driver:age", "driver:rating", "driver:null_value"],
project="driver_project",
) # type: GetOnlineFeaturesResponse
mocked_client._serving_service_stub.GetOnlineFeaturesV2.assert_called_with(
request, metadata=auth_metadata, timeout=10)
got_fields = got_response.field_values[1].fields
got_statuses = got_response.field_values[1].statuses
assert (got_fields["driver_id"] == ValueProto.Value(int64_val=1)
and got_statuses["driver_id"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver_id2"] == ValueProto.Value(int64_val=1)
and got_statuses["driver_id2"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver:age"] == ValueProto.Value(int64_val=1)
and got_statuses["driver:age"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT and
got_fields["driver:rating"] == ValueProto.Value(string_val="9")
and got_statuses["driver:rating"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver:null_value"] == ValueProto.Value()
and got_statuses["driver:null_value"]
== GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE)
@patch("grpc.channel_ready_future")
def test_secure_channel_creation_with_secure_client(
self, _mocked_obj, core_server, serving_server):
client = Client(
core_url=f"localhost:{core_server}",
serving_url=f"localhost:{serving_server}",
serving_enable_ssl=True,
core_enable_ssl=True,
)
with mock.patch("grpc.secure_channel") as _grpc_mock, mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value="test")) as _mocked_credentials:
_ = client._serving_service
_grpc_mock.assert_called_with(
client.serving_url,
credentials=_mocked_credentials.return_value)
@mock.patch("grpc.channel_ready_future")
def test_secure_channel_creation_with_secure_serving_url(
self, _mocked_obj, core_server):
client = Client(core_url=f"localhost:{core_server}",
serving_url="localhost:443")
with mock.patch("grpc.secure_channel") as _grpc_mock, mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value="test")) as _mocked_credentials:
_ = client._serving_service
_grpc_mock.assert_called_with(
client.serving_url,
credentials=_mocked_credentials.return_value)
@patch("grpc.channel_ready_future")
def test_secure_channel_creation_with_secure_core_url(
self, _mocked_obj, secure_serving_server):
client = Client(
core_url="localhost:443",
serving_url=f"localhost:{secure_serving_server}",
)
with mock.patch("grpc.secure_channel") as _grpc_mock, mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value="test")) as _mocked_credentials:
_ = client._core_service
_grpc_mock.assert_called_with(
client.core_url, credentials=_mocked_credentials.return_value)
@mock.patch("grpc.channel_ready_future")
def test_auth_success_with_secure_channel_on_core_url(
self, secure_core_client_with_auth):
secure_core_client_with_auth.list_feature_tables()
def test_auth_success_with_insecure_channel_on_core_url(
self, insecure_core_server_with_auth):
client = Client(
core_url=f"localhost:{insecure_core_server_with_auth}",
enable_auth=True,
auth_token=_FAKE_JWT_TOKEN,
)
client.list_feature_tables()
def test_no_auth_sent_when_auth_disabled(
self, insecure_core_server_that_blocks_auth):
client = Client(
core_url=f"localhost:{insecure_core_server_that_blocks_auth}")
client.list_feature_tables()
import pytest
from pytest_lazyfixture import lazy_fixture
# Fixtures must be visible for lazy_fixture() calls.
from .fixtures import * # noqa
@pytest.fixture(
params=(
lazy_fixture('add'),
lazy_fixture('divide_by_mean'),
lazy_fixture('multiply'),
lazy_fixture('null'),
lazy_fixture('power'),
lazy_fixture('replace_fitness'),
lazy_fixture('sequence'),
lazy_fixture('shift_up'),
lazy_fixture('sum'),
), )
def case_data(request):
return request.param
@pytest.fixture
def local_writable_url(tmp_path):
"""Return a writable local URL."""
path = str(tmp_path)
url = f"file://{path}"
yield url
TREE = [
"2018-01-01/type=one/file.ext",
"2018-01-02/type=two/file.ext",
"2018-01-03/type=three/file1.ext",
"2018-01-03/type=three/file2.ext",
]
@pytest.fixture(
params=[
lazy_fixture("s3_writable_url"),
# lazy_fixture("local_writable_url")
]
)
def file_tree(request):
base_url = request.param
tree = [base_url + "/" + sub_path for sub_path in TREE]
for file_url in tree:
with fsspec.open(file_url, "w") as f:
f.write("bla")
return base_url, tree
from ..fixtures import *
from dacapo.store import create_config_store
import pytest
from pytest_lazyfixture import lazy_fixture
@pytest.mark.parametrize(
"array_config",
[
lazy_fixture("cellmap_array"),
lazy_fixture("zarr_array"),
lazy_fixture("dummy_array"),
],
)
def test_array_api(options, array_config):
# create_config_store (uses options behind the scenes)
store = create_config_store()
# Test store/retrieve
store.store_array_config(array_config)
fetched_array_config = store.retrieve_array_config(array_config.name)
assert fetched_array_config == array_config
# Create Array from config
array = array_config.array_type(array_config)
# Test API
# channels/axes
if "c" in array.axes:
import pytest
import stk
import numpy as np
from pytest_lazyfixture import lazy_fixture
from ...case_data import CaseData
vertices = stk.polymer.vertices
@pytest.fixture(
params=(
lazy_fixture('center'),
lazy_fixture('head'),
lazy_fixture('tail'),
), )
def case_data(request):
return request.param
@pytest.fixture
def center(id, position, flip):
return CaseData(
vertex=vertices._LinearVertex(id, position, flip),
id=id,
position=position,
cell=np.array([0, 0, 0]),
)
@pytest.fixture
class TestSystemAPI(object):
@pytest.mark.gen_test
def test_get(self, http_client, base_url, system_dict, mongo_system, system_id):
mongo_system.deep_save()
response = yield http_client.fetch(base_url + "/api/v1/systems/" + system_id)
assert 200 == response.code
assert system_dict == json.loads(response.body.decode("utf-8"))
@pytest.mark.gen_test
def test_get_404(self, http_client, base_url, system_id):
response = yield http_client.fetch(
base_url + "/api/v1/systems/" + system_id, raise_error=False
)
assert 404 == response.code
@pytest.mark.gen_test
@pytest.mark.parametrize(
"field,value,dev,succeed",
[
# No changes
(None, None, True, True),
(None, None, False, True),
# Command name change
("name", "new", True, True),
("name", "new", False, False),
# Parameter name change
("parameters", lazy_fixture("key_parameter"), True, True),
("parameters", lazy_fixture("key_parameter"), False, False),
# Parameter choices change
("parameters", lazy_fixture("choices_parameter"), True, True),
("parameters", lazy_fixture("choices_parameter"), False, True),
],
)
def test_patch_commands(
self,
http_client,
base_url,
mongo_system,
system_id,
bg_command,
field,
value,
dev,
succeed,
):
if dev:
mongo_system.version += ".dev"
mongo_system.deep_save()
# Make changes to the new command
if field:
if field == "parameters":
value = [value]
setattr(bg_command, field, value)
# Also delete the id, otherwise mongo gets really confused
delattr(bg_command, "id")
body = PatchOperation(
operation="replace",
path="/commands",
value=SchemaParser.serialize_command(
[bg_command], to_string=False, many=True
),
)
request = HTTPRequest(
base_url + "/api/v1/systems/" + system_id,
method="PATCH",
headers={"content-type": "application/json"},
body=SchemaParser.serialize_patch(body),
)
response = yield http_client.fetch(request, raise_error=False)
if succeed:
assert response.code == 200
updated = SchemaParser.parse_system(
response.body.decode("utf-8"), from_string=True
)
assert_command_equal(bg_command, updated.commands[0])
else:
assert response.code == 400
@pytest.fixture
def feature_store_with_s3_registry():
return FeatureStore(config=RepoConfig(
registry=
f"s3://feast-integration-tests/registries/{int(time.time() * 1000)}/registry.db",
project="default",
provider="aws",
online_store=DynamoDBOnlineStoreConfig(region="us-west-2"),
offline_store=FileOfflineStoreConfig(),
))
@pytest.mark.parametrize(
"test_feature_store",
[lazy_fixture("feature_store_with_local_registry")],
)
def test_apply_entity_success(test_feature_store):
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
tags={"team": "matchmaking"},
)
# Register Entity
test_feature_store.apply(entity)
entities = test_feature_store.list_entities()
entity = entities[0]
conditions = [
clf_dataset_name != exp_dataset_name,
]
return any(conditions)
@pytest.mark.uncollect_if(func=uncollect_if_test_explainer)
@pytest.mark.parametrize('n_explainer_runs', [5], ids='n_exp_runs={}'.format)
@pytest.mark.parametrize('at_defaults', [0.9, 0.95],
ids='threshold={}'.format,
indirect=True)
@pytest.mark.parametrize(
'rf_classifier',
[
lazy_fixture('iris_data'),
], # lazy_fixture('adult_data')],
indirect=True,
ids='clf=rf_{}'.format,
)
@pytest.mark.parametrize(
'explainer',
[
lazy_fixture('at_iris_explainer'),
], # lazy_fixture('at_adult_explainer')],
ids='exp={}'.format,
)
@pytest.mark.parametrize('test_instance_idx', [0],
ids='test_instance_idx={}'.format)
def test_explainer(n_explainer_runs, at_defaults, rf_classifier, explainer,
test_instance_idx, caplog):
def parametrize_global_problems(func: Any) -> Any:
fixtures = [
lazy_fixture(f"global_problem_{i}")
for i in range(GLOBAL_PROBLEM_COUNT)
]
return pytest.mark.parametrize("problem", fixtures)(func)
return test_module.one()
if option != "failure" and per_task_actor:
Actor = Actor.options(runtime_env=env)
a = Actor.remote()
if option == "failure":
with pytest.raises(ImportError):
assert ray.get(a.test_import.remote()) == 2
else:
assert ray.get(a.test_import.remote()) == 2
@pytest.mark.skipif(sys.platform == "win32", reason="Fail to create temp dir.")
@pytest.mark.parametrize("option", ["working_dir", "py_modules"])
@pytest.mark.parametrize("source", [*REMOTE_URIS, lazy_fixture("tmp_working_dir")])
def test_multi_node(start_cluster, option: str, source: str):
"""Tests that the working_dir is propagated across multi-node clusters."""
NUM_NODES = 3
cluster, address = start_cluster
for i in range(NUM_NODES - 1): # Head node already added.
cluster.add_node(num_cpus=1, runtime_env_dir_name=f"node_{i}_runtime_resources")
if option == "working_dir":
ray.init(address, runtime_env={"working_dir": source})
elif option == "py_modules":
if source not in REMOTE_URIS:
source = str(Path(source) / "test_module")
ray.init(address, runtime_env={"py_modules": [source]})
@ray.remote(num_cpus=1)
def command_context(autospec, message) -> discord.ext.commands.Context:
"""Returns a context with nested properties set, for each channel type a command can be sent to."""
c = autospec.of(discord.ext.commands.Context)
c.message = message
c.channel = message.channel
c.author = message.author
return c
@pytest.fixture
def interaction_context(autospec, interaction, command) -> duckbot.slash.InteractionContext:
"""Returns an interaction context with nested properties set, for each channel type a slash command can be sent to."""
c = autospec.of(duckbot.slash.InteractionContext)
c.interaction = interaction
c.command = command
c.message = interaction.message
c.channel = interaction.channel
c.author = interaction.user
return c
@pytest.fixture(
params=[
lazy_fixture(command_context.__name__),
lazy_fixture(interaction_context.__name__),
]
)
def context(request) -> Union[discord.ext.commands.Context, duckbot.slash.InteractionContext]:
"""Returns a set of discord.py command contexts and slash command contexts."""
return request.param
import pytest
from pytest_lazyfixture import lazy_fixture
# All fixtures must be visible for lazy_fixture() call.
from .octahedral import * # noqa
from .paddlewheel import * # noqa
from .porphyrin import * # noqa
from .square_planar import * # noqa
@pytest.fixture(
params=(
lazy_fixture('metal_complex_octahedral'),
lazy_fixture('metal_complex_octahedral_lambda'),
lazy_fixture('metal_complex_octahedral_delta'),
lazy_fixture('metal_complex_porphyrin'),
lazy_fixture('metal_complex_paddlewheel'),
lazy_fixture('metal_complex_square_planar'),
lazy_fixture('metal_complex_bidentate_square_planar'),
lazy_fixture('metal_complex_cis_protected_square_planar'),
), )
def metal_complex(request):
return request.param
import pytest
from pytest_lazyfixture import lazy_fixture
from .case_data import CaseData
# Fixtures need to be visible for lazy_fixture() calls.
from .fixtures import * # noqa
@pytest.fixture(
params=(lazy_fixture('constructed_molecule_mongo_db'), ), )
def case_data(request) -> CaseData:
return request.param
if option != "failure" and per_task_actor:
Actor = Actor.options(runtime_env=env)
a = Actor.remote()
if option == "failure":
with pytest.raises(ImportError):
assert ray.get(a.test_import.remote()) == 2
else:
assert ray.get(a.test_import.remote()) == 2
@pytest.mark.skipif(sys.platform == "win32", reason="Fail to create temp dir.")
@pytest.mark.parametrize("option", ["working_dir", "py_modules"])
@pytest.mark.parametrize(
"source", [*REMOTE_URIS, lazy_fixture("tmp_working_dir")])
def test_multi_node(start_cluster, option: str, source: str):
"""Tests that the working_dir is propagated across multi-node clusters."""
NUM_NODES = 3
cluster, address = start_cluster
for i in range(NUM_NODES - 1): # Head node already added.
cluster.add_node(num_cpus=1,
runtime_env_dir_name=f"node_{i}_runtime_resources")
if option == "working_dir":
ray.init(address, runtime_env={"working_dir": source})
elif option == "py_modules":
if source not in REMOTE_URIS:
source = str(Path(source) / "test_module")
ray.init(address, runtime_env={"py_modules": [source]})
"unique_identifier":"******************************",
"original_transaction_id":"******************************",
"expires_date":"******************************",
"transaction_id":"******************************",
"quantity":"1",
"product_id":"******************************",
"item_id":"******************************",
"bid":"******************************",
"unique_vendor_identifier":"******************************",
"web_order_line_item_id":"******************************",
"bvrs":"1.1.6",
"expires_date_formatted":"2017-09-27 15:09:30 Etc/GMT",
"purchase_date":"2017-09-27 15:04:30 Etc/GMT",
"purchase_date_ms":"1506524670000",
"expires_date_formatted_pst":"2017-09-27 08:09:30 America/Los_Angeles",
"purchase_date_pst":"2017-09-27 08:04:30 America/Los_Angeles",
"original_purchase_date":"2017-06-28 14:31:51 Etc/GMT",
"original_purchase_date_ms":"1498660311000"
},
"latest_receipt":"******************************"
}''')
@pytest.fixture(params=[
lazy_fixture('itunes_autorenew_response1'),
lazy_fixture('itunes_autorenew_response2'),
lazy_fixture('itunes_autorenew_response3'),
])
def itunes_autorenew_response(request):
return request.param
import pytest
from pytest_lazyfixture import lazy_fixture
# Fixtures must be visible for lazy_fixture() calls.
from .fixtures import * # noqa
@pytest.fixture(
params=(
lazy_fixture('roulette'),
lazy_fixture('stochastic_universal_sampling'),
lazy_fixture('tournament'),
), )
def case_data(request):
return request.param
