def server(self):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
server.add_insecure_port(f"[::]:{free_port}")
server.start()
yield server
server.stop(0)
def secure_core_server(self, server_credentials):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
port = find_free_port()
server.add_secure_port(f"[::]:{port}", server_credentials)
server.start()
yield port
server.stop(0)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
server.add_insecure_port("[::]:50051")
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
def insecure_core_server_that_blocks_auth(self, server_credentials):
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=10),
interceptors=(DisallowAuthInterceptor(), ),
)
Core.add_CoreServiceServicer_to_server(CoreServicer(), server)
port = find_free_port()
server.add_insecure_port(f"[::]:{port}")
server.start()
yield port
server.stop(0)
def test_ingest_csv(self, mocked_client, mocker, tmp_path):
"""
Test ingestion with local FileSource, using CSV file.
Partition column is provided.
"""
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeatureTable",
return_value=_ingest_test_getfeaturetable_mocked_resp(
f"file://{tmp_path}", "date"),
)
partitioned_df = pd.read_csv(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"./data/dev_featuretable.csv",
),
parse_dates=["datetime"],
)
mocked_client.set_project("my_project")
ft = mocked_client.get_feature_table("ingest_featuretable")
mocked_client.ingest(ft, partitioned_df, timeout=600)
pq_df = pq.read_table(tmp_path).to_pandas().drop(columns=["date"])
partitioned_df, pq_df = _ingest_test_format_dataframes(
partitioned_df, pq_df, True)
assert_frame_equal(partitioned_df, pq_df)
def test_ingest_dataframe_no_partition(self, mocked_client, mocker,
non_partitioned_df, tmp_path):
"""
Test ingestion with local FileSource, using DataFrame.
Partition column not stated.
"""
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeatureTable",
return_value=_ingest_test_getfeaturetable_mocked_resp(
f"file://{tmp_path}"),
)
mocked_client.set_project("my_project")
ft = mocked_client.get_feature_table("ingest_featuretable")
mocked_client.ingest(ft, non_partitioned_df, timeout=600)
# Since not partitioning, we're only looking for single file
single_file = [
f for f in os.listdir(tmp_path)
if os.path.isfile(os.path.join(tmp_path, f))
][0]
pq_df = pq.read_table(tmp_path / single_file).to_pandas()
non_partitioned_df, pq_df = _ingest_test_format_dataframes(
non_partitioned_df, pq_df)
assert_frame_equal(non_partitioned_df, pq_df)
def test_ingest_dataframe_partition(self, mocked_client, mocker,
partitioned_df, tmp_path):
"""
Test ingestion with local FileSource, using DataFrame.
Partition column stated but not provided in Dataset.
"""
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeatureTable",
return_value=_ingest_test_getfeaturetable_mocked_resp(
f"file://{tmp_path}", "date"),
)
mocked_client.set_project("my_project")
ft = mocked_client.get_feature_table("ingest_featuretable")
mocked_client.ingest(ft, partitioned_df, timeout=600)
pq_df = pq.read_table(tmp_path).to_pandas().drop(columns=["date"])
partitioned_df, pq_df = _ingest_test_format_dataframes(
partitioned_df, pq_df, True)
assert_frame_equal(partitioned_df, pq_df)
def test_list_features(self, test_client, mocker):
mocker.patch.object(
test_client,
"_core_service_stub",
return_value=Core.CoreServiceStub(grpc.insecure_channel("")),
)
feature1_proto = FeatureSpecProto(
name="feature_1", value_type=ValueProto.ValueType.FLOAT)
feature2_proto = FeatureSpecProto(
name="feature_2", value_type=ValueProto.ValueType.STRING)
mocker.patch.object(
test_client._core_service_stub,
"ListFeatures",
return_value=ListFeaturesResponse(
features={
"driver_car:feature_1": feature1_proto,
"driver_car:feature_2": feature2_proto,
}),
)
features = test_client.list_features_by_ref(project="test")
assert len(features) == 2
native_feature_list = []
for _, feature_proto in features.items():
native_feature_list.append(feature_proto)
assert sorted(native_feature_list) == sorted([
Feature.from_proto(feature1_proto),
Feature.from_proto(feature2_proto)
])
def test_version(self, mocked_client, mocker):
mocked_client._core_service_stub = Core.CoreServiceStub(
grpc.insecure_channel(""))
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
mocker.patch.object(
mocked_client._core_service_stub,
"GetFeastCoreVersion",
return_value=GetFeastCoreVersionResponse(version="0.3.2"),
)
mocker.patch.object(
mocked_client._serving_service_stub,
"GetFeastServingInfo",
return_value=GetFeastServingInfoResponse(version="0.3.2"),
)
status = mocked_client.version()
assert (status["core"]["url"] == CORE_URL
and status["core"]["version"] == "0.3.2"
and status["serving"]["url"] == SERVING_URL
and status["serving"]["version"] == "0.3.2")