def test_telemetry_on_v09(mocker):
# Setup environment
old_environ = dict(os.environ)
os.environ["FEAST_IS_TELEMETRY_TEST"] = "True"
test_telemetry_id = str(uuid.uuid4())
os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id
test_client = Client(serving_url=None, core_url=None, telemetry=True)
test_client.set_project("project1")
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
mocker.patch.object(
test_client, "_apply_entity", return_value=None,
)
test_client.apply(entity)
os.environ.clear()
os.environ.update(old_environ)
ensure_bigquery_telemetry_id_with_retry(test_telemetry_id)
def test_telemetry_off_v09(mocker):
old_environ = dict(os.environ)
os.environ["FEAST_IS_TELEMETRY_TEST"] = "True"
test_telemetry_id = str(uuid.uuid4())
os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id
os.environ["FEAST_TELEMETRY"] = "False"
test_client = Client(serving_url=None, core_url=None, telemetry=False)
test_client.set_project("project1")
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
mocker.patch.object(
test_client, "_apply_entity", return_value=None,
)
test_client.apply(entity)
os.environ.clear()
os.environ.update(old_environ)
sleep(30)
rows = read_bigquery_telemetry_id(test_telemetry_id)
assert rows.total_rows == 0
def feast_client(
pytestconfig,
ingestion_job_jar,
redis_server: RedisExecutor,
feast_core: Tuple[str, int],
feast_serving: Tuple[str, int],
local_staging_path,
feast_jobservice: Optional[Tuple[str, int]],
enable_auth,
):
if feast_jobservice is None:
job_service_env = dict()
else:
job_service_env = dict(
job_service_url=f"{feast_jobservice[0]}:{feast_jobservice[1]}")
if pytestconfig.getoption("env") == "local":
import pyspark
return Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
spark_launcher="standalone",
spark_standalone_master="local",
spark_home=os.getenv("SPARK_HOME")
or os.path.dirname(pyspark.__file__),
spark_ingestion_jar=ingestion_job_jar,
redis_host=redis_server.host,
redis_port=redis_server.port,
spark_staging_location=os.path.join(local_staging_path, "spark"),
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
ingestion_drop_invalid_rows=True,
**job_service_env,
)
elif pytestconfig.getoption("env") == "gcloud":
c = Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
spark_launcher="dataproc",
dataproc_cluster_name=pytestconfig.getoption(
"dataproc_cluster_name"),
dataproc_project=pytestconfig.getoption("dataproc_project"),
dataproc_region=pytestconfig.getoption("dataproc_region"),
spark_staging_location=os.path.join(local_staging_path,
"dataproc"),
spark_ingestion_jar=ingestion_job_jar,
redis_host=pytestconfig.getoption("redis_url").split(":")[0],
redis_port=pytestconfig.getoption("redis_url").split(":")[1],
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
ingestion_drop_invalid_rows=True,
grpc_connection_timeout=30,
**job_service_env,
)
elif pytestconfig.getoption("env") == "aws":
return Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
spark_launcher="emr",
emr_cluster_id=pytestconfig.getoption("emr_cluster_id"),
emr_region=pytestconfig.getoption("emr_region"),
spark_staging_location=os.path.join(local_staging_path, "emr"),
emr_log_location=os.path.join(local_staging_path, "emr_logs"),
spark_ingestion_jar=ingestion_job_jar,
redis_host=pytestconfig.getoption("redis_url").split(":")[0],
redis_port=pytestconfig.getoption("redis_url").split(":")[1],
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
ingestion_drop_invalid_rows=True,
)
elif pytestconfig.getoption("env") == "k8s":
return Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
spark_launcher="k8s",
spark_staging_location=os.path.join(local_staging_path, "k8s"),
spark_ingestion_jar=ingestion_job_jar,
redis_host=pytestconfig.getoption("redis_url").split(":")[0],
redis_port=pytestconfig.getoption("redis_url").split(":")[1],
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
)
else:
raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}")
c.set_project(pytestconfig.getoption("feast_project"))
return c
def tfrecord_feast_client(
pytestconfig,
feast_core: Tuple[str, int],
local_staging_path,
feast_jobservice: Optional[Tuple[str, int]],
enable_auth,
):
if feast_jobservice is None:
job_service_env = dict()
else:
job_service_env = dict(
job_service_url=f"{feast_jobservice[0]}:{feast_jobservice[1]}")
if pytestconfig.getoption("env") == "local":
import pyspark
return Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
spark_launcher="standalone",
spark_standalone_master="local",
spark_home=os.getenv("SPARK_HOME")
or os.path.dirname(pyspark.__file__),
spark_staging_location=os.path.join(local_staging_path, "spark"),
historical_feature_output_format="tfrecord",
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
**job_service_env,
)
elif pytestconfig.getoption("env") == "gcloud":
c = Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
spark_launcher="dataproc",
dataproc_cluster_name=pytestconfig.getoption(
"dataproc_cluster_name"),
dataproc_project=pytestconfig.getoption("dataproc_project"),
dataproc_region=pytestconfig.getoption("dataproc_region"),
spark_staging_location=os.path.join(local_staging_path,
"dataproc"),
historical_feature_output_format="tfrecord",
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
ingestion_drop_invalid_rows=True,
**job_service_env,
)
elif pytestconfig.getoption("env") == "aws":
return Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
spark_launcher="emr",
emr_cluster_id=pytestconfig.getoption("emr_cluster_id"),
emr_region=pytestconfig.getoption("emr_region"),
spark_staging_location=os.path.join(local_staging_path, "emr"),
emr_log_location=os.path.join(local_staging_path, "emr_logs"),
historical_feature_output_format="tfrecord",
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
)
elif pytestconfig.getoption("env") == "k8s":
return Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
spark_launcher="k8s",
spark_staging_location=os.path.join(local_staging_path, "k8s"),
historical_feature_output_format="tfrecord",
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
**job_service_env,
)
elif pytestconfig.getoption("env") == "synapse":
return Client(
core_url=f"{feast_core[0]}:{feast_core[1]}",
spark_launcher="synapse",
azure_synapse_dev_url=pytestconfig.getoption(
"azure_synapse_dev_url"),
azure_synapse_pool_name=pytestconfig.getoption(
"azure_synapse_pool_name"),
azure_synapse_datalake_dir=pytestconfig.getoption(
"azure_synapse_datalake_dir"),
spark_staging_location=os.path.join(local_staging_path, "synapse"),
azure_blob_account_name=pytestconfig.getoption(
"azure_blob_account_name"),
azure_blob_account_access_key=pytestconfig.getoption(
"azure_blob_account_access_key"),
historical_feature_output_format="tfrecord",
historical_feature_output_location=os.path.join(
local_staging_path, "historical_output"),
)
else:
raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}")
c.set_project(pytestconfig.getoption("feast_project"))
return c
