def test_telemetry_on_v09(mocker): # Setup environment old_environ = dict(os.environ) os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" test_telemetry_id = str(uuid.uuid4()) os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id test_client = Client(serving_url=None, core_url=None, telemetry=True) test_client.set_project("project1") entity = Entity( name="driver_car_id", description="Car driver id", value_type=ValueType.STRING, labels={"team": "matchmaking"}, ) mocker.patch.object( test_client, "_apply_entity", return_value=None, ) test_client.apply(entity) os.environ.clear() os.environ.update(old_environ) ensure_bigquery_telemetry_id_with_retry(test_telemetry_id)
def test_telemetry_off_v09(mocker): old_environ = dict(os.environ) os.environ["FEAST_IS_TELEMETRY_TEST"] = "True" test_telemetry_id = str(uuid.uuid4()) os.environ["FEAST_FORCE_TELEMETRY_UUID"] = test_telemetry_id os.environ["FEAST_TELEMETRY"] = "False" test_client = Client(serving_url=None, core_url=None, telemetry=False) test_client.set_project("project1") entity = Entity( name="driver_car_id", description="Car driver id", value_type=ValueType.STRING, labels={"team": "matchmaking"}, ) mocker.patch.object( test_client, "_apply_entity", return_value=None, ) test_client.apply(entity) os.environ.clear() os.environ.update(old_environ) sleep(30) rows = read_bigquery_telemetry_id(test_telemetry_id) assert rows.total_rows == 0
def feast_client( pytestconfig, ingestion_job_jar, redis_server: RedisExecutor, feast_core: Tuple[str, int], feast_serving: Tuple[str, int], local_staging_path, feast_jobservice: Optional[Tuple[str, int]], enable_auth, ): if feast_jobservice is None: job_service_env = dict() else: job_service_env = dict( job_service_url=f"{feast_jobservice[0]}:{feast_jobservice[1]}") if pytestconfig.getoption("env") == "local": import pyspark return Client( core_url=f"{feast_core[0]}:{feast_core[1]}", serving_url=f"{feast_serving[0]}:{feast_serving[1]}", spark_launcher="standalone", spark_standalone_master="local", spark_home=os.getenv("SPARK_HOME") or os.path.dirname(pyspark.__file__), spark_ingestion_jar=ingestion_job_jar, redis_host=redis_server.host, redis_port=redis_server.port, spark_staging_location=os.path.join(local_staging_path, "spark"), historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), ingestion_drop_invalid_rows=True, **job_service_env, ) elif pytestconfig.getoption("env") == "gcloud": c = Client( core_url=f"{feast_core[0]}:{feast_core[1]}", serving_url=f"{feast_serving[0]}:{feast_serving[1]}", spark_launcher="dataproc", dataproc_cluster_name=pytestconfig.getoption( "dataproc_cluster_name"), dataproc_project=pytestconfig.getoption("dataproc_project"), dataproc_region=pytestconfig.getoption("dataproc_region"), spark_staging_location=os.path.join(local_staging_path, "dataproc"), spark_ingestion_jar=ingestion_job_jar, redis_host=pytestconfig.getoption("redis_url").split(":")[0], redis_port=pytestconfig.getoption("redis_url").split(":")[1], historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), ingestion_drop_invalid_rows=True, grpc_connection_timeout=30, **job_service_env, ) elif pytestconfig.getoption("env") == "aws": return Client( core_url=f"{feast_core[0]}:{feast_core[1]}", serving_url=f"{feast_serving[0]}:{feast_serving[1]}", spark_launcher="emr", emr_cluster_id=pytestconfig.getoption("emr_cluster_id"), emr_region=pytestconfig.getoption("emr_region"), spark_staging_location=os.path.join(local_staging_path, "emr"), emr_log_location=os.path.join(local_staging_path, "emr_logs"), spark_ingestion_jar=ingestion_job_jar, redis_host=pytestconfig.getoption("redis_url").split(":")[0], redis_port=pytestconfig.getoption("redis_url").split(":")[1], historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), ingestion_drop_invalid_rows=True, ) elif pytestconfig.getoption("env") == "k8s": return Client( core_url=f"{feast_core[0]}:{feast_core[1]}", serving_url=f"{feast_serving[0]}:{feast_serving[1]}", spark_launcher="k8s", spark_staging_location=os.path.join(local_staging_path, "k8s"), spark_ingestion_jar=ingestion_job_jar, redis_host=pytestconfig.getoption("redis_url").split(":")[0], redis_port=pytestconfig.getoption("redis_url").split(":")[1], historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), ) else: raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}") c.set_project(pytestconfig.getoption("feast_project")) return c
def tfrecord_feast_client( pytestconfig, feast_core: Tuple[str, int], local_staging_path, feast_jobservice: Optional[Tuple[str, int]], enable_auth, ): if feast_jobservice is None: job_service_env = dict() else: job_service_env = dict( job_service_url=f"{feast_jobservice[0]}:{feast_jobservice[1]}") if pytestconfig.getoption("env") == "local": import pyspark return Client( core_url=f"{feast_core[0]}:{feast_core[1]}", spark_launcher="standalone", spark_standalone_master="local", spark_home=os.getenv("SPARK_HOME") or os.path.dirname(pyspark.__file__), spark_staging_location=os.path.join(local_staging_path, "spark"), historical_feature_output_format="tfrecord", historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), **job_service_env, ) elif pytestconfig.getoption("env") == "gcloud": c = Client( core_url=f"{feast_core[0]}:{feast_core[1]}", spark_launcher="dataproc", dataproc_cluster_name=pytestconfig.getoption( "dataproc_cluster_name"), dataproc_project=pytestconfig.getoption("dataproc_project"), dataproc_region=pytestconfig.getoption("dataproc_region"), spark_staging_location=os.path.join(local_staging_path, "dataproc"), historical_feature_output_format="tfrecord", historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), ingestion_drop_invalid_rows=True, **job_service_env, ) elif pytestconfig.getoption("env") == "aws": return Client( core_url=f"{feast_core[0]}:{feast_core[1]}", spark_launcher="emr", emr_cluster_id=pytestconfig.getoption("emr_cluster_id"), emr_region=pytestconfig.getoption("emr_region"), spark_staging_location=os.path.join(local_staging_path, "emr"), emr_log_location=os.path.join(local_staging_path, "emr_logs"), historical_feature_output_format="tfrecord", historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), ) elif pytestconfig.getoption("env") == "k8s": return Client( core_url=f"{feast_core[0]}:{feast_core[1]}", spark_launcher="k8s", spark_staging_location=os.path.join(local_staging_path, "k8s"), historical_feature_output_format="tfrecord", historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), **job_service_env, ) elif pytestconfig.getoption("env") == "synapse": return Client( core_url=f"{feast_core[0]}:{feast_core[1]}", spark_launcher="synapse", azure_synapse_dev_url=pytestconfig.getoption( "azure_synapse_dev_url"), azure_synapse_pool_name=pytestconfig.getoption( "azure_synapse_pool_name"), azure_synapse_datalake_dir=pytestconfig.getoption( "azure_synapse_datalake_dir"), spark_staging_location=os.path.join(local_staging_path, "synapse"), azure_blob_account_name=pytestconfig.getoption( "azure_blob_account_name"), azure_blob_account_access_key=pytestconfig.getoption( "azure_blob_account_access_key"), historical_feature_output_format="tfrecord", historical_feature_output_location=os.path.join( local_staging_path, "historical_output"), ) else: raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}") c.set_project(pytestconfig.getoption("feast_project")) return c