def test_restarting_failed_jobs(feature_table):
""" If configured - restart failed jobs """
feast_client = FeastClient(
job_service_pause_between_jobs=0,
job_service_retry_failed_jobs=True,
options={"whitelisted_projects": "default,ride"},
)
feast_client.list_projects = Mock(return_value=["default"])
feast_client.list_feature_tables = Mock()
spark_client = Client(feast_client)
spark_client.list_jobs = Mock()
spark_client.start_stream_to_online_ingestion = Mock()
spark_client.feature_store.list_feature_tables.return_value = [
feature_table
]
spark_client.list_jobs.return_value = []
ensure_stream_ingestion_jobs(spark_client, all_projects=True)
spark_client.list_jobs.assert_called_once_with(include_terminated=False)
spark_client.start_stream_to_online_ingestion.assert_called_once_with(
feature_table, [], project="default")
def start_job_service() -> None:
"""
Start Feast Job Service
"""
log_fmt = "%(asctime)s %(levelname)s %(message)s"
logging.basicConfig(level=logging.INFO, format=log_fmt)
feast_client = FeastClient()
client = Client(feast_client)
if client.config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP):
# Start the control loop thread only if it's enabled from configs
thread = threading.Thread(target=start_control_loop, daemon=True)
thread.start()
server = grpc.server(ThreadPoolExecutor(),
interceptors=(LoggingInterceptor(), ))
JobService_pb2_grpc.add_JobServiceServicer_to_server(
JobServiceServicer(client), server)
LegacyJobService_pb2_grpc.add_JobServiceServicer_to_server(
JobServiceServicer(client), server)
add_HealthServicer_to_server(HealthServicerImpl(), server)
server.add_insecure_port("[::]:6568")
server.start()
logging.info("Feast Job Service is listening on port :6568")
server.wait_for_termination()
def start_job_service() -> None:
"""
Start Feast Job Service
"""
feast_client = FeastClient()
client = Client(feast_client)
if client.config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP):
# Start the control loop thread only if it's enabled from configs
thread = threading.Thread(target=start_control_loop, daemon=True)
thread.start()
metricServerThread = threading.Thread(
target=start_prometheus_serving,
daemon=True,
args=[client.config.getint(opt.JOB_SERVICE_PROMETHEUS_METRIC_PORT)],
)
metricServerThread.start()
server = grpc.server(ThreadPoolExecutor(),
interceptors=(LoggingInterceptor(), ))
JobService_pb2_grpc.add_JobServiceServicer_to_server(
JobServiceServicer(client), server)
LegacyJobService_pb2_grpc.add_JobServiceServicer_to_server(
JobServiceServicer(client), server)
add_HealthServicer_to_server(HealthServicerImpl(), server)
server.add_insecure_port("[::]:6568")
server.start()
logger.info("Feast Job Service is listening on port :6568")
server.wait_for_termination()
def feast_client():
c = FeastClient(
job_service_pause_between_jobs=0,
options={"whitelisted_projects": "default,ride"},
)
c.list_projects = Mock(return_value=["default", "ride", "invalid_project"])
c.list_feature_tables = Mock()
yield c
def start_control_loop() -> None:
"""Starts control loop that continuously ensures that correct jobs are being run.
Currently this affects only the stream ingestion jobs. Please refer to
ensure_stream_ingestion_jobs for full documentation on how the check works.
"""
logger.info(
"Feast Job Service is starting a control loop in a background thread, "
"which will ensure that stream ingestion jobs are successfully running."
)
try:
feature_store = FeastClient()
client = Client(feature_store)
while True:
ensure_stream_ingestion_jobs(client, all_projects=True)
time.sleep(1)
except Exception:
traceback.print_exc()
finally:
# Send interrupt signal to the main thread to kill the server if control loop fails
os.kill(os.getpid(), signal.SIGINT)
def feast_client():
c = FeastClient(job_service_pause_between_jobs=0)
c.list_projects = Mock(return_value=["default"])
c.list_feature_tables = Mock()
yield c