def StartStreamToOnlineIngestionJob(
self, request: StartStreamToOnlineIngestionJobRequest, context):
"""Start job to ingest data from stream into online store"""
feature_table = self.client.get_feature_table(request.table_name,
request.project)
if self.client._config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP):
# If the control loop is enabled, return existing stream ingestion job id instead of starting a new one
params = get_stream_to_online_ingestion_params(
self.client, request.project, feature_table, [])
job_hash = params.get_job_hash()
for job in list_jobs(include_terminated=True, client=self.client):
if isinstance(
job,
StreamIngestionJob) and job.get_hash() == job_hash:
return StartStreamToOnlineIngestionJobResponse(
id=job.get_id())
raise RuntimeError(
"Feast Job Service has control loop enabled, but couldn't find the existing stream ingestion job for the given FeatureTable"
)
# TODO: add extra_jars to request
job = start_stream_to_online_ingestion(
client=self.client,
project=request.project,
feature_table=feature_table,
extra_jars=[],
)
return StartStreamToOnlineIngestionJobResponse(id=job.get_id())
def ListJobs(self, request, context):
"""List all types of jobs"""
jobs = list_jobs(
include_terminated=request.include_terminated,
table_name=request.table_name,
client=self.client,
)
return ListJobsResponse(jobs=[_job_to_proto(job) for job in jobs])
def list_jobs(self, include_terminated: bool) -> List[SparkJob]:
if not self._use_job_service:
return list_jobs(include_terminated, self)
else:
request = ListJobsRequest(include_terminated=include_terminated)
response = self._job_service.ListJobs(request)
return [
get_remote_job_from_proto(self._job_service,
self._extra_grpc_params, job)
for job in response.jobs
]
def list_jobs(self, include_terminated: bool) -> List[SparkJob]:
return list_jobs(include_terminated, self)