def list_jobs(
self,
include_terminated: bool,
project: Optional[str] = None,
table_name: Optional[str] = None,
) -> List[SparkJob]:
"""
List ingestion jobs currently running in Feast.
Args:
include_terminated: Flag to include terminated jobs or not
project: Optionally specify the project to use as filter when retrieving jobs
table_name: Optionally specify name of feature table to use as filter when retrieving jobs
Returns:
List of SparkJob ingestion jobs.
"""
if not self._use_job_service:
return list_jobs(include_terminated, self, project, table_name)
else:
request = ListJobsRequest(
include_terminated=include_terminated,
project=project,
table_name=cast(str, table_name),
)
response = self._job_service.ListJobs(request)
return [
get_remote_job_from_proto(self._job_service,
self._feast._extra_grpc_params, job)
for job in response.jobs
]
def ListJobs(self, request, context):
"""List all types of jobs"""
jobs = list_jobs(
include_terminated=request.include_terminated,
table_name=request.table_name,
client=self.client,
)
return ListJobsResponse(jobs=[_job_to_proto(job) for job in jobs])
def StartStreamToOnlineIngestionJob(
self, request: StartStreamToOnlineIngestionJobRequest, context):
"""Start job to ingest data from stream into online store"""
job_submission_count.labels("streaming", request.project,
request.table_name).inc()
if not self.is_whitelisted(request.project):
raise ValueError(
f"Project {request.project} is not whitelisted. Please contact your Feast administrator to whitelist it."
)
feature_table = self.client.feature_store.get_feature_table(
request.table_name, request.project)
if self.client.config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP):
# If the control loop is enabled, return existing stream ingestion job id instead of starting a new one
params = get_stream_to_online_ingestion_params(
self.client, request.project, feature_table, [])
job_hash = params.get_job_hash()
for job in list_jobs(include_terminated=True, client=self.client):
if isinstance(
job,
StreamIngestionJob) and job.get_hash() == job_hash:
job_start_timestamp = Timestamp()
job_start_timestamp.FromDatetime(job.get_start_time())
return StartStreamToOnlineIngestionJobResponse(
id=job.get_id(),
job_start_time=job_start_timestamp,
table_name=job.get_feature_table(),
log_uri=job.get_log_uri(), # type: ignore
)
raise RuntimeError(
"Feast Job Service has control loop enabled, "
"but couldn't find the existing stream ingestion job for the given FeatureTable"
)
# TODO: add extra_jars to request
job = start_stream_to_online_ingestion(
client=self.client,
project=request.project,
feature_table=feature_table,
extra_jars=[],
)
job_start_timestamp = Timestamp()
job_start_timestamp.FromDatetime(job.get_start_time())
return StartStreamToOnlineIngestionJobResponse(
id=job.get_id(),
job_start_time=job_start_timestamp,
table_name=request.table_name,
log_uri=job.get_log_uri(), # type: ignore
)
def list_jobs(self,
include_terminated: bool,
table_name: Optional[str] = None) -> List[SparkJob]:
if not self._use_job_service:
return list_jobs(include_terminated, self, table_name)
else:
request = ListJobsRequest(include_terminated=include_terminated,
table_name=table_name)
response = self._job_service.ListJobs(request)
return [
get_remote_job_from_proto(self._job_service,
self._feast._extra_grpc_params, job)
for job in response.jobs
]
def ListJobs(self, request, context):
"""List all types of jobs"""
if not self.is_whitelisted(request.project):
raise ValueError(
f"Project {request.project} is not whitelisted. Please contact your Feast administrator to whitelist it."
)
jobs = list_jobs(
include_terminated=request.include_terminated,
project=request.project,
table_name=request.table_name,
client=self.client,
)
return ListJobsResponse(jobs=[_job_to_proto(job) for job in jobs])
