def _get_expected_job_hash_to_tables( client: Client, projects: List[str]) -> Dict[str, Tuple[str, FeatureTable]]: """ Checks all feature tables for the requires project(s) and determines all required stream ingestion jobs from them. Outputs a map of the expected job_hash to a tuple of (project, table_name). Args: all_projects (bool): If true, runs the check for all project. Otherwise only checks the current project. Returns: Dict[str, Tuple[str, str]]: Map of job_hash -> (project, table_name) for expected stream ingestion jobs """ job_hash_to_table_refs = {} for project in projects: feature_tables = client.feature_store.list_feature_tables(project) for feature_table in feature_tables: if feature_table.stream_source is not None: params = get_stream_to_online_ingestion_params( client, project, feature_table, []) job_hash = params.get_job_hash() job_hash_to_table_refs[job_hash] = (project, feature_table) return job_hash_to_table_refs
def StartStreamToOnlineIngestionJob( self, request: StartStreamToOnlineIngestionJobRequest, context): """Start job to ingest data from stream into online store""" job_submission_count.labels("streaming", request.project, request.table_name).inc() if not self.is_whitelisted(request.project): raise ValueError( f"Project {request.project} is not whitelisted. Please contact your Feast administrator to whitelist it." ) feature_table = self.client.feature_store.get_feature_table( request.table_name, request.project) if self.client.config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP): # If the control loop is enabled, return existing stream ingestion job id instead of starting a new one params = get_stream_to_online_ingestion_params( self.client, request.project, feature_table, []) job_hash = params.get_job_hash() for job in list_jobs(include_terminated=True, client=self.client): if isinstance( job, StreamIngestionJob) and job.get_hash() == job_hash: job_start_timestamp = Timestamp() job_start_timestamp.FromDatetime(job.get_start_time()) return StartStreamToOnlineIngestionJobResponse( id=job.get_id(), job_start_time=job_start_timestamp, table_name=job.get_feature_table(), log_uri=job.get_log_uri(), # type: ignore ) raise RuntimeError( "Feast Job Service has control loop enabled, " "but couldn't find the existing stream ingestion job for the given FeatureTable" ) # TODO: add extra_jars to request job = start_stream_to_online_ingestion( client=self.client, project=request.project, feature_table=feature_table, extra_jars=[], ) job_start_timestamp = Timestamp() job_start_timestamp.FromDatetime(job.get_start_time()) return StartStreamToOnlineIngestionJobResponse( id=job.get_id(), job_start_time=job_start_timestamp, table_name=request.table_name, log_uri=job.get_log_uri(), # type: ignore )