def start_stream_to_online_ingestion(
self, ingestion_job_params: StreamIngestionJobParameters
) -> StreamIngestionJob:
"""
Starts a stream ingestion job to a Spark cluster.
Raises:
SparkJobFailure: The spark job submission failed, encountered error
during execution, or timeout.
Returns:
StreamIngestionJob: wrapper around remote job.
"""
jar_s3_path = self._upload_jar(
ingestion_job_params.get_main_file_path())
extra_jar_paths: List[str] = []
for extra_jar in ingestion_job_params.get_extra_jar_paths():
extra_jar_paths.append(self._upload_jar(extra_jar))
job_hash = ingestion_job_params.get_job_hash()
job_id = _generate_job_id()
resource = _prepare_job_resource(
job_template=self._stream_ingestion_template,
job_id=job_id,
job_type=STREAM_TO_ONLINE_JOB_TYPE,
main_application_file=jar_s3_path,
main_class=ingestion_job_params.get_class_name(),
packages=[],
jars=extra_jar_paths,
extra_metadata={METADATA_JOBHASH: job_hash},
azure_credentials=self._get_azure_credentials(),
arguments=ingestion_job_params.get_arguments(),
namespace=self._namespace,
extra_labels={
LABEL_FEATURE_TABLE:
_truncate_label(ingestion_job_params.get_feature_table_name()),
LABEL_FEATURE_TABLE_HASH:
_generate_project_table_hash(
ingestion_job_params.get_project(),
ingestion_job_params.get_feature_table_name(),
),
LABEL_PROJECT:
ingestion_job_params.get_project(),
},
)
job_info = _submit_job(
api=self._api,
resource=resource,
namespace=self._namespace,
)
return cast(StreamIngestionJob, self._job_from_job_info(job_info))
def start_stream_to_online_ingestion(
self, ingestion_job_params: StreamIngestionJobParameters
) -> StreamIngestionJob:
job_id = str(uuid.uuid4())
ui_port = _find_free_port()
job = StandaloneClusterStreamingIngestionJob(
job_id,
ingestion_job_params.get_name(),
self.spark_submit(ingestion_job_params, ui_port),
ui_port,
ingestion_job_params.get_job_hash(),
ingestion_job_params.get_feature_table_name(),
)
global_job_cache.add_job(job)
return job
def get_stream_to_online_ingestion_params(
client: "Client", project: str, feature_table: FeatureTable,
extra_jars: List[str]) -> StreamIngestionJobParameters:
return StreamIngestionJobParameters(
jar=client.config.get(opt.SPARK_INGESTION_JAR),
extra_jars=extra_jars,
source=_source_to_argument(feature_table.stream_source, client.config),
feature_table=_feature_table_to_argument(client, project,
feature_table),
redis_host=client.config.get(opt.REDIS_HOST),
redis_port=bool(client.config.get(opt.REDIS_HOST))
and client.config.getint(opt.REDIS_PORT),
redis_ssl=client.config.getboolean(opt.REDIS_SSL),
bigtable_project=client.config.get(opt.BIGTABLE_PROJECT),
bigtable_instance=client.config.get(opt.BIGTABLE_INSTANCE),
statsd_host=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.get(opt.STATSD_HOST),
statsd_port=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.getint(opt.STATSD_PORT),
deadletter_path=client.config.get(opt.DEADLETTER_PATH),
checkpoint_path=client.config.get(opt.CHECKPOINT_PATH),
stencil_url=client.config.get(opt.STENCIL_URL),
drop_invalid_rows=client.config.get(opt.INGESTION_DROP_INVALID_ROWS),
triggering_interval=client.config.getint(
opt.SPARK_STREAMING_TRIGGERING_INTERVAL, default=None),
)
def start_stream_to_online_ingestion(
self, ingestion_job_params: StreamIngestionJobParameters
) -> StreamIngestionJob:
"""
Starts a stream ingestion job to a Spark cluster.
Raises:
SparkJobFailure: The spark job submission failed, encountered error
during execution, or timeout.
Returns:
StreamIngestionJob: wrapper around remote job.
"""
main_file = self._datalake.upload_file(
ingestion_job_params.get_main_file_path())
extra_jar_paths: List[str] = []
for extra_jar in ingestion_job_params.get_extra_jar_paths():
extra_jar_paths.append(self._datalake.upload_file(extra_jar))
tags = _prepare_job_tags(ingestion_job_params,
STREAM_TO_ONLINE_JOB_TYPE)
tags[METADATA_JOBHASH] = ingestion_job_params.get_job_hash()
job_info = _submit_job(
self._api,
ingestion_job_params.get_project() + "_stream_to_online_ingestion",
main_file,
main_class=ingestion_job_params.get_class_name(),
arguments=ingestion_job_params.get_arguments(),
reference_files=extra_jar_paths,
configuration=None,
tags=tags)
return cast(StreamIngestionJob, self._job_from_job_info(job_info))
def start_stream_to_online_ingestion(
self, ingestion_job_params: StreamIngestionJobParameters
) -> StreamIngestionJob:
job, refresh_fn, cancel_fn = self.dataproc_submit(ingestion_job_params, {})
job_hash = ingestion_job_params.get_job_hash()
return DataprocStreamingIngestionJob(
job=job,
refresh_fn=refresh_fn,
cancel_fn=cancel_fn,
project=self.project_id,
region=self.region,
job_hash=job_hash,
)
def get_stream_to_online_ingestion_params(
client: "Client", project: str, feature_table: FeatureTable, extra_jars: List[str]
) -> StreamIngestionJobParameters:
return StreamIngestionJobParameters(
jar=client.config.get(opt.SPARK_INGESTION_JAR),
extra_jars=extra_jars,
source=_source_to_argument(feature_table.stream_source, client.config),
feature_table=_feature_table_to_argument(client, project, feature_table),
redis_host=client.config.get(opt.REDIS_HOST),
redis_port=client.config.getint(opt.REDIS_PORT),
redis_ssl=client.config.getboolean(opt.REDIS_SSL),
statsd_host=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.get(opt.STATSD_HOST),
statsd_port=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.getint(opt.STATSD_PORT),
deadletter_path=client.config.get(opt.DEADLETTER_PATH),
stencil_url=client.config.get(opt.STENCIL_URL),
drop_invalid_rows=client.config.get(opt.INGESTION_DROP_INVALID_ROWS),
)
def start_stream_to_online_ingestion(
self, ingestion_job_params: StreamIngestionJobParameters
) -> StreamIngestionJob:
"""
Starts a stream ingestion job on a Spark cluster.
Returns:
StreamIngestionJob: wrapper around remote job that can be used to check on the job.
"""
jar_s3_path = _upload_jar(self._staging_location,
ingestion_job_params.get_main_file_path())
extra_jar_paths: List[str] = []
for extra_jar in ingestion_job_params.get_extra_jar_paths():
if extra_jar.startswith("s3://"):
extra_jar_paths.append(extra_jar)
else:
extra_jar_paths.append(
_upload_jar(self._staging_location, extra_jar))
job_hash = ingestion_job_params.get_job_hash()
step = _stream_ingestion_step(
jar_path=jar_s3_path,
extra_jar_paths=extra_jar_paths,
project=ingestion_job_params.get_project(),
feature_table_name=ingestion_job_params.get_feature_table_name(),
args=ingestion_job_params.get_arguments(),
job_hash=job_hash,
)
job_ref = self._submit_emr_job(step)
return EmrStreamIngestionJob(
self._emr_client(),
job_ref,
job_hash,
ingestion_job_params.get_project(),
ingestion_job_params.get_feature_table_name(),
)
