def get_stream_to_online_ingestion_params(
client: "Client", project: str, feature_table: FeatureTable,
extra_jars: List[str]) -> StreamIngestionJobParameters:
return StreamIngestionJobParameters(
jar=client.config.get(opt.SPARK_INGESTION_JAR),
extra_jars=extra_jars,
source=_source_to_argument(feature_table.stream_source, client.config),
feature_table=_feature_table_to_argument(client, project,
feature_table),
redis_host=client.config.get(opt.REDIS_HOST),
redis_port=bool(client.config.get(opt.REDIS_HOST))
and client.config.getint(opt.REDIS_PORT),
redis_ssl=client.config.getboolean(opt.REDIS_SSL),
bigtable_project=client.config.get(opt.BIGTABLE_PROJECT),
bigtable_instance=client.config.get(opt.BIGTABLE_INSTANCE),
statsd_host=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.get(opt.STATSD_HOST),
statsd_port=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.getint(opt.STATSD_PORT),
deadletter_path=client.config.get(opt.DEADLETTER_PATH),
checkpoint_path=client.config.get(opt.CHECKPOINT_PATH),
stencil_url=client.config.get(opt.STENCIL_URL),
drop_invalid_rows=client.config.get(opt.INGESTION_DROP_INVALID_ROWS),
triggering_interval=client.config.getint(
opt.SPARK_STREAMING_TRIGGERING_INTERVAL, default=None),
)
def get_stream_to_online_ingestion_params(
client: "Client", project: str, feature_table: FeatureTable, extra_jars: List[str]
) -> StreamIngestionJobParameters:
return StreamIngestionJobParameters(
jar=client.config.get(opt.SPARK_INGESTION_JAR),
extra_jars=extra_jars,
source=_source_to_argument(feature_table.stream_source, client.config),
feature_table=_feature_table_to_argument(client, project, feature_table),
redis_host=client.config.get(opt.REDIS_HOST),
redis_port=client.config.getint(opt.REDIS_PORT),
redis_ssl=client.config.getboolean(opt.REDIS_SSL),
statsd_host=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.get(opt.STATSD_HOST),
statsd_port=client.config.getboolean(opt.STATSD_ENABLED)
and client.config.getint(opt.STATSD_PORT),
deadletter_path=client.config.get(opt.DEADLETTER_PATH),
stencil_url=client.config.get(opt.STENCIL_URL),
drop_invalid_rows=client.config.get(opt.INGESTION_DROP_INVALID_ROWS),
)