def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
type=DataSourceProto.BATCH_REDSHIFT,
field_mapping=self.field_mapping,
redshift_options=self.redshift_options.to_proto(),
)
data_source_proto.event_timestamp_column = self.event_timestamp_column
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column
return data_source_proto
def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
type=DataSourceProto.BATCH_BIGQUERY,
field_mapping=self.field_mapping,
bigquery_options=self.bigquery_options.to_proto(),
)
data_source_proto.event_timestamp_column = self.event_timestamp_column
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column
return data_source_proto
def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
type=DataSourceProto.STREAM_KINESIS,
field_mapping=self.field_mapping,
kinesis_options=self.kinesis_options.to_proto(),
)
data_source_proto.event_timestamp_column = self.event_timestamp_column
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column
return data_source_proto
def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
type=DataSourceProto.CUSTOM_SOURCE,
data_source_class_type=
"feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source.PostgreSQLSource",
field_mapping=self.field_mapping,
custom_options=self._postgres_options.to_proto(),
)
data_source_proto.timestamp_field = self.timestamp_field
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column
return data_source_proto
def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
name=self.name,
type=DataSourceProto.BATCH_BIGQUERY,
field_mapping=self.field_mapping,
bigquery_options=self.bigquery_options.to_proto(),
description=self.description,
tags=self.tags,
owner=self.owner,
)
data_source_proto.timestamp_field = self.timestamp_field
data_source_proto.created_timestamp_column = self.created_timestamp_column
return data_source_proto
def _ingest_test_getfeaturetable_mocked_resp(file_url: str,
date_partition_col: str = ""):
return GetFeatureTableResponse(table=FeatureTableProto(
spec=FeatureTableSpecProto(
name="ingest_featuretable",
max_age=Duration(seconds=3600),
features=[
FeatureSpecProto(
name="dev_feature_float",
value_type=ValueProto.ValueType.FLOAT,
),
FeatureSpecProto(
name="dev_feature_string",
value_type=ValueProto.ValueType.STRING,
),
],
entities=["dev_entity"],
batch_source=DataSourceProto(
file_options=DataSourceProto.FileOptions(
file_format=ParquetFormat().to_proto(), file_url=file_url),
event_timestamp_column="datetime",
created_timestamp_column="timestamp",
date_partition_column=date_partition_col,
),
),
meta=FeatureTableMetaProto(),
))
def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
name=self.name,
type=DataSourceProto.BATCH_TRINO,
field_mapping=self.field_mapping,
trino_options=self.trino_options.to_proto(),
description=self.description,
tags=self.tags,
owner=self.owner,
)
data_source_proto.timestamp_field = self.timestamp_field
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column
return data_source_proto
def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
name=self.name,
type=DataSourceProto.BATCH_SPARK,
data_source_class_type=
"feast.infra.offline_stores.contrib.spark_offline_store.spark_source.SparkSource",
field_mapping=self.field_mapping,
spark_options=self.spark_options.to_proto(),
description=self.description,
tags=self.tags,
owner=self.owner,
)
data_source_proto.timestamp_field = self.timestamp_field
data_source_proto.created_timestamp_column = self.created_timestamp_column
return data_source_proto
def to_proto(self) -> DataSourceProto.CustomSourceOptions:
postgres_options_proto = DataSourceProto.CustomSourceOptions(
configuration=json.dumps({
"name": self._name,
"query": self._query
}).encode())
return postgres_options_proto
def to_proto(self) -> DataSourceProto:
"""
Converts a RedshiftSource object to its protobuf representation.
Returns:
A DataSourceProto object.
"""
data_source_proto = DataSourceProto(
type=DataSourceProto.BATCH_REDSHIFT,
field_mapping=self.field_mapping,
redshift_options=self.redshift_options.to_proto(),
)
data_source_proto.event_timestamp_column = self.event_timestamp_column
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column
return data_source_proto
def to_proto(self) -> DataSourceProto:
data_source_proto = DataSourceProto(
name=self.name,
type=DataSourceProto.STREAM_KAFKA,
field_mapping=self.field_mapping,
kafka_options=self.kafka_options.to_proto(),
description=self.description,
tags=self.tags,
owner=self.owner,
)
data_source_proto.timestamp_field = self.timestamp_field
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column
if self.batch_source:
data_source_proto.batch_source.MergeFrom(
self.batch_source.to_proto())
return data_source_proto
def to_proto(self) -> DataSourceProto:
schema_pb = {}
for key, value in self._schema.items():
schema_pb[key] = value.value
options = DataSourceProto.RequestDataOptions(name=self._name,
schema=schema_pb)
data_source_proto = DataSourceProto(
type=DataSourceProto.REQUEST_SOURCE, request_data_options=options)
return data_source_proto
def to_proto(self) -> DataSourceProto:
"""
Converts a RedshiftSource object to its protobuf representation.
Returns:
A DataSourceProto object.
"""
data_source_proto = DataSourceProto(
type=DataSourceProto.BATCH_REDSHIFT,
field_mapping=self.field_mapping,
redshift_options=self.redshift_options.to_proto(),
description=self.description,
tags=self.tags,
owner=self.owner,
)
data_source_proto.timestamp_field = self.timestamp_field
data_source_proto.created_timestamp_column = self.created_timestamp_column
return data_source_proto
def from_proto(data_source: DataSourceProto):
assert data_source.HasField("batch_source")
batch_source = DataSource.from_proto(data_source.batch_source)
return PushSource(
name=data_source.name,
batch_source=batch_source,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
def to_proto(self) -> DataSourceProto:
"""
Converts a SnowflakeSource object to its protobuf representation.
Returns:
A DataSourceProto object.
"""
data_source_proto = DataSourceProto(
type=DataSourceProto.BATCH_SNOWFLAKE,
field_mapping=self.field_mapping,
snowflake_options=self.snowflake_options.to_proto(),
description=self.description,
tags=self.tags,
owner=self.owner,
)
data_source_proto.timestamp_field = self.timestamp_field
data_source_proto.created_timestamp_column = self.created_timestamp_column
return data_source_proto
def to_proto(self) -> DataSourceProto.BigQueryOptions:
"""
Converts an BigQueryOptionsProto object to its protobuf representation.
Returns:
BigQueryOptionsProto protobuf
"""
bigquery_options_proto = DataSourceProto.BigQueryOptions(
table_ref=self.table_ref, )
return bigquery_options_proto
def to_proto(self) -> DataSourceProto.TrinoOptions:
"""
Converts an TrinoOptionsProto object to its protobuf representation.
Returns:
TrinoOptionsProto protobuf
"""
trino_options_proto = DataSourceProto.TrinoOptions(
table=self.table, query=self.query,
)
return trino_options_proto
def from_proto(data_source: DataSourceProto):
assert data_source.HasField("custom_options")
postgres_options = json.loads(data_source.custom_options.configuration)
return PostgreSQLSource(
name=postgres_options["name"],
query=postgres_options["query"],
field_mapping=dict(data_source.field_mapping),
timestamp_field=data_source.timestamp_field,
created_timestamp_column=data_source.created_timestamp_column,
date_partition_column=data_source.date_partition_column,
)
def to_proto(self) -> DataSourceProto.RedshiftOptions:
"""
Converts an RedshiftOptionsProto object to its protobuf representation.
Returns:
A RedshiftOptionsProto protobuf.
"""
redshift_options_proto = DataSourceProto.RedshiftOptions(
table=self.table, schema=self.schema, query=self.query,
)
return redshift_options_proto
def from_proto(data_source: DataSourceProto):
assert data_source.HasField("bigquery_options")
return BigQuerySource(
field_mapping=dict(data_source.field_mapping),
table_ref=data_source.bigquery_options.table_ref,
event_timestamp_column=data_source.event_timestamp_column,
created_timestamp_column=data_source.created_timestamp_column,
date_partition_column=data_source.date_partition_column,
query=data_source.bigquery_options.query,
)
def to_proto(self) -> DataSourceProto.SparkOptions:
"""
Converts an SparkOptionsProto object to its protobuf representation.
Returns:
SparkOptionsProto protobuf
"""
spark_options_proto = DataSourceProto.SparkOptions(
table=self.table,
query=self.query,
path=self.path,
file_format=self.file_format,
)
return spark_options_proto
def from_proto(data_source: DataSourceProto):
assert data_source.HasField("bigquery_options")
return BigQuerySource(
name=data_source.name,
field_mapping=dict(data_source.field_mapping),
table=data_source.bigquery_options.table,
timestamp_field=data_source.timestamp_field,
created_timestamp_column=data_source.created_timestamp_column,
query=data_source.bigquery_options.query,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
def to_proto(self) -> DataSourceProto.KinesisOptions:
"""
Converts an KinesisOptionsProto object to its protobuf representation.
Returns:
KinesisOptionsProto protobuf
"""
kinesis_options_proto = DataSourceProto.KinesisOptions(
record_format=self.record_format.to_proto(),
region=self.region,
stream_name=self.stream_name,
)
return kinesis_options_proto
def to_proto(self) -> DataSourceProto.KafkaOptions:
"""
Converts an KafkaOptionsProto object to its protobuf representation.
Returns:
KafkaOptionsProto protobuf
"""
kafka_options_proto = DataSourceProto.KafkaOptions(
bootstrap_servers=self.bootstrap_servers,
message_format=self.message_format.to_proto(),
topic=self.topic,
)
return kafka_options_proto
def to_proto(self) -> DataSourceProto.FileOptions:
"""
Converts an FileOptionsProto object to its protobuf representation.
Returns:
FileOptionsProto protobuf
"""
file_options_proto = DataSourceProto.FileOptions(
file_format=(None if self.file_format is None else
self.file_format.to_proto()),
file_url=self.file_url,
)
return file_options_proto
def to_proto(self) -> DataSourceProto.SnowflakeOptions:
"""
Converts an SnowflakeOptionsProto object to its protobuf representation.
Returns:
A SnowflakeOptionsProto protobuf.
"""
snowflake_options_proto = DataSourceProto.SnowflakeOptions(
database=self.database,
schema=self.schema,
table=self.table,
query=self.query,
warehouse=self.warehouse,
)
return snowflake_options_proto
def from_proto(data_source: DataSourceProto) -> Any:
assert data_source.HasField("spark_options")
spark_options = SparkOptions.from_proto(data_source.spark_options)
return SparkSource(
name=data_source.name,
field_mapping=dict(data_source.field_mapping),
table=spark_options.table,
query=spark_options.query,
path=spark_options.path,
file_format=spark_options.file_format,
timestamp_field=data_source.timestamp_field,
created_timestamp_column=data_source.created_timestamp_column,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
