def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( type=DataSourceProto.BATCH_REDSHIFT, field_mapping=self.field_mapping, redshift_options=self.redshift_options.to_proto(), ) data_source_proto.event_timestamp_column = self.event_timestamp_column data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column return data_source_proto
def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( type=DataSourceProto.BATCH_BIGQUERY, field_mapping=self.field_mapping, bigquery_options=self.bigquery_options.to_proto(), ) data_source_proto.event_timestamp_column = self.event_timestamp_column data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column return data_source_proto
def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( type=DataSourceProto.STREAM_KINESIS, field_mapping=self.field_mapping, kinesis_options=self.kinesis_options.to_proto(), ) data_source_proto.event_timestamp_column = self.event_timestamp_column data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column return data_source_proto
def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( type=DataSourceProto.CUSTOM_SOURCE, data_source_class_type= "feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source.PostgreSQLSource", field_mapping=self.field_mapping, custom_options=self._postgres_options.to_proto(), ) data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column return data_source_proto
def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( name=self.name, type=DataSourceProto.BATCH_BIGQUERY, field_mapping=self.field_mapping, bigquery_options=self.bigquery_options.to_proto(), description=self.description, tags=self.tags, owner=self.owner, ) data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column return data_source_proto
def _ingest_test_getfeaturetable_mocked_resp(file_url: str, date_partition_col: str = ""): return GetFeatureTableResponse(table=FeatureTableProto( spec=FeatureTableSpecProto( name="ingest_featuretable", max_age=Duration(seconds=3600), features=[ FeatureSpecProto( name="dev_feature_float", value_type=ValueProto.ValueType.FLOAT, ), FeatureSpecProto( name="dev_feature_string", value_type=ValueProto.ValueType.STRING, ), ], entities=["dev_entity"], batch_source=DataSourceProto( file_options=DataSourceProto.FileOptions( file_format=ParquetFormat().to_proto(), file_url=file_url), event_timestamp_column="datetime", created_timestamp_column="timestamp", date_partition_column=date_partition_col, ), ), meta=FeatureTableMetaProto(), ))
def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( name=self.name, type=DataSourceProto.BATCH_TRINO, field_mapping=self.field_mapping, trino_options=self.trino_options.to_proto(), description=self.description, tags=self.tags, owner=self.owner, ) data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column return data_source_proto
def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( name=self.name, type=DataSourceProto.BATCH_SPARK, data_source_class_type= "feast.infra.offline_stores.contrib.spark_offline_store.spark_source.SparkSource", field_mapping=self.field_mapping, spark_options=self.spark_options.to_proto(), description=self.description, tags=self.tags, owner=self.owner, ) data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column return data_source_proto
def to_proto(self) -> DataSourceProto.CustomSourceOptions: postgres_options_proto = DataSourceProto.CustomSourceOptions( configuration=json.dumps({ "name": self._name, "query": self._query }).encode()) return postgres_options_proto
def to_proto(self) -> DataSourceProto: """ Converts a RedshiftSource object to its protobuf representation. Returns: A DataSourceProto object. """ data_source_proto = DataSourceProto( type=DataSourceProto.BATCH_REDSHIFT, field_mapping=self.field_mapping, redshift_options=self.redshift_options.to_proto(), ) data_source_proto.event_timestamp_column = self.event_timestamp_column data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column return data_source_proto
def to_proto(self) -> DataSourceProto: data_source_proto = DataSourceProto( name=self.name, type=DataSourceProto.STREAM_KAFKA, field_mapping=self.field_mapping, kafka_options=self.kafka_options.to_proto(), description=self.description, tags=self.tags, owner=self.owner, ) data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column if self.batch_source: data_source_proto.batch_source.MergeFrom( self.batch_source.to_proto()) return data_source_proto
def to_proto(self) -> DataSourceProto: schema_pb = {} for key, value in self._schema.items(): schema_pb[key] = value.value options = DataSourceProto.RequestDataOptions(name=self._name, schema=schema_pb) data_source_proto = DataSourceProto( type=DataSourceProto.REQUEST_SOURCE, request_data_options=options) return data_source_proto
def to_proto(self) -> DataSourceProto: """ Converts a RedshiftSource object to its protobuf representation. Returns: A DataSourceProto object. """ data_source_proto = DataSourceProto( type=DataSourceProto.BATCH_REDSHIFT, field_mapping=self.field_mapping, redshift_options=self.redshift_options.to_proto(), description=self.description, tags=self.tags, owner=self.owner, ) data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column return data_source_proto
def from_proto(data_source: DataSourceProto): assert data_source.HasField("batch_source") batch_source = DataSource.from_proto(data_source.batch_source) return PushSource( name=data_source.name, batch_source=batch_source, description=data_source.description, tags=dict(data_source.tags), owner=data_source.owner, )
def to_proto(self) -> DataSourceProto: """ Converts a SnowflakeSource object to its protobuf representation. Returns: A DataSourceProto object. """ data_source_proto = DataSourceProto( type=DataSourceProto.BATCH_SNOWFLAKE, field_mapping=self.field_mapping, snowflake_options=self.snowflake_options.to_proto(), description=self.description, tags=self.tags, owner=self.owner, ) data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column return data_source_proto
def to_proto(self) -> DataSourceProto.BigQueryOptions: """ Converts an BigQueryOptionsProto object to its protobuf representation. Returns: BigQueryOptionsProto protobuf """ bigquery_options_proto = DataSourceProto.BigQueryOptions( table_ref=self.table_ref, ) return bigquery_options_proto
def to_proto(self) -> DataSourceProto.TrinoOptions: """ Converts an TrinoOptionsProto object to its protobuf representation. Returns: TrinoOptionsProto protobuf """ trino_options_proto = DataSourceProto.TrinoOptions( table=self.table, query=self.query, ) return trino_options_proto
def from_proto(data_source: DataSourceProto): assert data_source.HasField("custom_options") postgres_options = json.loads(data_source.custom_options.configuration) return PostgreSQLSource( name=postgres_options["name"], query=postgres_options["query"], field_mapping=dict(data_source.field_mapping), timestamp_field=data_source.timestamp_field, created_timestamp_column=data_source.created_timestamp_column, date_partition_column=data_source.date_partition_column, )
def to_proto(self) -> DataSourceProto.RedshiftOptions: """ Converts an RedshiftOptionsProto object to its protobuf representation. Returns: A RedshiftOptionsProto protobuf. """ redshift_options_proto = DataSourceProto.RedshiftOptions( table=self.table, schema=self.schema, query=self.query, ) return redshift_options_proto
def from_proto(data_source: DataSourceProto): assert data_source.HasField("bigquery_options") return BigQuerySource( field_mapping=dict(data_source.field_mapping), table_ref=data_source.bigquery_options.table_ref, event_timestamp_column=data_source.event_timestamp_column, created_timestamp_column=data_source.created_timestamp_column, date_partition_column=data_source.date_partition_column, query=data_source.bigquery_options.query, )
def to_proto(self) -> DataSourceProto.SparkOptions: """ Converts an SparkOptionsProto object to its protobuf representation. Returns: SparkOptionsProto protobuf """ spark_options_proto = DataSourceProto.SparkOptions( table=self.table, query=self.query, path=self.path, file_format=self.file_format, ) return spark_options_proto
def from_proto(data_source: DataSourceProto): assert data_source.HasField("bigquery_options") return BigQuerySource( name=data_source.name, field_mapping=dict(data_source.field_mapping), table=data_source.bigquery_options.table, timestamp_field=data_source.timestamp_field, created_timestamp_column=data_source.created_timestamp_column, query=data_source.bigquery_options.query, description=data_source.description, tags=dict(data_source.tags), owner=data_source.owner, )
def to_proto(self) -> DataSourceProto.KinesisOptions: """ Converts an KinesisOptionsProto object to its protobuf representation. Returns: KinesisOptionsProto protobuf """ kinesis_options_proto = DataSourceProto.KinesisOptions( record_format=self.record_format.to_proto(), region=self.region, stream_name=self.stream_name, ) return kinesis_options_proto
def to_proto(self) -> DataSourceProto.KafkaOptions: """ Converts an KafkaOptionsProto object to its protobuf representation. Returns: KafkaOptionsProto protobuf """ kafka_options_proto = DataSourceProto.KafkaOptions( bootstrap_servers=self.bootstrap_servers, message_format=self.message_format.to_proto(), topic=self.topic, ) return kafka_options_proto
def to_proto(self) -> DataSourceProto.FileOptions: """ Converts an FileOptionsProto object to its protobuf representation. Returns: FileOptionsProto protobuf """ file_options_proto = DataSourceProto.FileOptions( file_format=(None if self.file_format is None else self.file_format.to_proto()), file_url=self.file_url, ) return file_options_proto
def to_proto(self) -> DataSourceProto.SnowflakeOptions: """ Converts an SnowflakeOptionsProto object to its protobuf representation. Returns: A SnowflakeOptionsProto protobuf. """ snowflake_options_proto = DataSourceProto.SnowflakeOptions( database=self.database, schema=self.schema, table=self.table, query=self.query, warehouse=self.warehouse, ) return snowflake_options_proto
def from_proto(data_source: DataSourceProto) -> Any: assert data_source.HasField("spark_options") spark_options = SparkOptions.from_proto(data_source.spark_options) return SparkSource( name=data_source.name, field_mapping=dict(data_source.field_mapping), table=spark_options.table, query=spark_options.query, path=spark_options.path, file_format=spark_options.file_format, timestamp_field=data_source.timestamp_field, created_timestamp_column=data_source.created_timestamp_column, description=data_source.description, tags=dict(data_source.tags), owner=data_source.owner, )