def from_proto(data_source: DataSourceProto):
assert data_source.HasField("batch_source")
batch_source = DataSource.from_proto(data_source.batch_source)
return PushSource(
name=data_source.name,
batch_source=batch_source,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
def from_proto(data_source: DataSourceProto):
assert data_source.HasField("bigquery_options")
return BigQuerySource(
field_mapping=dict(data_source.field_mapping),
table_ref=data_source.bigquery_options.table_ref,
event_timestamp_column=data_source.event_timestamp_column,
created_timestamp_column=data_source.created_timestamp_column,
date_partition_column=data_source.date_partition_column,
query=data_source.bigquery_options.query,
)
def from_proto(data_source: DataSourceProto):
assert data_source.HasField("custom_options")
postgres_options = json.loads(data_source.custom_options.configuration)
return PostgreSQLSource(
name=postgres_options["name"],
query=postgres_options["query"],
field_mapping=dict(data_source.field_mapping),
timestamp_field=data_source.timestamp_field,
created_timestamp_column=data_source.created_timestamp_column,
date_partition_column=data_source.date_partition_column,
)
def from_proto(data_source: DataSourceProto):
assert data_source.HasField("bigquery_options")
return BigQuerySource(
name=data_source.name,
field_mapping=dict(data_source.field_mapping),
table=data_source.bigquery_options.table,
timestamp_field=data_source.timestamp_field,
created_timestamp_column=data_source.created_timestamp_column,
query=data_source.bigquery_options.query,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
def from_proto(data_source: DataSourceProto) -> Any:
assert data_source.HasField("spark_options")
spark_options = SparkOptions.from_proto(data_source.spark_options)
return SparkSource(
name=data_source.name,
field_mapping=dict(data_source.field_mapping),
table=spark_options.table,
query=spark_options.query,
path=spark_options.path,
file_format=spark_options.file_format,
timestamp_field=data_source.timestamp_field,
created_timestamp_column=data_source.created_timestamp_column,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
