def expand(self, pvalue):
if self._handle.endswith('.csv'):
# The input is CSV file(s).
schema = reddit.make_input_schema(mode=self._mode)
csv_coder = reddit.make_csv_coder(schema, mode=self._mode)
return (pvalue.pipeline
| 'ReadFromText' >> beam.io.ReadFromText(
self._handle,
# TODO(b/35653662): Obviate the need for setting this.
coder=beam.coders.BytesCoder())
| 'ParseCSV' >> beam.Map(csv_coder.decode))
else:
# The input is BigQuery table name(s).
query = reddit.make_standard_sql(self._handle, mode=self._mode)
return (pvalue.pipeline
| 'ReadFromBigQuery' >> beam.io.Read(
beam.io.BigQuerySource(query=query, use_standard_sql=True)))
def expand(self, pvalue):
if self._handle.endswith('.csv'):
# The input is CSV file(s).
schema = reddit.make_input_schema(mode=self._mode)
csv_coder = reddit.make_csv_coder(schema, mode=self._mode)
return (pvalue.pipeline
| 'ReadFromText' >> beam.io.ReadFromText(
self._handle,
# TODO(b/35653662): Obviate the need for setting this.
coder=beam.coders.BytesCoder())
| 'ParseCSV' >> beam.Map(csv_coder.decode))
else:
# The input is BigQuery table name(s).
query = reddit.make_standard_sql(self._handle, mode=self._mode)
return (pvalue.pipeline
| 'ReadFromBigQuery' >> beam.io.Read(
beam.io.BigQuerySource(query=query, use_standard_sql=True)))