def bq_query_input_solid(context, sql_queries: List[str]) -> List[DataFrame]:
query_job_config = _preprocess_config(context.solid_config.get('query_job_config', {}))
results = []
for sql_query in sql_queries:
cfg = QueryJobConfig(**query_job_config) if query_job_config else None
context.log.info(
'executing query %s with config: %s'
% (sql_query, cfg.to_api_repr() if cfg else '(no config provided)')
)
results.append(context.resources.bigquery.query(sql_query, job_config=cfg).to_dataframe())
return results
def _compute_fn(context, _):
query_job_config = _preprocess_config(context.solid_config.get('query_job_config', {}))
# Retrieve results as pandas DataFrames
results = []
for sql_query in sql_queries:
# We need to construct a new QueryJobConfig for each query.
# See: https://bit.ly/2VjD6sl
cfg = QueryJobConfig(**query_job_config) if query_job_config else None
context.log.info(
'executing query %s with config: %s'
% (sql_query, cfg.to_api_repr() if cfg else '(no config provided)')
)
results.append(context.resources.bq.query(sql_query, job_config=cfg).to_dataframe())
yield Result(results)
def _solid(context): # pylint: disable=unused-argument
query_job_config = _preprocess_config(context.solid_config.get("query_job_config", {}))
# Retrieve results as pandas DataFrames
results = []
for sql_query in sql_queries:
# We need to construct a new QueryJobConfig for each query.
# See: https://bit.ly/2VjD6sl
cfg = QueryJobConfig(**query_job_config) if query_job_config else None
context.log.info(
"executing query %s with config: %s"
% (sql_query, cfg.to_api_repr() if cfg else "(no config provided)")
)
results.append(
context.resources.bigquery.query(sql_query, job_config=cfg).to_dataframe()
)
return results