def bq_query_input_solid(context, sql_queries: List[str]) -> List[DataFrame]: query_job_config = _preprocess_config(context.solid_config.get('query_job_config', {})) results = [] for sql_query in sql_queries: cfg = QueryJobConfig(**query_job_config) if query_job_config else None context.log.info( 'executing query %s with config: %s' % (sql_query, cfg.to_api_repr() if cfg else '(no config provided)') ) results.append(context.resources.bigquery.query(sql_query, job_config=cfg).to_dataframe()) return results
def _compute_fn(context, _): query_job_config = _preprocess_config(context.solid_config.get('query_job_config', {})) # Retrieve results as pandas DataFrames results = [] for sql_query in sql_queries: # We need to construct a new QueryJobConfig for each query. # See: https://bit.ly/2VjD6sl cfg = QueryJobConfig(**query_job_config) if query_job_config else None context.log.info( 'executing query %s with config: %s' % (sql_query, cfg.to_api_repr() if cfg else '(no config provided)') ) results.append(context.resources.bq.query(sql_query, job_config=cfg).to_dataframe()) yield Result(results)
def _solid(context): # pylint: disable=unused-argument query_job_config = _preprocess_config(context.solid_config.get("query_job_config", {})) # Retrieve results as pandas DataFrames results = [] for sql_query in sql_queries: # We need to construct a new QueryJobConfig for each query. # See: https://bit.ly/2VjD6sl cfg = QueryJobConfig(**query_job_config) if query_job_config else None context.log.info( "executing query %s with config: %s" % (sql_query, cfg.to_api_repr() if cfg else "(no config provided)") ) results.append( context.resources.bigquery.query(sql_query, job_config=cfg).to_dataframe() ) return results