def test_to_user_agent_minimal(): info = client_info.ClientInfo( python_version="1", api_core_version="2", grpc_version=None ) user_agent = info.to_user_agent() assert user_agent == "gl-python/1 gax/2"
def test_constructor_defaults(): info = client_info.ClientInfo() assert info.python_version is not None assert info.grpc_version is not None assert info.api_core_version is not None assert info.gapic_version is None assert info.client_library_version is None
def _cell_magic(line, query): """Underlying function for bigquery cell magic Note: This function contains the underlying logic for the 'bigquery' cell magic. This function is not meant to be called directly. Args: line (str): "%%bigquery" followed by arguments as required query (str): SQL query to run Returns: pandas.DataFrame: the query results. """ args = magic_arguments.parse_argstring(_cell_magic, line) params = [] if args.params is not None: try: params = _helpers.to_query_parameters( ast.literal_eval("".join(args.params))) except Exception: raise SyntaxError( "--params is not a correctly formatted JSON string or a JSON " "serializable dictionary") project = args.project or context.project client = bigquery.Client( project=project, credentials=context.credentials, default_query_job_config=context.default_query_job_config, client_info=client_info.ClientInfo( user_agent="ipython-{}".format(IPython.__version__)), ) if context._connection: client._connection = context._connection bqstorage_client = _make_bqstorage_client( args.use_bqstorage_api or context.use_bqstorage_api, context.credentials) job_config = bigquery.job.QueryJobConfig() job_config.query_parameters = params job_config.use_legacy_sql = args.use_legacy_sql if args.maximum_bytes_billed == "None": job_config.maximum_bytes_billed = 0 elif args.maximum_bytes_billed is not None: value = int(args.maximum_bytes_billed) job_config.maximum_bytes_billed = value query_job = _run_query(client, query, job_config) if not args.verbose: display.clear_output() result = query_job.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) else: return result
def get_bigquery_client(cls, profile_credentials): database = profile_credentials.database creds = cls.get_bigquery_credentials(profile_credentials) location = getattr(profile_credentials, 'location', None) info = client_info.ClientInfo(user_agent=f'dbt-{dbt_version}') return google.cloud.bigquery.Client( database, creds, location=location, client_info=info, )
def test_to_user_agent_full(): info = client_info.ClientInfo( python_version="1", grpc_version="2", api_core_version="3", gapic_version="4", client_library_version="5", user_agent="app-name/1.0", ) user_agent = info.to_user_agent() assert user_agent == "app-name/1.0 gl-python/1 grpc/2 gax/3 gapic/4 gccl/5"
def test_it(): client = mock.Mock( _credentials="creds", secure=True, host="thehost", stub=object(), spec=("_credentials", "secure", "host", "stub"), client_info=client_info.ClientInfo( user_agent="google-cloud-ndb/{}".format(__version__)), ) context = context_module.Context(client) with context.use(): assert _api.stub() is client.stub
def get_bigquery_client(cls, profile_credentials): if profile_credentials.impersonate_service_account: creds =\ cls.get_impersonated_bigquery_credentials(profile_credentials) else: creds = cls.get_bigquery_credentials(profile_credentials) database = profile_credentials.database location = getattr(profile_credentials, 'location', None) info = client_info.ClientInfo(user_agent=f'dbt-{dbt_version}') return google.cloud.bigquery.Client( database, creds, location=location, client_info=info, )
def test_constructor_options(): info = client_info.ClientInfo( python_version="1", grpc_version="2", api_core_version="3", gapic_version="4", client_library_version="5", user_agent="6" ) assert info.python_version == "1" assert info.grpc_version == "2" assert info.api_core_version == "3" assert info.gapic_version == "4" assert info.client_library_version == "5" assert info.user_agent == "6"
def test_secure_channel(datastore_pb2_grpc, _helpers): channel = _helpers.make_secure_channel.return_value client = mock.Mock( _credentials="creds", secure=True, host="thehost", spec=("_credentials", "secure", "host"), client_info=client_info.ClientInfo( user_agent="google-cloud-ndb/{}".format(__version__) ), ) context = context_module.Context(client) with context.use(): stub = _api.stub() assert _api.stub() is stub # one stub per context assert stub is datastore_pb2_grpc.DatastoreStub.return_value datastore_pb2_grpc.DatastoreStub.assert_called_once_with(channel) _helpers.make_secure_channel.assert_called_once_with( "creds", client.client_info.to_user_agent(), "thehost" )
def _cell_magic(line, query): """Underlying function for bigquery cell magic Note: This function contains the underlying logic for the 'bigquery' cell magic. This function is not meant to be called directly. Args: line (str): "%%bigquery" followed by arguments as required query (str): SQL query to run Returns: pandas.DataFrame: the query results. """ args = magic_arguments.parse_argstring(_cell_magic, line) if args.use_bqstorage_api is not None: warnings.warn( "Deprecated option --use_bqstorage_api, the BigQuery " "Storage API is already used by default.", category=DeprecationWarning, ) use_bqstorage_api = not args.use_rest_api params = [] if args.params is not None: try: params = _helpers.to_query_parameters( ast.literal_eval("".join(args.params))) except Exception: raise SyntaxError( "--params is not a correctly formatted JSON string or a JSON " "serializable dictionary") project = args.project or context.project client = bigquery.Client( project=project, credentials=context.credentials, default_query_job_config=context.default_query_job_config, client_info=client_info.ClientInfo(user_agent=IPYTHON_USER_AGENT), ) if context._connection: client._connection = context._connection bqstorage_client = _make_bqstorage_client(use_bqstorage_api, context.credentials) close_transports = functools.partial(_close_transports, client, bqstorage_client) try: if args.max_results: max_results = int(args.max_results) else: max_results = None query = query.strip() if not query: error = ValueError("Query is missing.") _handle_error(error, args.destination_var) return # Any query that does not contain whitespace (aside from leading and trailing whitespace) # is assumed to be a table id if not re.search(r"\s", query): try: rows = client.list_rows(query, max_results=max_results) except Exception as ex: _handle_error(ex, args.destination_var) return result = rows.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) return else: return result job_config = bigquery.job.QueryJobConfig() job_config.query_parameters = params job_config.use_legacy_sql = args.use_legacy_sql job_config.dry_run = args.dry_run if args.destination_table: split = args.destination_table.split(".") if len(split) != 2: raise ValueError( "--destination_table should be in a <dataset_id>.<table_id> format." ) dataset_id, table_id = split job_config.allow_large_results = True dataset_ref = bigquery.dataset.DatasetReference( client.project, dataset_id) destination_table_ref = dataset_ref.table(table_id) job_config.destination = destination_table_ref job_config.create_disposition = "CREATE_IF_NEEDED" job_config.write_disposition = "WRITE_TRUNCATE" _create_dataset_if_necessary(client, dataset_id) if args.maximum_bytes_billed == "None": job_config.maximum_bytes_billed = 0 elif args.maximum_bytes_billed is not None: value = int(args.maximum_bytes_billed) job_config.maximum_bytes_billed = value try: query_job = _run_query(client, query, job_config=job_config) except Exception as ex: _handle_error(ex, args.destination_var) return if not args.verbose: display.clear_output() if args.dry_run and args.destination_var: IPython.get_ipython().push({args.destination_var: query_job}) return elif args.dry_run: print("Query validated. This query will process {} bytes.".format( query_job.total_bytes_processed)) return query_job if max_results: result = query_job.result(max_results=max_results).to_dataframe( bqstorage_client=bqstorage_client) else: result = query_job.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) else: return result finally: close_transports()
def _cell_magic(line, query): """Underlying function for bigquery cell magic Note: This function contains the underlying logic for the 'bigquery' cell magic. This function is not meant to be called directly. Args: line (str): "%%bigquery" followed by arguments as required query (str): SQL query to run Returns: pandas.DataFrame: the query results. """ # The built-in parser does not recognize Python structures such as dicts, thus # we extract the "--params" option and inteprpret it separately. try: params_option_value, rest_of_args = _split_args_line(line) except lap.exceptions.QueryParamsParseError as exc: rebranded_error = SyntaxError( "--params is not a correctly formatted JSON string or a JSON " "serializable dictionary") six.raise_from(rebranded_error, exc) except lap.exceptions.DuplicateQueryParamsError as exc: rebranded_error = ValueError("Duplicate --params option.") six.raise_from(rebranded_error, exc) except lap.exceptions.ParseError as exc: rebranded_error = ValueError( "Unrecognized input, are option values correct? " "Error details: {}".format(exc.args[0])) six.raise_from(rebranded_error, exc) args = magic_arguments.parse_argstring(_cell_magic, rest_of_args) if args.use_bqstorage_api is not None: warnings.warn( "Deprecated option --use_bqstorage_api, the BigQuery " "Storage API is already used by default.", category=DeprecationWarning, ) use_bqstorage_api = not args.use_rest_api params = [] if params_option_value: # A non-existing params variable is not expanded and ends up in the input # in its raw form, e.g. "$query_params". if params_option_value.startswith("$"): msg = 'Parameter expansion failed, undefined variable "{}".'.format( params_option_value[1:]) raise NameError(msg) params = _helpers.to_query_parameters( ast.literal_eval(params_option_value)) project = args.project or context.project bigquery_client_options = copy.deepcopy(context.bigquery_client_options) if args.bigquery_api_endpoint: if isinstance(bigquery_client_options, dict): bigquery_client_options[ "api_endpoint"] = args.bigquery_api_endpoint else: bigquery_client_options.api_endpoint = args.bigquery_api_endpoint client = bigquery.Client( project=project, credentials=context.credentials, default_query_job_config=context.default_query_job_config, client_info=client_info.ClientInfo(user_agent=IPYTHON_USER_AGENT), client_options=bigquery_client_options, ) if context._connection: client._connection = context._connection bqstorage_client_options = copy.deepcopy(context.bqstorage_client_options) if args.bqstorage_api_endpoint: if isinstance(bqstorage_client_options, dict): bqstorage_client_options[ "api_endpoint"] = args.bqstorage_api_endpoint else: bqstorage_client_options.api_endpoint = args.bqstorage_api_endpoint bqstorage_client = _make_bqstorage_client( use_bqstorage_api, context.credentials, bqstorage_client_options, ) close_transports = functools.partial(_close_transports, client, bqstorage_client) try: if args.max_results: max_results = int(args.max_results) else: max_results = None query = query.strip() if not query: error = ValueError("Query is missing.") _handle_error(error, args.destination_var) return # Any query that does not contain whitespace (aside from leading and trailing whitespace) # is assumed to be a table id if not re.search(r"\s", query): try: rows = client.list_rows(query, max_results=max_results) except Exception as ex: _handle_error(ex, args.destination_var) return result = rows.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) return else: return result job_config = bigquery.job.QueryJobConfig() job_config.query_parameters = params job_config.use_legacy_sql = args.use_legacy_sql job_config.dry_run = args.dry_run if args.destination_table: split = args.destination_table.split(".") if len(split) != 2: raise ValueError( "--destination_table should be in a <dataset_id>.<table_id> format." ) dataset_id, table_id = split job_config.allow_large_results = True dataset_ref = bigquery.dataset.DatasetReference( client.project, dataset_id) destination_table_ref = dataset_ref.table(table_id) job_config.destination = destination_table_ref job_config.create_disposition = "CREATE_IF_NEEDED" job_config.write_disposition = "WRITE_TRUNCATE" _create_dataset_if_necessary(client, dataset_id) if args.maximum_bytes_billed == "None": job_config.maximum_bytes_billed = 0 elif args.maximum_bytes_billed is not None: value = int(args.maximum_bytes_billed) job_config.maximum_bytes_billed = value try: query_job = _run_query(client, query, job_config=job_config) except Exception as ex: _handle_error(ex, args.destination_var) return if not args.verbose: display.clear_output() if args.dry_run and args.destination_var: IPython.get_ipython().push({args.destination_var: query_job}) return elif args.dry_run: print("Query validated. This query will process {} bytes.".format( query_job.total_bytes_processed)) return query_job if max_results: result = query_job.result(max_results=max_results).to_dataframe( bqstorage_client=bqstorage_client) else: result = query_job.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) else: return result finally: close_transports()
from google.api_core import client_info # noqa _HAVE_GRPC = False else: from google.api_core.gapic_v1 import client_info _HAVE_GRPC = True from google.cloud.client import ClientWithProject from google.cloud.error_reporting import __version__ from google.cloud.error_reporting._logging import _ErrorReportingLoggingAPI from google.cloud.environment_vars import DISABLE_GRPC _DISABLE_GRPC = os.getenv(DISABLE_GRPC, False) _USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC _CLIENT_INFO = client_info.ClientInfo(client_library_version=__version__) class HTTPContext(object): """HTTPContext defines an object that captures the parameter for the httpRequest part of Error Reporting API :type method: str :param method: The type of HTTP request, such as GET, POST, etc. :type url: str :param url: The URL of the request :type user_agent: str :param user_agent: The user agent information that is provided with the request.
def _cell_magic(line, query): """Underlying function for bigquery cell magic Note: This function contains the underlying logic for the 'bigquery' cell magic. This function is not meant to be called directly. Args: line (str): "%%bigquery" followed by arguments as required query (str): SQL query to run Returns: pandas.DataFrame: the query results. """ args = magic_arguments.parse_argstring(_cell_magic, line) params = [] if args.params is not None: try: params = _helpers.to_query_parameters( ast.literal_eval("".join(args.params)) ) except Exception: raise SyntaxError( "--params is not a correctly formatted JSON string or a JSON " "serializable dictionary" ) project = args.project or context.project client = bigquery.Client( project=project, credentials=context.credentials, default_query_job_config=context.default_query_job_config, client_info=client_info.ClientInfo(user_agent=IPYTHON_USER_AGENT), ) if context._connection: client._connection = context._connection bqstorage_client = _make_bqstorage_client( args.use_bqstorage_api or context.use_bqstorage_api, context.credentials ) job_config = bigquery.job.QueryJobConfig() job_config.query_parameters = params job_config.use_legacy_sql = args.use_legacy_sql job_config.dry_run = args.dry_run if args.maximum_bytes_billed == "None": job_config.maximum_bytes_billed = 0 elif args.maximum_bytes_billed is not None: value = int(args.maximum_bytes_billed) job_config.maximum_bytes_billed = value error = None try: query_job = _run_query(client, query, job_config) except Exception as ex: error = str(ex) if not args.verbose: display.clear_output() if error: if args.destination_var: print( "Could not save output to variable '{}'.".format(args.destination_var), file=sys.stderr, ) print("\nERROR:\n", error, file=sys.stderr) return if args.dry_run and args.destination_var: IPython.get_ipython().push({args.destination_var: query_job}) return elif args.dry_run: print( "Query validated. This query will process {} bytes.".format( query_job.total_bytes_processed ) ) return query_job result = query_job.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) else: return result
import grpc import os import requests from google.api_core import client_info from google.cloud import environment_vars from google.cloud import _helpers from google.cloud import client as google_client from google.cloud.datastore_v1.gapic import datastore_client from google.cloud.datastore_v1.proto import datastore_pb2_grpc from google.cloud.ndb import __version__ from google.cloud.ndb import context as context_module from google.cloud.ndb import key as key_module _CLIENT_INFO = client_info.ClientInfo( user_agent="google-cloud-ndb/{}".format(__version__)) DATASTORE_API_HOST = datastore_client.DatastoreClient.SERVICE_ADDRESS.rsplit( ":", 1)[0] def _get_gcd_project(): """Gets the GCD application ID if it can be inferred.""" return os.getenv(environment_vars.GCD_DATASET) def _determine_default_project(project=None): """Determine default project explicitly or implicitly as fall-back. In implicit case, supports four environments. In order of precedence, the implicit environments are:
def google_client_info(): user_agent = USER_AGENT_TEMPLATE.format(sqlalchemy.__version__) return client_info.ClientInfo(user_agent=user_agent)
def _cell_magic(line, query): """Underlying function for bigquery cell magic Note: This function contains the underlying logic for the 'bigquery' cell magic. This function is not meant to be called directly. Args: line (str): "%%bigquery" followed by arguments as required query (str): SQL query to run Returns: pandas.DataFrame: the query results. """ args = magic_arguments.parse_argstring(_cell_magic, line) params = [] if args.params is not None: try: params = _helpers.to_query_parameters( ast.literal_eval("".join(args.params))) except Exception: raise SyntaxError( "--params is not a correctly formatted JSON string or a JSON " "serializable dictionary") project = args.project or context.project client = bigquery.Client( project=project, credentials=context.credentials, default_query_job_config=context.default_query_job_config, client_info=client_info.ClientInfo(user_agent=IPYTHON_USER_AGENT), ) if context._connection: client._connection = context._connection bqstorage_client = _make_bqstorage_client( args.use_bqstorage_api or context.use_bqstorage_api, context.credentials) if args.max_results: max_results = int(args.max_results) else: max_results = None query = query.strip() # Any query that does not contain whitespace (aside from leading and trailing whitespace) # is assumed to be a table id if not re.search(r"\s", query): try: rows = client.list_rows(query, max_results=max_results) except Exception as ex: _print_error(str(ex), args.destination_var) return result = rows.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) return else: return result job_config = bigquery.job.QueryJobConfig() job_config.query_parameters = params job_config.use_legacy_sql = args.use_legacy_sql job_config.dry_run = args.dry_run if args.maximum_bytes_billed == "None": job_config.maximum_bytes_billed = 0 elif args.maximum_bytes_billed is not None: value = int(args.maximum_bytes_billed) job_config.maximum_bytes_billed = value try: query_job = _run_query(client, query, job_config=job_config) except Exception as ex: _print_error(str(ex), args.destination_var) return if not args.verbose: display.clear_output() if args.dry_run and args.destination_var: IPython.get_ipython().push({args.destination_var: query_job}) return elif args.dry_run: print("Query validated. This query will process {} bytes.".format( query_job.total_bytes_processed)) return query_job if max_results: result = query_job.result(max_results=max_results).to_dataframe( bqstorage_client=bqstorage_client) else: result = query_job.to_dataframe(bqstorage_client=bqstorage_client) if args.destination_var: IPython.get_ipython().push({args.destination_var: result}) else: return result
def get_http_client_info(): return http_client_info.ClientInfo(user_agent=USER_AGENT)