def client(): client = storage.Client( project=_CONF_TEST_PROJECT_ID, credentials=AnonymousCredentials(), client_options={"api_endpoint": _HOST}, ) return client
def setUpModule(): if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials emulator_project = os.getenv("GCLOUD_PROJECT", "emulator-test-project") Config.CLIENT = Client(project=emulator_project, credentials=AnonymousCredentials()) else: Config.CLIENT = Client() retry = RetryErrors(exceptions.ServiceUnavailable) configs = list(retry(Config.CLIENT.list_instance_configs)()) instances = retry(_list_instances)() EXISTING_INSTANCES[:] = instances if CREATE_INSTANCE: if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. configs = [config for config in configs if "-us-" in config.name] if not configs: raise ValueError("List instance configs failed in module set up.") Config.INSTANCE_CONFIG = configs[0] config_name = configs[0].name Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name) created_op = Config.INSTANCE.create() created_op.result(30) # block until completion else: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload()
def __init__( self, project=None, credentials=None, database=DEFAULT_DATABASE, client_info=_CLIENT_INFO, client_options=None, ) -> None: # NOTE: This API has no use for the _http argument, but sending it # will have no impact since the _http() @property only lazily # creates a working HTTP object. self._emulator_host = os.getenv(_FIRESTORE_EMULATOR_HOST) if self._emulator_host is not None: if credentials is None: credentials = AnonymousCredentials() if project is None: project = _DEFAULT_EMULATOR_PROJECT super(BaseClient, self).__init__( project=project, credentials=credentials, client_options=client_options, _http=None, ) self._client_info = client_info if client_options: if type(client_options) == dict: client_options = google.api_core.client_options.from_dict( client_options) self._client_options = client_options self._database = database
def _create_dummy_storage_client(): fake_host = os.getenv('STORAGE_PORT_4443_TCP_ADDR') external_url = 'https://{}:4443'.format(fake_host) storage.blob._API_ACCESS_ENDPOINT = 'https://storage.gcs.{}.nip.io:4443'.format(fake_host) storage.blob._DOWNLOAD_URL_TEMPLATE = ( "%s/download/storage/v1{path}?alt=media" % external_url ) storage.blob._BASE_UPLOAD_TEMPLATE = ( "%s/upload/storage/v1{bucket_path}/o?uploadType=" % external_url ) storage.blob._MULTIPART_URL_TEMPLATE = storage.blob._BASE_UPLOAD_TEMPLATE + "multipart" storage.blob._RESUMABLE_URL_TEMPLATE = storage.blob._BASE_UPLOAD_TEMPLATE + "resumable" my_http = requests.Session() my_http.verify = False # disable SSL validation urllib3.disable_warnings( urllib3.exceptions.InsecureRequestWarning ) # disable https warnings for https insecure certs storage_client = storage.Client( credentials=AnonymousCredentials(), project='test', _http=my_http, client_options=ClientOptions(api_endpoint=external_url)) if len(list(storage_client.list_buckets())) == 0: bucket = storage_client.create_bucket(_get_bucket_name()) return storage_client
def setup_engine_options(self, engine_options): self.credentials = None self.instance_id = None self.project_id = None bigtable_emu = os.environ.get('BIGTABLE_EMULATOR_HOST', None) if bigtable_emu or ("emulator" in engine_options and engine_options["emulator"]): self.credentials = AnonymousCredentials() elif "credentials" not in engine_options: raise ValueError("missing credentials option for bigtable engine") else: self.credentials = engine_options["credentials"] # use this if credentials is a path to a key.json service account # self.credentials = service_account.Credentials.from_service_account_file(engine_options["credentials"]) if "project_id" not in engine_options: raise ValueError("missing project_id option for bigtable engine") else: self.project_id = engine_options["project_id"] if "instance_id" not in engine_options: raise ValueError("missing instance_id option for bigtable engine") else: self.instance_id = engine_options["instance_id"]
def test_database_admin_api_emulator_code(self): from google.auth.credentials import AnonymousCredentials from google.api_core.client_options import ClientOptions credentials = AnonymousCredentials() client_info = mock.Mock() client_options = ClientOptions(api_endpoint="emulator.host") client = self._make_one( project=self.PROJECT, credentials=credentials, client_info=client_info, client_options=client_options, ) db_module = "google.cloud.spanner_v1.client.DatabaseAdminClient" with mock.patch(db_module) as database_admin_client: api = client.database_admin_api self.assertIs(api, database_admin_client.return_value) # API instance is cached again = client.database_admin_api self.assertIs(again, api) self.assertEqual(len(database_admin_client.call_args_list), 1) called_args, called_kw = database_admin_client.call_args self.assertEqual(called_args, ()) self.assertEqual(called_kw["client_info"], client_info) self.assertEqual(called_kw["client_options"], client_options) self.assertIn("transport", called_kw) self.assertNotIn("credentials", called_kw)
def create_storage_client(test): if test: EXTERNAL_URL = "https://127.0.0.1:4443" PUBLIC_HOST = "storage.gcs.127.0.0.1.nip.io:4443" storage.blob._API_ACCESS_ENDPOINT = "https://" + PUBLIC_HOST storage.blob._DOWNLOAD_URL_TEMPLATE = ( u"%s/download/storage/v1{path}?alt=media" % EXTERNAL_URL) storage.blob._BASE_UPLOAD_TEMPLATE = ( u"%s/upload/storage/v1{bucket_path}/o?uploadType=" % EXTERNAL_URL) storage.blob._MULTIPART_URL_TEMPLATE = storage.blob._BASE_UPLOAD_TEMPLATE + u"multipart" storage.blob._RESUMABLE_URL_TEMPLATE = storage.blob._BASE_UPLOAD_TEMPLATE + u"resumable" my_http = requests.Session() my_http.verify = False # disable SSL validation urllib3.disable_warnings( urllib3.exceptions.InsecureRequestWarning ) # disable https warnings for https insecure certs storage_client = storage.Client( credentials=AnonymousCredentials(), project="test", _http=my_http, client_options=ClientOptions(api_endpoint=EXTERNAL_URL), ) else: storage_client = storage.Client() return storage_client
def gc_credentials(): parser = ConfigParser() parser.read("google_cloud_credentials.ini") credentials_path = parser.get("google-cloud-tests", "credentials_json_path", fallback=None) emulator_endpoint = parser.get("google-cloud-tests", "emulator_endpoint", fallback=None) assert ( credentials_path or emulator_endpoint ), "Either set endpoint (for gc emulation) or credentials_json_path (for actual gc)" if emulator_endpoint: # google's client library looks for this env var # if we didn't set it it would use the standard endpoint # at https://storage.googleapis.com os.environ["STORAGE_EMULATOR_HOST"] = emulator_endpoint credentials = AnonymousCredentials() else: # if no endpoint was defined we're running against actual GC and need credentials credentials = credentials_path yield credentials # unset the env var if emulator_endpoint: del os.environ["STORAGE_EMULATOR_HOST"]
def gcsbucket(): client = storage.Client( credentials=AnonymousCredentials(), client_options={ "api_endpoint": os.environ.get("GCS_API_ENDPOINT", "http://127.0.0.1:4443") }, project="test", ) class GCSBucket(Bucket): def __init__(self, name): self.name = name self._client = client self._client.create_bucket(name) def put(self, key, content=""): self._client.get_bucket( self.name).blob(key).upload_from_string(content) def get(self, key): return self._client.get_bucket(self.name).get_blob(key) @property def root(self): return f"gs://{self.name}/" # To connect with fake-gcs-server, recreate gcs client paaaaath.gcs.GCSPath.register_client(client) bucket = "".join([ random.choice("0123456789abcdefghijklmnopqrstuvwxyz") for _ in range(32) ]) yield GCSBucket(bucket)
def commit(self, byte_data, override_blob_name=None): # this means we're testing if os.environ.get("STORAGE_EMULATOR_HOST") is not None: client = storage.Client( credentials=AnonymousCredentials(), project=self.project, ) else: # pragma: no cover client = storage.Client(project=self.project, credentials=self.credentials) self.gcs_bucket = client.get_bucket(self.bucket) self.filename = self.filename_without_bucket # if we've been given the filename, use that, otherwise get the # name from the path builder if override_blob_name: blob_name = override_blob_name else: blob_name = self._build_path() try: blob = self.gcs_bucket.blob(blob_name) self.retry(blob.upload_from_string)( byte_data, content_type="application/octet-stream" ) return blob_name except Exception as err: import traceback logger = get_logger() logger.error( f"Error Saving Blob to GCS {type(err).__name__} - {err}\n{traceback.format_exc()}" ) raise err
def run_gapic(): from helloworld import GreeterClient from helloworld import HelloRequest from helloworld import HelloReply import grpc from helloworld.services.greeter.transports import GreeterGrpcTransport from google.auth.credentials import AnonymousCredentials creds = AnonymousCredentials() # normally use creds but since we have custom CA certs.. #c = GreeterClient(credentials=cred) # load custom certs with open('CA_crt.pem', 'rb') as fh: root_ca = fh.read() channel_creds = grpc.ssl_channel_credentials(root_certificates=root_ca, ) transport = GreeterGrpcTransport( credentials=creds, ssl_channel_credentials=channel_creds, ) c = GreeterClient(transport=transport) req = HelloRequest() req.name = 'sal' resp = c.say_hello(request=req) logging.info(resp.message)
def setUpModule(): if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials emulator_project = os.getenv("GCLOUD_PROJECT", "emulator-test-project") Config.CLIENT = Client(project=emulator_project, credentials=AnonymousCredentials()) else: Config.CLIENT = Client() retry = RetryErrors(exceptions.ServiceUnavailable) configs = list(retry(Config.CLIENT.list_instance_configs)()) instances = retry(_list_instances)() EXISTING_INSTANCES[:] = instances # Delete test instances that are older than an hour. cutoff = int(time.time()) - 1 * 60 * 60 for instance_pb in Config.CLIENT.list_instances( "labels.python-spanner-dbapi-systests:true"): instance = Instance.from_pb(instance_pb, Config.CLIENT) if "created" not in instance.labels: continue create_time = int(instance.labels["created"]) if create_time > cutoff: continue # Instance cannot be deleted while backups exist. for backup_pb in instance.list_backups(): backup = Backup.from_pb(backup_pb, instance) backup.delete() instance.delete() if CREATE_INSTANCE: if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. configs = [config for config in configs if "-us-" in config.name] if not configs: raise ValueError("List instance configs failed in module set up.") Config.INSTANCE_CONFIG = configs[0] config_name = configs[0].name create_time = str(int(time.time())) labels = { "python-spanner-dbapi-systests": "true", "created": create_time } Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name, labels=labels) created_op = Config.INSTANCE.create() created_op.result( SPANNER_OPERATION_TIMEOUT_IN_SECONDS) # block until completion else: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload()
def client(): host = os.environ.get(_STORAGE_EMULATOR_ENV_VAR) client = storage.Client( project=_CONF_TEST_PROJECT_ID, credentials=AnonymousCredentials(), client_options={"api_endpoint": host}, ) return client
async def test_progress(self, monkeypatch): fake_creds = AnonymousCredentials() mock_client = mock.create_autospec( ClusterControllerClient(credentials=fake_creds)) mock_logging_client = mock.create_autospec( logging_v2.LoggingServiceV2Client(credentials=fake_creds)) spawner = DataprocSpawner(hub=Hub(), dataproc=mock_client, user=MockUser(), _mock=True, logging=mock_logging_client, gcs_notebooks=self.gcs_notebooks) spawner.project = "test-progress" async def collect(ait): items = [] async for value in ait: items.append(value) return items def create_logs(): entries = [] for i in range(5): e = LogEntry(insert_id=f'entry_{i}', json_payload=ParseDict( { 'method': 'method', 'message': f'message_{i}' }, Struct())) entries.append(e) return entries def create_expected(): progress = 5 expected = [] i = 0 for e in create_logs(): progress += math.ceil((90 - progress) / 4) expected.append({ 'progress': progress, 'message': f'method: message_{i}' }) i += 1 expected.append({'message': 'operation.done()', 'progress': 71}) return expected def test_list_log_entries(*args, **kwargs): return create_logs() op = MockOperation('op1', 'cluster1-op1') monkeypatch.setattr(mock_logging_client, 'list_log_entries', test_list_log_entries) monkeypatch.setattr(spawner, 'operation', op) _, _ = await spawner.start() assert await collect(spawner.progress()) == create_expected()
def test_minimium_cluster_definition(self, monkeypatch): """ Some keys must always be present for JupyterHub to work. """ import yaml def test_read_file(*args, **kwargs): config_string = open('./tests/test_data/minimum.yaml', 'r').read() return config_string def test_clustername(*args, **kwargs): return 'test-clustername' fake_creds = AnonymousCredentials() mock_dataproc_client = mock.create_autospec( ClusterControllerClient(credentials=fake_creds)) mock_gcs_client = mock.create_autospec( storage.Client(credentials=fake_creds, project='project')) spawner = DataprocSpawner(hub=Hub(), dataproc=mock_dataproc_client, gcs=mock_gcs_client, user=MockUser(), _mock=True, gcs_notebooks=self.gcs_notebooks) # Prevents a call to GCS. We return the local file instead. monkeypatch.setattr(spawner, "read_gcs_file", test_read_file) monkeypatch.setattr(spawner, "clustername", test_clustername) spawner.project = "test-project" spawner.zone = "test-self1-b" spawner.env_str = "test-env-str" spawner.args_str = "test-args-str" config_built = spawner._build_cluster_config() assert 'project_id' in config_built assert 'cluster_name' in config_built assert config_built['project_id'] == 'test-project' assert config_built['cluster_name'] == 'test-clustername' assert config_built['config']['gce_cluster_config']['zone_uri'].split( '/')[-1] == 'test-self1-b' assert Component['JUPYTER'].value in config_built['config'][ 'software_config']['optional_components'] assert Component['ANACONDA'].value in config_built['config'][ 'software_config']['optional_components'] assert 'dataproc:jupyter.hub.args' in config_built['config'][ 'software_config']['properties'] assert 'dataproc:jupyter.hub.enabled' in config_built['config'][ 'software_config']['properties'] # assert 'dataproc:jupyter.notebook.gcs.dir' in config_built['config']['software_config']['properties'] assert 'dataproc:jupyter.hub.env' in config_built['config'][ 'software_config']['properties']
def spanner_client(): if _helpers.USE_EMULATOR: from google.auth.credentials import AnonymousCredentials credentials = AnonymousCredentials() return spanner_v1.Client( project=_helpers.EMULATOR_PROJECT, credentials=credentials, ) else: return spanner_v1.Client() # use google.auth.default credentials
def test_constructor_emulator_host_warning(self, mock_warn, mock_em): from google.cloud.spanner_v1 import client as MUT from google.auth.credentials import AnonymousCredentials expected_scopes = None creds = _make_credentials() mock_em.return_value = "http://emulator.host.com" with mock.patch("google.cloud.spanner_v1.client.AnonymousCredentials") as patch: expected_creds = patch.return_value = AnonymousCredentials() self._constructor_test_helper(expected_scopes, creds, expected_creds) mock_warn.assert_called_once_with(MUT._EMULATOR_HOST_HTTP_SCHEME)
def connect(self, method=None): if method == 'browser': cred = self._connect_browser() elif method == "cache": cred = self._connect_cache() elif method == 'anon': cred = AnonymousCredentials() else: raise ValueError(f"Invalid connection method `{method}`.") srv = build('drive', 'v3', credentials=cred) self._drives = srv.drives() self.service = srv.files()
def test_locations(self, monkeypatch): import yaml def test_read_file(*args, **kwargs): config_string = open('./tests/test_data/basic_uri.yaml', 'r').read() return config_string def test_clustername(*args, **kwargs): return 'test-clustername' fake_creds = AnonymousCredentials() mock_dataproc_client = mock.create_autospec( ClusterControllerClient(credentials=fake_creds)) mock_gcs_client = mock.create_autospec( storage.Client(credentials=fake_creds, project='project')) spawner = DataprocSpawner(hub=Hub(), dataproc=mock_dataproc_client, gcs=mock_gcs_client, user=MockUser(), _mock=True, gcs_notebooks=self.gcs_notebooks) # Prevents a call to GCS. We return the local file instead. monkeypatch.setattr(spawner, "read_gcs_file", test_read_file) monkeypatch.setattr(spawner, "clustername", test_clustername) spawner.project = "test-project" spawner.region = "us-east1" spawner.zone = "us-east1-d" spawner.env_str = "test-env-str" spawner.args_str = "test-args-str" spawner.user_options = { 'cluster_type': 'basic_uri.yaml', 'cluster_zone': 'us-east1-d' } user_zone = spawner.user_options['cluster_zone'] user_region = user_zone[:-2] config_built = spawner._build_cluster_config() assert config_built['config']['gce_cluster_config'][ 'subnetwork_uri'].split('/')[-3] == user_region assert config_built['config']['master_config'][ 'machine_type_uri'] == 'n1-standard-4' assert config_built['config']['worker_config'][ 'machine_type_uri'] == 'n1-highmem-16' assert config_built['config']['secondary_worker_config'][ 'machine_type_uri'] == 'n1-standard-4' assert config_built['config']['master_config']['accelerators'][0][ 'accelerator_type_uri'] == 'nvidia-tesla-v100'
def get_bucket(): options = dict(project=project_id) if settings.ENVIRONMENT != 'production': options['credentials'] = AnonymousCredentials() client = storage.Client(**options) bucket = client.bucket(bucket_name) if not bucket.exists(): bucket.create(client) return bucket
def _get_mock_gcs_client(): my_http = requests.Session() my_http.verify = False # disable SSL validation urllib3.disable_warnings( urllib3.exceptions.InsecureRequestWarning ) # disable https warnings for https insecure certs return storage.Client( credentials=AnonymousCredentials(), project="test", _http=my_http, client_options=ClientOptions(api_endpoint="https://127.0.0.1:4443"), )
def __init__(self, **kwargs): # Sanity check: Is our goal to use the emulator? # If so, create a grpc insecure channel with the emulator host # as the target. if os.environ.get("PUBSUB_EMULATOR_HOST"): kwargs["client_options"] = { "api_endpoint": os.environ.get("PUBSUB_EMULATOR_HOST") } kwargs["credentials"] = AnonymousCredentials() # Instantiate the underlying GAPIC client. self._api = subscriber_client.SubscriberClient(**kwargs) self._target = self._api._transport._host
def get_blob(project: str, bucket: str, blob_name: str): # this means we're testing if os.environ.get("STORAGE_EMULATOR_HOST") is not None: client = storage.Client( credentials=AnonymousCredentials(), project=project, ) else: # pragma: no cover client = storage.Client(project=project) gcs_bucket = client.get_bucket(bucket) blob = gcs_bucket.get_blob(blob_name) return blob
def create_anonymous_client(cls): """Factory: return client with anonymous credentials. .. note:: Such a client has only limited access to "public" buckets: listing their contents and downloading their blobs. :rtype: :class:`google.cloud.storage.client.Client` :returns: Instance w/ anonymous credentials and no project. """ client = cls(project="<none>", credentials=AnonymousCredentials()) client.project = None return client
def test_cluster_definition_overrides(self, monkeypatch): """Check that config settings incompatible with JupyterHub are overwritten correctly.""" import yaml def test_read_file(*args, **kwargs): config_string = open('./tests/test_data/export.yaml', 'r').read() return config_string def test_clustername(*args, **kwargs): return 'test-clustername' fake_creds = AnonymousCredentials() mock_dataproc_client = mock.create_autospec( ClusterControllerClient(credentials=fake_creds)) mock_gcs_client = mock.create_autospec( storage.Client(credentials=fake_creds, project='project')) spawner = DataprocSpawner(hub=Hub(), dataproc=mock_dataproc_client, gcs=mock_gcs_client, user=MockUser(), _mock=True, gcs_notebooks=self.gcs_notebooks) # Prevents a call to GCS. We return the local file instead. monkeypatch.setattr(spawner, "read_gcs_file", test_read_file) monkeypatch.setattr(spawner, "clustername", test_clustername) spawner.project = "test-project" spawner.region = "us-east1" spawner.zone = "us-east1-d" spawner.env_str = "test-env-str" spawner.args_str = "test-args-str" spawner.user_options = { 'cluster_type': 'export.yaml', 'cluster_zone': 'test-form1-a' } config_built = spawner._build_cluster_config() # Verify that we disable Component Gateway (temporarily) assert config_built['config']['endpoint_config'][ 'enable_http_port_access'] == False # Verify that we disable preemptibility (temporarily) assert 'preemptibility' not in config_built['config']['master_config'] assert 'preemptibility' not in config_built['config']['worker_config'] # Verify that we removed cluster-specific namenode properties assert 'hdfs:dfs.namenode.lifeline.rpc-address' not in config_built[ 'config']['software_config']['properties'] assert 'hdfs:dfs.namenode.servicerpc-address' not in config_built[ 'config']['software_config']['properties']
def set_up(): os.environ["STORAGE_EMULATOR_HOST"] = "http://localhost:9090" client = storage.Client( credentials=AnonymousCredentials(), project="testing", ) bucket = client.bucket(BUCKET_NAME) try: bucket.delete(force=True) except: # pragma: no cover pass bucket = client.create_bucket(BUCKET_NAME)
def test_image_version_supports_component_gateway(self): fake_creds = AnonymousCredentials() mock_dataproc_client = mock.create_autospec( ClusterControllerClient(credentials=fake_creds)) mock_gcs_client = mock.create_autospec( storage.Client(credentials=fake_creds, project='project')) spawner = DataprocSpawner(hub=Hub(), dataproc=mock_dataproc_client, gcs=mock_gcs_client, user=MockUser(), _mock=True, gcs_notebooks=self.gcs_notebooks) assert spawner._validate_image_version_supports_component_gateway( '1.3') is True assert spawner._validate_image_version_supports_component_gateway( '1.3-debian9') is True assert spawner._validate_image_version_supports_component_gateway( '1.3.6-debian9') is False assert spawner._validate_image_version_supports_component_gateway( '1.3.59-debian9') is True assert spawner._validate_image_version_supports_component_gateway( '1.3.999-debian9') is True assert spawner._validate_image_version_supports_component_gateway( '1.4-debian10') is True assert spawner._validate_image_version_supports_component_gateway( '1.4.6-debian10') is False assert spawner._validate_image_version_supports_component_gateway( '1.4.31-debian10') is True assert spawner._validate_image_version_supports_component_gateway( '1.5-debian10') is True assert spawner._validate_image_version_supports_component_gateway( '1.5.0-debian10') is False assert spawner._validate_image_version_supports_component_gateway( '1.5.5-debian10') is True assert spawner._validate_image_version_supports_component_gateway( '2') is True assert spawner._validate_image_version_supports_component_gateway( '2.0') is True assert spawner._validate_image_version_supports_component_gateway( '2.0.0') is True assert spawner._validate_image_version_supports_component_gateway( '2.3.0') is True assert spawner._validate_image_version_supports_component_gateway( '2.0.0-RC1-preview') is True assert spawner._validate_image_version_supports_component_gateway( 'weird-unexpected-version-124.3.v2.2020-02-15') is True assert spawner._validate_image_version_supports_component_gateway( '1.3.weird-version-again') is True
def test_cluster_definition_keep_core_values(self, monkeypatch): """ Some system's default values must remain no matter what. """ import yaml def test_read_file(*args, **kwargs): config_string = open('./tests/test_data/basic.yaml', 'r').read() return config_string def test_clustername(*args, **kwargs): return 'test-clustername' fake_creds = AnonymousCredentials() mock_dataproc_client = mock.create_autospec( ClusterControllerClient(credentials=fake_creds)) mock_gcs_client = mock.create_autospec( storage.Client(credentials=fake_creds, project='project')) spawner = DataprocSpawner(hub=Hub(), dataproc=mock_dataproc_client, gcs=mock_gcs_client, user=MockUser(), _mock=True, gcs_notebooks=self.gcs_notebooks) # Prevents a call to GCS. We return the local file instead. monkeypatch.setattr(spawner, "read_gcs_file", test_read_file) monkeypatch.setattr(spawner, "clustername", test_clustername) spawner.project = "test-project" spawner.region = "us-east1" spawner.zone = "us-east1-d" spawner.env_str = "test-env-str" spawner.args_str = "test-args-str" spawner.user_options = { 'cluster_type': 'basic.yaml', 'cluster_zone': 'test-form1-a' } config_built = spawner._build_cluster_config() assert config_built['project_id'] == 'test-project' assert config_built['cluster_name'] == 'test-clustername' assert config_built['config']['software_config']['properties'][ 'dataproc:jupyter.hub.args'] == 'test-args-str' assert config_built['config']['software_config']['properties'][ 'dataproc:jupyter.hub.enabled'] == 'true' # assert config_built['config']['software_config']['properties']['dataproc:jupyter.notebook.gcs.dir'] == f'gs://users-notebooks/fake' assert config_built['config']['software_config']['properties'][ 'dataproc:jupyter.hub.env'] == 'test-env-str'
def __init__(self, **kwargs): # Sanity check: Is our goal to use the emulator? # If so, create a grpc insecure channel with the emulator host # as the target. if os.environ.get("PUBSUB_EMULATOR_HOST"): kwargs["channel"] = grpc.insecure_channel( target=os.environ.get("PUBSUB_EMULATOR_HOST")) kwargs["credentials"] = AnonymousCredentials() # The GAPIC client has mTLS logic to determine the api endpoint and the # ssl credentials to use. Here we create a GAPIC client to help compute the # api endpoint and ssl credentials. The api endpoint will be used to set # `self._target`, and ssl credentials will be passed to # `grpc_helpers.create_channel` to establish a mTLS channel (if ssl # credentials is not None). client_options = kwargs.get("client_options", None) credentials = kwargs.get("credentials", None) client_for_mtls_info = subscriber_client.SubscriberClient( credentials=credentials, client_options=client_options) self._target = client_for_mtls_info._transport._host # Use a custom channel. # We need this in order to set appropriate default message size and # keepalive options. if "transport" not in kwargs: channel = kwargs.pop("channel", None) if channel is None: channel = grpc_helpers.create_channel( credentials=kwargs.pop("credentials", None), target=self.target, ssl_credentials=client_for_mtls_info._transport. _ssl_channel_credentials, scopes=subscriber_client.SubscriberClient._DEFAULT_SCOPES, options={ "grpc.max_send_message_length": -1, "grpc.max_receive_message_length": -1, "grpc.keepalive_time_ms": 30000, }.items(), ) # cannot pass both 'channel' and 'credentials' kwargs.pop("credentials", None) transport = subscriber_grpc_transport.SubscriberGrpcTransport( channel=channel) kwargs["transport"] = transport # Add the metrics headers, and instantiate the underlying GAPIC # client. self._api = subscriber_client.SubscriberClient(**kwargs)
def __init__( self, project=None, credentials=None, client_info=_CLIENT_INFO, user_agent=None, client_options=None, query_options=None, ): self._emulator_host = _get_spanner_emulator_host() if client_options and type(client_options) == dict: self._client_options = google.api_core.client_options.from_dict( client_options ) else: self._client_options = client_options if self._emulator_host: credentials = AnonymousCredentials() elif isinstance(credentials, AnonymousCredentials): self._emulator_host = self._client_options.api_endpoint # NOTE: This API has no use for the _http argument, but sending it # will have no impact since the _http() @property only lazily # creates a working HTTP object. super(Client, self).__init__( project=project, credentials=credentials, client_options=client_options, _http=None, ) self._client_info = client_info env_query_options = ExecuteSqlRequest.QueryOptions( optimizer_version=_get_spanner_optimizer_version() ) # Environment flag config has higher precedence than application config. self._query_options = _merge_query_options(query_options, env_query_options) if user_agent is not None: warnings.warn(_USER_AGENT_DEPRECATED, DeprecationWarning, stacklevel=2) self.user_agent = user_agent if self._emulator_host is not None and ( "http://" in self._emulator_host or "https://" in self._emulator_host ): warnings.warn(_EMULATOR_HOST_HTTP_SCHEME)