def setup(self): # setting up client to connect with cloud spanner spanner_client = Client(self._spanner_configuration.project) instance = spanner_client.instance(self._spanner_configuration.instance) self._database = instance.database( self._spanner_configuration.database, pool=self._spanner_configuration.pool)
def setup(self): spanner_client = Client(self._spanner_configuration.project) instance = spanner_client.instance( self._spanner_configuration.instance) self._db_instance = instance.database( self._spanner_configuration.database, pool=self._spanner_configuration.pool)
def setUpModule(): if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials emulator_project = os.getenv("GCLOUD_PROJECT", "emulator-test-project") Config.CLIENT = Client(project=emulator_project, credentials=AnonymousCredentials()) else: Config.CLIENT = Client() retry = RetryErrors(exceptions.ServiceUnavailable) configs = list(retry(Config.CLIENT.list_instance_configs)()) instances = retry(_list_instances)() EXISTING_INSTANCES[:] = instances if CREATE_INSTANCE: if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. configs = [config for config in configs if "-us-" in config.name] if not configs: raise ValueError("List instance configs failed in module set up.") Config.INSTANCE_CONFIG = configs[0] config_name = configs[0].name Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name) created_op = Config.INSTANCE.create() created_op.result(30) # block until completion else: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload()
def setup(self): spanner_client = Client(self._spanner_configuration.project) instance = spanner_client.instance(self._spanner_configuration.instance) self._database = instance.database( self._spanner_configuration.database, pool=self._spanner_configuration.pool) self._snapshot = self._database.batch_snapshot( **self._spanner_configuration.snapshot_options)
def setup(self): spanner_client = Client( project=self._spanner_configuration.project, credentials=self._spanner_configuration.credentials) instance = spanner_client.instance(self._spanner_configuration.instance) self._database = instance.database( self._spanner_configuration.database, pool=self._spanner_configuration.pool) self._snapshot = self._database.batch_snapshot( **self._spanner_configuration.snapshot_options) self._snapshot_dict = self._snapshot.to_dict()
def setUpModule(): Config.CLIENT = Client() retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable) configs = list(retry(Config.CLIENT.list_instance_configs)()) # Defend against back-end returning configs for regions we aren't # actually allowed to use. configs = [config for config in configs if '-us-' in config.name] if len(configs) < 1: raise ValueError('List instance configs failed in module set up.') Config.INSTANCE_CONFIG = configs[0] config_name = configs[0].name instances = retry(_list_instances)() EXISTING_INSTANCES[:] = instances if CREATE_INSTANCE: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name) created_op = Config.INSTANCE.create() created_op.result(30) # block until completion else: Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload()
def _connect(instance_id, database_id, service_account_json, pool_size, time_out, ping_interval): """ Connect to the spanner database :type database_id: str :param instance_id: Cloud Spanner instance ID. :type database_id: str :param database_id: Cloud Spanner database ID. """ try: if path.exists(service_account_json) is False: raise RuntimeError('Service account json file not exist') # Spanner DB Connection spanner_client = Client.from_service_account_json(service_account_json) instance = spanner_client.instance(instance_id) pool = PingingPool(size=pool_size, default_timeout=time_out, ping_interval=ping_interval) Connection._db_instance = instance.database(database_id, pool=pool) Connection.start_connection_thread(pool.ping) with Connection._db_instance.snapshot() as snapshot: snapshot.execute_sql("SELECT * FROM information_schema.tables AS t WHERE t.table_schema = ''") logging.debug('Cloud Spanner Db Connected \n Instance Id: %s \n Database Id: %s', instance_id, database_id) except Exception: raise SpannerException('Fail to Connect Spanner Database')
class EmsSpannerClient: def __init__(self, project_id: str, instance_id, db_id): self.project_id = project_id self.__db = Client(project_id).instance(instance_id).database(db_id) def execute_sql(self, query: str): with self.__db.snapshot() as snapshot: results = snapshot.execute_sql(query) for row in results: yield row def execute_update(self, query: str): def run_query(transaction): transaction.execute_update(query) self.__db.run_in_transaction(run_query)
class _CreateTransactionFn(DoFn): """ A DoFn to create the transaction of cloud spanner. It connects to the database and and returns the transaction_id and session_id by using the batch_snapshot.to_dict() method available in the google cloud spanner sdk. https://googleapis.dev/python/spanner/latest/database-api.html?highlight= batch_snapshot#google.cloud.spanner_v1.database.BatchSnapshot.to_dict """ def __init__( self, project_id, instance_id, database_id, credentials, pool, read_timestamp, exact_staleness): self._project_id = project_id self._instance_id = instance_id self._database_id = database_id self._credentials = credentials self._pool = pool self._snapshot_options = {} if read_timestamp: self._snapshot_options['read_timestamp'] = read_timestamp if exact_staleness: self._snapshot_options['exact_staleness'] = exact_staleness self._snapshot = None def setup(self): self._spanner_client = Client( project=self._project_id, credentials=self._credentials) self._instance = self._spanner_client.instance(self._instance_id) self._database = self._instance.database(self._database_id, pool=self._pool) def process(self, element, *args, **kwargs): self._snapshot = self._database.batch_snapshot(**self._snapshot_options) return [_SPANNER_TRANSACTION(self._snapshot.to_dict())]
def setup(self): self._spanner_client = Client( project=self._project_id, credentials=self._credentials) self._instance = self._spanner_client.instance(self._instance_id) self._database = self._instance.database(self._database_id, pool=self._pool)
from google.cloud.spanner import Client # Import relative to the script's directory from streaming_utils import DATABASE_NAME from streaming_utils import INSTANCE_NAME from streaming_utils import print_func def remove_database(client): instance = client.instance(INSTANCE_NAME) if not instance.exists(): print_func("Instance does not exist: {}".format(INSTANCE_NAME)) return print_func("Instance exists: {}".format(INSTANCE_NAME)) instance.reload() database = instance.database(DATABASE_NAME) if not database.exists(): print_func("Database does not exist: {}".format(DATABASE_NAME)) return print_func("Dropping database: {}".format(DATABASE_NAME)) database.drop() if __name__ == "__main__": client = Client() remove_database(client)
from google.cloud.spanner import Client from google.cloud.spanner_v1.pool import TransactionPingingPool _INSTANCE_NAME = 'testing' _DATABASE_NAME = 'test' if __name__ == '__main__': client = Client() instance = client.instance(_INSTANCE_NAME) custom_pool = TransactionPingingPool(size=1000, default_timeout=5, ping_interval=300) database = instance.database(_DATABASE_NAME, pool=custom_pool)
def __init__(self, project_id: str, instance_id, db_id): self.project_id = project_id self.__db = Client(project_id).instance(instance_id).database(db_id)
from google.cloud.spanner import Client from google.cloud.spanner_v1.session import Session from faker import Faker PROJECT_NAME = 'grass-clump-479' INSTANCE_NAME = 'python-write' DATABASE_NAME = 'pythontest' client = Client(project=PROJECT_NAME) instance = client.instance(INSTANCE_NAME) database = instance.database(DATABASE_NAME) session = Session(database) session.create() transaction = session.transaction() transaction.begin() fake = Faker() for i in range(10000): transaction.insert(table='table_python', columns=( 'key', 'textcolumn', ), values=[ (i, fake.name()), ]) transaction.commit() with database.snapshot() as snapshot: results = snapshot.execute_sql('SELECT key,textcolumn from table_python') for row in results: