def setup(self):
# setting up client to connect with cloud spanner
spanner_client = Client(self._spanner_configuration.project)
instance = spanner_client.instance(self._spanner_configuration.instance)
self._database = instance.database(
self._spanner_configuration.database,
pool=self._spanner_configuration.pool)
def setup(self):
spanner_client = Client(self._spanner_configuration.project)
instance = spanner_client.instance(
self._spanner_configuration.instance)
self._db_instance = instance.database(
self._spanner_configuration.database,
pool=self._spanner_configuration.pool)
def setUpModule():
if USE_EMULATOR:
from google.auth.credentials import AnonymousCredentials
emulator_project = os.getenv("GCLOUD_PROJECT", "emulator-test-project")
Config.CLIENT = Client(project=emulator_project,
credentials=AnonymousCredentials())
else:
Config.CLIENT = Client()
retry = RetryErrors(exceptions.ServiceUnavailable)
configs = list(retry(Config.CLIENT.list_instance_configs)())
instances = retry(_list_instances)()
EXISTING_INSTANCES[:] = instances
if CREATE_INSTANCE:
if not USE_EMULATOR:
# Defend against back-end returning configs for regions we aren't
# actually allowed to use.
configs = [config for config in configs if "-us-" in config.name]
if not configs:
raise ValueError("List instance configs failed in module set up.")
Config.INSTANCE_CONFIG = configs[0]
config_name = configs[0].name
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name)
created_op = Config.INSTANCE.create()
created_op.result(30) # block until completion
else:
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID)
Config.INSTANCE.reload()
def setup(self):
spanner_client = Client(self._spanner_configuration.project)
instance = spanner_client.instance(self._spanner_configuration.instance)
self._database = instance.database(
self._spanner_configuration.database,
pool=self._spanner_configuration.pool)
self._snapshot = self._database.batch_snapshot(
**self._spanner_configuration.snapshot_options)
def setup(self):
spanner_client = Client(
project=self._spanner_configuration.project,
credentials=self._spanner_configuration.credentials)
instance = spanner_client.instance(self._spanner_configuration.instance)
self._database = instance.database(
self._spanner_configuration.database,
pool=self._spanner_configuration.pool)
self._snapshot = self._database.batch_snapshot(
**self._spanner_configuration.snapshot_options)
self._snapshot_dict = self._snapshot.to_dict()
def setUpModule():
Config.CLIENT = Client()
retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable)
configs = list(retry(Config.CLIENT.list_instance_configs)())
# Defend against back-end returning configs for regions we aren't
# actually allowed to use.
configs = [config for config in configs if '-us-' in config.name]
if len(configs) < 1:
raise ValueError('List instance configs failed in module set up.')
Config.INSTANCE_CONFIG = configs[0]
config_name = configs[0].name
instances = retry(_list_instances)()
EXISTING_INSTANCES[:] = instances
if CREATE_INSTANCE:
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, config_name)
created_op = Config.INSTANCE.create()
created_op.result(30) # block until completion
else:
Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID)
Config.INSTANCE.reload()
def _connect(instance_id, database_id, service_account_json, pool_size, time_out, ping_interval):
"""
Connect to the spanner database
:type database_id: str
:param instance_id: Cloud Spanner instance ID.
:type database_id: str
:param database_id: Cloud Spanner database ID.
"""
try:
if path.exists(service_account_json) is False:
raise RuntimeError('Service account json file not exist')
# Spanner DB Connection
spanner_client = Client.from_service_account_json(service_account_json)
instance = spanner_client.instance(instance_id)
pool = PingingPool(size=pool_size, default_timeout=time_out, ping_interval=ping_interval)
Connection._db_instance = instance.database(database_id, pool=pool)
Connection.start_connection_thread(pool.ping)
with Connection._db_instance.snapshot() as snapshot:
snapshot.execute_sql("SELECT * FROM information_schema.tables AS t WHERE t.table_schema = ''")
logging.debug('Cloud Spanner Db Connected \n Instance Id: %s \n Database Id: %s', instance_id,
database_id)
except Exception:
raise SpannerException('Fail to Connect Spanner Database')
class EmsSpannerClient:
def __init__(self, project_id: str, instance_id, db_id):
self.project_id = project_id
self.__db = Client(project_id).instance(instance_id).database(db_id)
def execute_sql(self, query: str):
with self.__db.snapshot() as snapshot:
results = snapshot.execute_sql(query)
for row in results:
yield row
def execute_update(self, query: str):
def run_query(transaction):
transaction.execute_update(query)
self.__db.run_in_transaction(run_query)
class _CreateTransactionFn(DoFn):
"""
A DoFn to create the transaction of cloud spanner.
It connects to the database and and returns the transaction_id and session_id
by using the batch_snapshot.to_dict() method available in the google cloud
spanner sdk.
https://googleapis.dev/python/spanner/latest/database-api.html?highlight=
batch_snapshot#google.cloud.spanner_v1.database.BatchSnapshot.to_dict
"""
def __init__(
self,
project_id,
instance_id,
database_id,
credentials,
pool,
read_timestamp,
exact_staleness):
self._project_id = project_id
self._instance_id = instance_id
self._database_id = database_id
self._credentials = credentials
self._pool = pool
self._snapshot_options = {}
if read_timestamp:
self._snapshot_options['read_timestamp'] = read_timestamp
if exact_staleness:
self._snapshot_options['exact_staleness'] = exact_staleness
self._snapshot = None
def setup(self):
self._spanner_client = Client(
project=self._project_id, credentials=self._credentials)
self._instance = self._spanner_client.instance(self._instance_id)
self._database = self._instance.database(self._database_id, pool=self._pool)
def process(self, element, *args, **kwargs):
self._snapshot = self._database.batch_snapshot(**self._snapshot_options)
return [_SPANNER_TRANSACTION(self._snapshot.to_dict())]
def setup(self):
self._spanner_client = Client(
project=self._project_id, credentials=self._credentials)
self._instance = self._spanner_client.instance(self._instance_id)
self._database = self._instance.database(self._database_id, pool=self._pool)
from google.cloud.spanner import Client
# Import relative to the script's directory
from streaming_utils import DATABASE_NAME
from streaming_utils import INSTANCE_NAME
from streaming_utils import print_func
def remove_database(client):
instance = client.instance(INSTANCE_NAME)
if not instance.exists():
print_func("Instance does not exist: {}".format(INSTANCE_NAME))
return
print_func("Instance exists: {}".format(INSTANCE_NAME))
instance.reload()
database = instance.database(DATABASE_NAME)
if not database.exists():
print_func("Database does not exist: {}".format(DATABASE_NAME))
return
print_func("Dropping database: {}".format(DATABASE_NAME))
database.drop()
if __name__ == "__main__":
client = Client()
remove_database(client)
from google.cloud.spanner import Client
from google.cloud.spanner_v1.pool import TransactionPingingPool
_INSTANCE_NAME = 'testing'
_DATABASE_NAME = 'test'
if __name__ == '__main__':
client = Client()
instance = client.instance(_INSTANCE_NAME)
custom_pool = TransactionPingingPool(size=1000,
default_timeout=5,
ping_interval=300)
database = instance.database(_DATABASE_NAME, pool=custom_pool)
def __init__(self, project_id: str, instance_id, db_id):
self.project_id = project_id
self.__db = Client(project_id).instance(instance_id).database(db_id)
from google.cloud.spanner import Client
from google.cloud.spanner_v1.session import Session
from faker import Faker
PROJECT_NAME = 'grass-clump-479'
INSTANCE_NAME = 'python-write'
DATABASE_NAME = 'pythontest'
client = Client(project=PROJECT_NAME)
instance = client.instance(INSTANCE_NAME)
database = instance.database(DATABASE_NAME)
session = Session(database)
session.create()
transaction = session.transaction()
transaction.begin()
fake = Faker()
for i in range(10000):
transaction.insert(table='table_python',
columns=(
'key',
'textcolumn',
),
values=[
(i, fake.name()),
])
transaction.commit()
with database.snapshot() as snapshot:
results = snapshot.execute_sql('SELECT key,textcolumn from table_python')
for row in results:
