def test_scale_bigtable(instance):
bigtable_client = bigtable.Client(admin=True)
instance = bigtable_client.instance(BIGTABLE_INSTANCE)
instance.reload()
cluster = instance.cluster(BIGTABLE_INSTANCE)
_nonzero_node_count = RetryInstanceState(
instance_predicate=lambda c: c.serve_nodes > 0,
max_tries=10,
)
_nonzero_node_count(cluster.reload)()
original_node_count = cluster.serve_nodes
scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True)
expected_count = original_node_count + SIZE_CHANGE_STEP
_scaled_node_count = RetryInstanceState(
instance_predicate=lambda c: c.serve_nodes == expected_count,
max_tries=10,
)
_scaled_node_count(cluster.reload)()
scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, False)
_restored_node_count = RetryInstanceState(
instance_predicate=lambda c: c.serve_nodes == original_node_count,
max_tries=10,
)
_restored_node_count(cluster.reload)()
def test_load_table_from_local_avro_file_then_dump_table(self):
TABLE_NAME = 'test_table_avro'
ROWS = [("violet", 400), ("indigo", 445), ("blue", 475),
("green", 510), ("yellow", 570), ("orange", 590), ("red", 650)]
dataset = Config.CLIENT.dataset(
_make_dataset_name('load_local_then_dump'))
retry_403(dataset.create)()
self.to_delete.append(dataset)
table = dataset.table(TABLE_NAME)
self.to_delete.insert(0, table)
with open(os.path.join(WHERE, 'data', 'colors.avro'), 'rb') as avrof:
job = table.upload_from_file(avrof,
source_format='AVRO',
write_disposition='WRITE_TRUNCATE')
def _job_done(instance):
return instance.state.lower() == 'done'
# Retry until done.
retry = RetryInstanceState(_job_done, max_tries=8)
retry(job.reload)()
self.assertEqual(job.output_rows, len(ROWS))
# Reload table to get the schema before fetching the rows.
table.reload()
rows = self._fetch_single_page(table)
by_wavelength = operator.itemgetter(1)
self.assertEqual(sorted(rows, key=by_wavelength),
sorted(ROWS, key=by_wavelength))
def _set_up_table(self, row_count):
from google.cloud.spanner import KeySet
def _row_data(max_index):
for index in range(max_index):
yield [
index,
'First%09d' % (index, ),
'Last09%d' % (index),
'*****@*****.**' % (index, )
]
keyset = KeySet(all_=True)
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.transaction() as transaction:
transaction.delete(self.TABLE, keyset)
transaction.insert(self.TABLE, self.COLUMNS, _row_data(row_count))
return session, keyset, transaction.committed
def _load_table_for_dml(self, rows, dataset_name, table_name):
from google.cloud._testing import _NamedTemporaryFile
dataset = Config.CLIENT.dataset(dataset_name)
retry_403(dataset.create)()
self.to_delete.append(dataset)
greeting = bigquery.SchemaField('greeting', 'STRING', mode='NULLABLE')
table = dataset.table(table_name, schema=[greeting])
table.create()
self.to_delete.insert(0, table)
with _NamedTemporaryFile() as temp:
with open(temp.name, 'w') as csv_write:
writer = csv.writer(csv_write)
writer.writerow(('Greeting', ))
writer.writerows(rows)
with open(temp.name, 'rb') as csv_read:
job = table.upload_from_file(
csv_read,
source_format='CSV',
skip_leading_rows=1,
create_disposition='CREATE_NEVER',
write_disposition='WRITE_EMPTY',
)
# Retry until done.
retry = RetryInstanceState(_job_done, max_tries=8)
retry(job.reload)()
self._fetch_single_page(table)
def test_job_cancel(self):
DATASET_NAME = _make_dataset_name('job_cancel')
JOB_NAME = 'fetch_' + DATASET_NAME
TABLE_NAME = 'test_table'
QUERY = 'SELECT * FROM %s.%s' % (DATASET_NAME, TABLE_NAME)
dataset = Config.CLIENT.dataset(DATASET_NAME)
retry_403(dataset.create)()
self.to_delete.append(dataset)
full_name = bigquery.SchemaField('full_name', 'STRING',
mode='REQUIRED')
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
table = dataset.table(TABLE_NAME, schema=[full_name, age])
table.create()
self.to_delete.insert(0, table)
job = Config.CLIENT.run_async_query(JOB_NAME, QUERY)
job.begin()
job.cancel()
def _job_done(instance):
return instance.state in ('DONE', 'done')
retry = RetryInstanceState(_job_done, max_tries=8)
retry(job.reload)()
def test_transaction_read_and_insert_or_update_then_commit(self):
from google.cloud.spanner import KeySet
keyset = KeySet(all_=True)
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.delete(self.TABLE, keyset)
with session.transaction() as transaction:
rows = list(transaction.read(self.TABLE, self.COLUMNS, keyset))
self.assertEqual(rows, [])
transaction.insert_or_update(
self.TABLE, self.COLUMNS, self.ROW_DATA)
# Inserted rows can't be read until after commit.
rows = list(transaction.read(self.TABLE, self.COLUMNS, keyset))
self.assertEqual(rows, [])
rows = list(session.read(self.TABLE, self.COLUMNS, keyset))
self._check_row_data(rows)
def test_transaction_read_and_insert_then_rollback(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.delete(self.TABLE, self.ALL)
transaction = session.transaction()
transaction.begin()
rows = list(transaction.read(self.TABLE, self.COLUMNS, self.ALL))
self.assertEqual(rows, [])
transaction.insert(self.TABLE, self.COLUMNS, self.ROW_DATA)
# Inserted rows can't be read until after commit.
rows = list(transaction.read(self.TABLE, self.COLUMNS, self.ALL))
self.assertEqual(rows, [])
transaction.rollback()
rows = list(session.read(self.TABLE, self.COLUMNS, self.ALL))
self.assertEqual(rows, [])
def test_transaction_read_w_abort(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
trigger = _ReadAbortTrigger()
with session.batch() as batch:
batch.delete(COUNTERS_TABLE, self.ALL)
batch.insert(COUNTERS_TABLE, COUNTERS_COLUMNS,
[[trigger.KEY1, 0], [trigger.KEY2, 0]])
provoker = threading.Thread(target=trigger.provoke_abort,
args=(self._db, ))
handler = threading.Thread(target=trigger.handle_abort,
args=(self._db, ))
provoker.start()
trigger.provoker_started.wait()
handler.start()
trigger.handler_done.wait()
provoker.join()
handler.join()
rows = list(session.read(COUNTERS_TABLE, COUNTERS_COLUMNS, self.ALL))
self._check_row_data(rows,
expected=[[trigger.KEY1, 1], [trigger.KEY2, 1]])
def test_db_batch_insert_then_db_snapshot_read(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
with self._db.batch() as batch:
batch.delete(self.TABLE, self.ALL)
batch.insert(self.TABLE, self.COLUMNS, self.ROW_DATA)
with self._db.snapshot(read_timestamp=batch.committed) as snapshot:
from_snap = list(snapshot.read(self.TABLE, self.COLUMNS, self.ALL))
self._check_row_data(from_snap)
def test_load_table_from_local_file_then_dump_table(self):
import csv
from google.cloud._testing import _NamedTemporaryFile
ROWS = [
('Phred Phlyntstone', 32),
('Bharney Rhubble', 33),
('Wylma Phlyntstone', 29),
('Bhettye Rhubble', 27),
]
TABLE_NAME = 'test_table'
dataset = Config.CLIENT.dataset(
_make_dataset_name('load_local_then_dump'))
retry_403(dataset.create)()
self.to_delete.append(dataset)
full_name = bigquery.SchemaField('full_name', 'STRING',
mode='REQUIRED')
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
table = dataset.table(TABLE_NAME, schema=[full_name, age])
table.create()
self.to_delete.insert(0, table)
with _NamedTemporaryFile() as temp:
with open(temp.name, 'w') as csv_write:
writer = csv.writer(csv_write)
writer.writerow(('Full Name', 'Age'))
writer.writerows(ROWS)
with open(temp.name, 'rb') as csv_read:
job = table.upload_from_file(
csv_read,
source_format='CSV',
skip_leading_rows=1,
create_disposition='CREATE_NEVER',
write_disposition='WRITE_EMPTY',
)
def _job_done(instance):
return instance.state.lower() == 'done'
# Retry until done.
retry = RetryInstanceState(_job_done, max_tries=8)
retry(job.reload)()
self.assertEqual(job.output_rows, len(ROWS))
rows = self._fetch_single_page(table)
by_age = operator.itemgetter(1)
self.assertEqual(sorted(rows, key=by_age),
sorted(ROWS, key=by_age))
def test_batch_insert_or_update_then_query(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.insert_or_update(self.TABLE, self.COLUMNS, self.ROW_DATA)
snapshot = session.snapshot(read_timestamp=batch.committed)
rows = list(snapshot.execute_sql(self.SQL))
self._check_row_data(rows)
def _set_up_table(self, row_count):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.transaction() as transaction:
transaction.delete(self.TABLE, self.ALL)
transaction.insert(self.TABLE, self.COLUMNS,
self._row_data(row_count))
return session, transaction.committed
def test_scale_bigtable(instance):
bigtable_client = bigtable.Client(admin=True)
instance = bigtable_client.instance(BIGTABLE_INSTANCE)
instance.reload()
cluster = instance.cluster(BIGTABLE_INSTANCE)
_nonzero_node_count = RetryInstanceState(
instance_predicate=lambda c: c.serve_nodes > 0,
max_tries=10,
)
_nonzero_node_count(cluster.reload)()
original_node_count = cluster.serve_nodes
scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, True)
scaled_node_count_predicate = ClusterNodeCountPredicate(
original_node_count + SIZE_CHANGE_STEP)
scaled_node_count_predicate.__name__ = "scaled_node_count_predicate"
_scaled_node_count = RetryInstanceState(
instance_predicate=scaled_node_count_predicate,
max_tries=10,
)
_scaled_node_count(cluster.reload)()
scale_bigtable(BIGTABLE_INSTANCE, BIGTABLE_INSTANCE, False)
restored_node_count_predicate = ClusterNodeCountPredicate(
original_node_count)
restored_node_count_predicate.__name__ = "restored_node_count_predicate"
_restored_node_count = RetryInstanceState(
instance_predicate=restored_node_count_predicate,
max_tries=10,
)
_restored_node_count(cluster.reload)()
def test_db_run_in_transaction_twice(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
with self._db.batch() as batch:
batch.delete(self.TABLE, self.ALL)
def _unit_of_work(transaction, test):
transaction.insert_or_update(test.TABLE, test.COLUMNS,
test.ROW_DATA)
self._db.run_in_transaction(_unit_of_work, test=self)
self._db.run_in_transaction(_unit_of_work, test=self)
rows = list(self._db.execute_sql(self.SQL))
self._check_row_data(rows)
def test_batch_insert_then_read(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
batch = session.batch()
batch.delete(self.TABLE, self.ALL)
batch.insert(self.TABLE, self.COLUMNS, self.ROW_DATA)
batch.commit()
snapshot = session.snapshot(read_timestamp=batch.committed)
rows = list(snapshot.read(self.TABLE, self.COLUMNS, self.ALL))
self._check_row_data(rows)
def test_message_pull_mode_e2e(self):
import operator
TOPIC_NAME = 'message-e2e' + unique_resource_id('-')
topic = Config.CLIENT.topic(TOPIC_NAME,
timestamp_messages=True)
self.assertFalse(topic.exists())
topic.create()
self.to_delete.append(topic)
SUBSCRIPTION_NAME = 'subscribing-now' + unique_resource_id('-')
subscription = topic.subscription(SUBSCRIPTION_NAME)
self.assertFalse(subscription.exists())
subscription.create()
self.to_delete.append(subscription)
MESSAGE_1 = b'MESSAGE ONE'
MESSAGE_2 = b'MESSAGE ONE'
EXTRA_1 = 'EXTRA 1'
EXTRA_2 = 'EXTRA 2'
topic.publish(MESSAGE_1, extra=EXTRA_1)
topic.publish(MESSAGE_2, extra=EXTRA_2)
class Hoover(object):
def __init__(self):
self.received = []
def done(self, *dummy):
return len(self.received) == 2
def suction(self):
with subscription.auto_ack(max_messages=2) as ack:
self.received.extend(ack.values())
hoover = Hoover()
retry = RetryInstanceState(hoover.done)
retry(hoover.suction)()
message1, message2 = sorted(hoover.received,
key=operator.attrgetter('timestamp'))
self.assertEqual(message1.data, MESSAGE_1)
self.assertEqual(message1.attributes['extra'], EXTRA_1)
self.assertIsNotNone(message1.service_timestamp)
self.assertEqual(message2.data, MESSAGE_2)
self.assertEqual(message2.attributes['extra'], EXTRA_2)
self.assertIsNotNone(message2.service_timestamp)
def test_transaction_read_and_insert_then_exception(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.delete(self.TABLE, self.ALL)
with self.assertRaises(CustomException):
session.run_in_transaction(self._transaction_read_then_raise)
# Transaction was rolled back.
rows = list(session.read(self.TABLE, self.COLUMNS, self.ALL))
self.assertEqual(rows, [])
def _transaction_concurrency_helper(self, unit_of_work, pkey):
INITIAL_VALUE = 123
NUM_THREADS = 3 # conforms to equivalent Java systest.
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.insert_or_update(
self.COUNTERS_TABLE,
self.COUNTERS_COLUMNS,
[[pkey, INITIAL_VALUE]])
# We don't want to run the threads' transactions in the current
# session, which would fail.
txn_sessions = []
for _ in range(NUM_THREADS):
txn_session = self._db.session()
txn_sessions.append(txn_session)
txn_session.create()
self.to_delete.append(txn_session)
threads = [
threading.Thread(
target=txn_session.run_in_transaction,
args=(unit_of_work, pkey))
for txn_session in txn_sessions]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
keyset = KeySet(keys=[(pkey,)])
rows = list(session.read(
self.COUNTERS_TABLE, self.COUNTERS_COLUMNS, keyset))
self.assertEqual(len(rows), 1)
_, value = rows[0]
self.assertEqual(value, INITIAL_VALUE + len(threads))
def test_batch_insert_then_read_all_datatypes(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.delete(self.ALL_TYPES_TABLE, self.ALL)
batch.insert(self.ALL_TYPES_TABLE, self.ALL_TYPES_COLUMNS,
self.ALL_TYPES_ROWDATA)
snapshot = session.snapshot(read_timestamp=batch.committed)
rows = list(
snapshot.read(self.ALL_TYPES_TABLE, self.ALL_TYPES_COLUMNS,
self.ALL))
self._check_row_data(rows, expected=self.ALL_TYPES_ROWDATA)
def test_batch_insert_then_read(self):
from google.cloud.spanner import KeySet
keyset = KeySet(all_=True)
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
batch = session.batch()
batch.delete(self.TABLE, keyset)
batch.insert(self.TABLE, self.COLUMNS, self.ROW_DATA)
batch.commit()
snapshot = session.snapshot(read_timestamp=batch.committed)
rows = list(snapshot.read(self.TABLE, self.COLUMNS, keyset))
self._check_row_data(rows)
def test_db_run_in_transaction_then_snapshot_execute_sql(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
with self._db.batch() as batch:
batch.delete(self.TABLE, self.ALL)
def _unit_of_work(transaction, test):
rows = list(transaction.read(test.TABLE, test.COLUMNS, self.ALL))
test.assertEqual(rows, [])
transaction.insert_or_update(test.TABLE, test.COLUMNS,
test.ROW_DATA)
self._db.run_in_transaction(_unit_of_work, test=self)
with self._db.snapshot() as after:
rows = list(after.execute_sql(self.SQL))
self._check_row_data(rows)
def _set_up_table(self, row_count, db=None):
if db is None:
db = self._db
retry = RetryInstanceState(_has_all_ddl)
retry(db.reload)()
session = db.session()
session.create()
self.to_delete.append(session)
def _unit_of_work(transaction, test):
transaction.delete(test.TABLE, test.ALL)
transaction.insert(test.TABLE, test.COLUMNS,
test._row_data(row_count))
committed = session.run_in_transaction(_unit_of_work, test=self)
return session, committed
def test_db_run_in_transaction_twice_4181(self):
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
with self._db.batch() as batch:
batch.delete(COUNTERS_TABLE, self.ALL)
def _unit_of_work(transaction, name):
transaction.insert(COUNTERS_TABLE, COUNTERS_COLUMNS, [[name, 0]])
self._db.run_in_transaction(_unit_of_work, name='id_1')
with self.assertRaises(errors.RetryError) as expected:
self._db.run_in_transaction(_unit_of_work, name='id_1')
self.assertEqual(exc_to_code(expected.exception.cause),
StatusCode.ALREADY_EXISTS)
self._db.run_in_transaction(_unit_of_work, name='id_2')
with self._db.snapshot() as after:
rows = list(after.read(COUNTERS_TABLE, COUNTERS_COLUMNS, self.ALL))
self.assertEqual(len(rows), 2)
def test_batch_insert_then_read_string_array_of_string(self):
TABLE = 'string_plus_array_of_string'
COLUMNS = ['id', 'name', 'tags']
ROWDATA = [
(0, None, None),
(1, 'phred', ['yabba', 'dabba', 'do']),
(2, 'bharney', []),
(3, 'wylma', ['oh', None, 'phred']),
]
retry = RetryInstanceState(_has_all_ddl)
retry(self._db.reload)()
session = self._db.session()
session.create()
self.to_delete.append(session)
with session.batch() as batch:
batch.delete(TABLE, self.ALL)
batch.insert(TABLE, COLUMNS, ROWDATA)
snapshot = session.snapshot(read_timestamp=batch.committed)
rows = list(snapshot.read(TABLE, COLUMNS, self.ALL))
self._check_row_data(rows, expected=ROWDATA)
"""Predicate: pass only exceptions for a failed copyTo."""
err_msg = bad_request.message
return err_msg.startswith(
"No file found in request. (POST") and "copyTo" in err_msg
def _no_event_based_hold(blob):
return not blob.event_based_hold
def _has_kms_key_name(blob):
return blob.kms_key_name is not None
retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold)
retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name)
def unique_name(prefix):
return prefix + unique_resource_id("-")
def empty_bucket(bucket):
for blob in list(bucket.list_blobs(versions=True)):
try:
blob.delete()
except exceptions.NotFound:
pass
signing_blob_content = b"This time for sure, Rocky!"
def _bad_copy(bad_request):
"""Predicate: pass only exceptions for a failed copyTo."""
err_msg = bad_request.message
return err_msg.startswith(
"No file found in request. (POST") and "copyTo" in err_msg
def _no_event_based_hold(blob):
return not blob.event_based_hold
retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy)
retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold)
def unique_name(prefix):
return prefix + unique_resource_id("-")
def empty_bucket(bucket):
for blob in list(bucket.list_blobs(versions=True)):
try:
blob.delete()
except exceptions.NotFound:
pass
def delete_blob(blob):
def test_load_table_from_storage_then_dump_table(self):
import csv
from google.cloud._testing import _NamedTemporaryFile
from google.cloud.storage import Client as StorageClient
local_id = unique_resource_id()
BUCKET_NAME = 'bq_load_test' + local_id
BLOB_NAME = 'person_ages.csv'
GS_URL = 'gs://%s/%s' % (BUCKET_NAME, BLOB_NAME)
ROWS = [
('Phred Phlyntstone', 32),
('Bharney Rhubble', 33),
('Wylma Phlyntstone', 29),
('Bhettye Rhubble', 27),
]
TABLE_NAME = 'test_table'
s_client = StorageClient()
# In the **very** rare case the bucket name is reserved, this
# fails with a ConnectionError.
bucket = s_client.create_bucket(BUCKET_NAME)
self.to_delete.append(bucket)
blob = bucket.blob(BLOB_NAME)
with _NamedTemporaryFile() as temp:
with open(temp.name, 'w') as csv_write:
writer = csv.writer(csv_write)
writer.writerow(('Full Name', 'Age'))
writer.writerows(ROWS)
with open(temp.name, 'rb') as csv_read:
blob.upload_from_file(csv_read, content_type='text/csv')
self.to_delete.insert(0, blob)
dataset = Config.CLIENT.dataset(
_make_dataset_name('load_gcs_then_dump'))
retry_403(dataset.create)()
self.to_delete.append(dataset)
full_name = bigquery.SchemaField('full_name', 'STRING',
mode='REQUIRED')
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
table = dataset.table(TABLE_NAME, schema=[full_name, age])
table.create()
self.to_delete.insert(0, table)
job = Config.CLIENT.load_table_from_storage(
'bq_load_storage_test_' + local_id, table, GS_URL)
job.create_disposition = 'CREATE_NEVER'
job.skip_leading_rows = 1
job.source_format = 'CSV'
job.write_disposition = 'WRITE_EMPTY'
job.begin()
def _job_done(instance):
return instance.state in ('DONE', 'done')
# Allow for 90 seconds of "warm up" before rows visible. See:
# https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataavailability
# 8 tries -> 1 + 2 + 4 + 8 + 16 + 32 + 64 = 127 seconds
retry = RetryInstanceState(_job_done, max_tries=8)
retry(job.reload)()
rows = self._fetch_single_page(table)
by_age = operator.itemgetter(1)
self.assertEqual(sorted(rows, key=by_age),
sorted(ROWS, key=by_age))
def test_load_table_from_storage_w_autodetect_schema(self):
from google.cloud._testing import _NamedTemporaryFile
from google.cloud.storage import Client as StorageClient
from google.cloud.bigquery import SchemaField
local_id = unique_resource_id()
bucket_name = 'bq_load_test' + local_id
blob_name = 'person_ages.csv'
gs_url = 'gs://{}/{}'.format(bucket_name, blob_name)
rows = [
('Phred Phlyntstone', 32),
('Bharney Rhubble', 33),
('Wylma Phlyntstone', 29),
('Bhettye Rhubble', 27),
] * 100 # BigQuery internally uses the first 100 rows to detect schema
table_name = 'test_table'
storage_client = StorageClient()
# In the **very** rare case the bucket name is reserved, this
# fails with a ConnectionError.
bucket = storage_client.create_bucket(bucket_name)
self.to_delete.append(bucket)
blob = bucket.blob(blob_name)
with _NamedTemporaryFile() as temp:
with open(temp.name, 'w') as csv_write:
writer = csv.writer(csv_write)
writer.writerow(('Full Name', 'Age'))
writer.writerows(rows)
with open(temp.name, 'rb') as csv_read:
blob.upload_from_file(csv_read, content_type='text/csv')
self.to_delete.insert(0, blob)
dataset = Config.CLIENT.dataset(
_make_dataset_name('load_gcs_then_dump'))
retry_403(dataset.create)()
self.to_delete.append(dataset)
table = dataset.table(table_name)
self.to_delete.insert(0, table)
job = Config.CLIENT.load_table_from_storage(
'bq_load_storage_test_' + local_id, table, gs_url)
job.autodetect = True
job.begin()
# Allow for 90 seconds of "warm up" before rows visible. See
# https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataavailability
# 8 tries -> 1 + 2 + 4 + 8 + 16 + 32 + 64 = 127 seconds
retry = RetryInstanceState(_job_done, max_tries=8)
retry(job.reload)()
table.reload()
field_name = SchemaField(u'Full_Name', u'string', u'NULLABLE', None,
())
field_age = SchemaField(u'Age', u'integer', u'NULLABLE', None, ())
self.assertEqual(table.schema, [field_name, field_age])
actual_rows = self._fetch_single_page(table)
by_age = operator.itemgetter(1)
self.assertEqual(sorted(actual_rows, key=by_age),
sorted(rows, key=by_age))