def test_load_table_from_file(client, to_delete):
"""Upload table data from a CSV file."""
DATASET_ID = 'table_upload_from_file_dataset_{}'.format(_millis())
TABLE_ID = 'table_upload_from_file_table_{}'.format(_millis())
dataset = bigquery.Dataset(client.dataset(DATASET_ID))
client.create_dataset(dataset)
to_delete.append(dataset)
# [START load_table_from_file]
csv_file = six.BytesIO(b"""full_name,age
Phred Phlyntstone,32
Wylma Phlyntstone,29
""")
table_ref = dataset.table(TABLE_ID)
job_config = bigquery.LoadJobConfig()
job_config.source_format = 'CSV'
job_config.skip_leading_rows = 1
job_config.autodetect = True
job = client.load_table_from_file(csv_file,
table_ref,
job_config=job_config) # API request
job.result() # Waits for table load to complete.
# [END load_table_from_file]
table = client.get_table(table_ref)
to_delete.insert(0, table)
found_rows = []
def do_something(row):
found_rows.append(row)
# [START table_list_rows]
for row in client.list_rows(table): # API request
do_something(row)
# [END table_list_rows]
assert len(found_rows) == 2
# [START table_list_rows_iterator_properties]
iterator = client.list_rows(table) # API request
page = six.next(iterator.pages)
rows = list(page)
total = iterator.total_rows
token = iterator.next_page_token
# [END table_list_rows_iterator_properties]
assert len(rows) == total == 2
assert token is None
# Order is not preserved, so compare individually
row1 = bigquery.Row(('Wylma Phlyntstone', 29), {'full_name': 0, 'age': 1})
assert row1 in rows
row2 = bigquery.Row(('Phred Phlyntstone', 32), {'full_name': 0, 'age': 1})
assert row2 in rows
def test_期待しているテーブル結果の量に満たない場合マークがマイナスになる差分が出る(self):
from google.cloud import bigquery
client = bigquery.Client()
expected = Table(
str(Path(__file__).parent / "testdata/test5.json"),
[
{"name": "name", "type": "STRING", "mode": "NULLABLE"},
{"name": "category", "type": "STRING", "mode": "NULLABLE"},
{"name": "value", "type": "INT64", "mode": "NULLABLE"},
],
"EXPECTED",
)
input_tables = [
Table(
str(Path(__file__).parent / "testdata/test3.json"),
[
{"name": "name", "type": "STRING", "mode": "NULLABLE"},
{"name": "category", "type": "STRING", "mode": "NULLABLE"},
{"name": "value", "type": "INT64", "mode": "NULLABLE"},
],
"INPUT_DATA",
)
]
query = Query("ACTUAL", """SELECT * FROM abc""", [], {"abc": "INPUT_DATA"})
qlt = QueryLogicTest(client, expected, input_tables, query)
success, records = qlt.run()
assert not success
assert records == [
bigquery.Row(
("-", "eee", "fff", 500, 3),
{"mark": 0, "name": 1, "category": 2, "value": 3, "n": 4},
)
]
def test_load_table_file(capsys, random_table_id, client):
samples_test_dir = os.path.abspath(os.path.dirname(__file__))
file_path = os.path.join(samples_test_dir, "..", "..", "tests", "data",
"people.csv")
table = load_table_file.load_table_file(file_path, random_table_id)
out, _ = capsys.readouterr()
assert "Loaded 2 rows and 2 columns" in out
rows = list(client.list_rows(table)) # Make an API request.
assert len(rows) == 2
# Order is not preserved, so compare individually
row1 = bigquery.Row(("Wylma Phlyntstone", 29), {"full_name": 0, "age": 1})
assert row1 in rows
row2 = bigquery.Row(("Phred Phlyntstone", 32), {"full_name": 0, "age": 1})
assert row2 in rows
def create_fake_bigquery_row(self,
dau=123,
submission_date=date(2020, 5, 28),
**kwargs):
data = {'dau': dau, 'submission_date': submission_date, **kwargs}
return bigquery.Row(
list(data.values()),
{key: idx
for idx, key in enumerate(data.keys())},
)
import http
import json
import socket
import unittest
import unittest.mock as mock
from google.cloud import bigquery
from googleapiclient import errors
from parameterized import parameterized
import constants
import main
from models import failure
from models import process_result
DUMMY_ROWS = [bigquery.Row(['0001'], {'item_id': 0})]
DUMMY_START_INDEX = 0
DUMMY_BATCH_SIZE = 1000
DUMMY_TIMESTAMP = '0001-01-01:00:00:00'
DUMMY_REQUEST_BODY = json.dumps({
'start_index': DUMMY_START_INDEX,
'batch_size': DUMMY_BATCH_SIZE,
'timestamp': DUMMY_TIMESTAMP,
})
DUMMY_SUCCESSES = ['0001', '0002', '0003']
DUMMY_FAILURES = [failure.Failure('0004', 'Error message')]
DUMMY_SKIPPED = ['0005']
INSERT_URL = '/insert_items'
DELETE_URL = '/delete_items'
PREVENT_EXPIRING_URL = '/prevent_expiring_items'
def create_bigquery_row(self, data):
return bigquery.Row(
list(data.values()),
{key: idx for idx, key in enumerate(data.keys())},
)
def generate_test_data(
method: constants.Method,
num_rows=1,
remove_merchant_id=False
) -> Tuple[List[bigquery.Row], constants.Batch, constants.BatchIdToItemId,
Dict[str, Any]]:
"""Generates a tuple containing a triplet of matching row, batch, response.
Args:
method: The API method for this batch (insert, delete)
num_rows: The number of rows to generate
remove_merchant_id: If true, set merchant_id to None
Returns:
A tuple containing row_data, batch, and response. The row_data represents
data pulled directly from BigQuery in as Row objects. The batch data is a
Content API request batch corresponding to the Row data but in JSON object
form. The response is a JSON object which represents the expected response
from the Content API call.
"""
rows = []
batch = {'entries': []}
batch_id_to_item_id = dict()
response = {
u'kind': u'content#productsCustomBatchResponse',
u'entries': []
}
for batch_id in range(0, num_rows):
merchant_id, item, api_item = generate_item_dict_api_pair()
if remove_merchant_id:
merchant_id = None
rows.append(
bigquery.Row(
(merchant_id, item['item_id'], item['title'],
item['description'], item['google_product_category'],
item['product_types'], item['link'], item['image_link'],
item['additional_image_link'], item['condition'],
item['availability'], item['price'], item['brand'],
item['gtin'], item['mpn'], item['shipping'],
item['loyalty_points'], item['ads_redirect'], item['color'],
item['size'], item['custom_label_0'], item['custom_label_1'],
item['custom_label_2'], item['custom_label_3'],
item['custom_label_4'], item['identifier_exists']),
ROW_SCHEMA))
batch_id_to_item_id[batch_id] = item['item_id']
if method == constants.Method.INSERT:
batch['entries'].append({
'batchId': batch_id,
'merchantId': str(merchant_id),
'method': method.value,
'product': api_item
})
response['entries'].append({
u'batchId': batch_id,
u'kind': u'content#productsCustomBatchResponseEntry',
u'product': {
u'color':
item['color'],
u'offerId':
item['item_id'],
u'gtin':
item['gtin'],
u'googleProductCategory':
item['google_product_category'],
u'availability':
item['availability'],
u'targetCountry':
constants.TARGET_COUNTRY,
u'title':
item['title'],
u'item_id':
'{}:{}:{}:{}'.format(api_item['channel'],
api_item['contentLanguage'],
api_item['targetCountry'],
api_item['offerId']),
u'customLabel1':
item['custom_label_1'],
u'price': {
u'currency': constants.TARGET_CURRENCY,
u'value': item['price']
},
u'channel':
api_item['channel'],
u'description':
item['description'],
u'contentLanguage':
api_item['contentLanguage'],
u'mpn':
item['mpn'],
u'brand':
item['brand'],
u'link':
item['link'],
u'adsRedirect':
item['ads_redirect'],
u'customLabel4':
item['custom_label_4'],
u'customLabel3':
item['custom_label_3'],
u'customLabel2':
item['custom_label_2'],
u'condition':
item['condition'],
u'customLabel0':
item['custom_label_0'],
u'kind':
u'content#product',
u'identifierExists':
item['identifier_exists'],
u'imageLink':
item['image_link'],
u'productTypes': [item['product_types']]
}
})
elif method == constants.Method.DELETE:
json_snippet = {
'batchId':
batch_id,
'merchantId':
str(merchant_id),
'method':
method.value,
'productId':
'{}:{}:{}:{}'.format(api_item['channel'],
api_item['contentLanguage'],
api_item['targetCountry'],
api_item['offerId'])
}
batch['entries'].append(json_snippet)
response['entries'].append(json_snippet)
return rows, batch, batch_id_to_item_id, response