def test_load_table_from_file(client, to_delete): """Upload table data from a CSV file.""" DATASET_ID = 'table_upload_from_file_dataset_{}'.format(_millis()) TABLE_ID = 'table_upload_from_file_table_{}'.format(_millis()) dataset = bigquery.Dataset(client.dataset(DATASET_ID)) client.create_dataset(dataset) to_delete.append(dataset) # [START load_table_from_file] csv_file = six.BytesIO(b"""full_name,age Phred Phlyntstone,32 Wylma Phlyntstone,29 """) table_ref = dataset.table(TABLE_ID) job_config = bigquery.LoadJobConfig() job_config.source_format = 'CSV' job_config.skip_leading_rows = 1 job_config.autodetect = True job = client.load_table_from_file(csv_file, table_ref, job_config=job_config) # API request job.result() # Waits for table load to complete. # [END load_table_from_file] table = client.get_table(table_ref) to_delete.insert(0, table) found_rows = [] def do_something(row): found_rows.append(row) # [START table_list_rows] for row in client.list_rows(table): # API request do_something(row) # [END table_list_rows] assert len(found_rows) == 2 # [START table_list_rows_iterator_properties] iterator = client.list_rows(table) # API request page = six.next(iterator.pages) rows = list(page) total = iterator.total_rows token = iterator.next_page_token # [END table_list_rows_iterator_properties] assert len(rows) == total == 2 assert token is None # Order is not preserved, so compare individually row1 = bigquery.Row(('Wylma Phlyntstone', 29), {'full_name': 0, 'age': 1}) assert row1 in rows row2 = bigquery.Row(('Phred Phlyntstone', 32), {'full_name': 0, 'age': 1}) assert row2 in rows
def test_期待しているテーブル結果の量に満たない場合マークがマイナスになる差分が出る(self): from google.cloud import bigquery client = bigquery.Client() expected = Table( str(Path(__file__).parent / "testdata/test5.json"), [ {"name": "name", "type": "STRING", "mode": "NULLABLE"}, {"name": "category", "type": "STRING", "mode": "NULLABLE"}, {"name": "value", "type": "INT64", "mode": "NULLABLE"}, ], "EXPECTED", ) input_tables = [ Table( str(Path(__file__).parent / "testdata/test3.json"), [ {"name": "name", "type": "STRING", "mode": "NULLABLE"}, {"name": "category", "type": "STRING", "mode": "NULLABLE"}, {"name": "value", "type": "INT64", "mode": "NULLABLE"}, ], "INPUT_DATA", ) ] query = Query("ACTUAL", """SELECT * FROM abc""", [], {"abc": "INPUT_DATA"}) qlt = QueryLogicTest(client, expected, input_tables, query) success, records = qlt.run() assert not success assert records == [ bigquery.Row( ("-", "eee", "fff", 500, 3), {"mark": 0, "name": 1, "category": 2, "value": 3, "n": 4}, ) ]
def test_load_table_file(capsys, random_table_id, client): samples_test_dir = os.path.abspath(os.path.dirname(__file__)) file_path = os.path.join(samples_test_dir, "..", "..", "tests", "data", "people.csv") table = load_table_file.load_table_file(file_path, random_table_id) out, _ = capsys.readouterr() assert "Loaded 2 rows and 2 columns" in out rows = list(client.list_rows(table)) # Make an API request. assert len(rows) == 2 # Order is not preserved, so compare individually row1 = bigquery.Row(("Wylma Phlyntstone", 29), {"full_name": 0, "age": 1}) assert row1 in rows row2 = bigquery.Row(("Phred Phlyntstone", 32), {"full_name": 0, "age": 1}) assert row2 in rows
def create_fake_bigquery_row(self, dau=123, submission_date=date(2020, 5, 28), **kwargs): data = {'dau': dau, 'submission_date': submission_date, **kwargs} return bigquery.Row( list(data.values()), {key: idx for idx, key in enumerate(data.keys())}, )
import http import json import socket import unittest import unittest.mock as mock from google.cloud import bigquery from googleapiclient import errors from parameterized import parameterized import constants import main from models import failure from models import process_result DUMMY_ROWS = [bigquery.Row(['0001'], {'item_id': 0})] DUMMY_START_INDEX = 0 DUMMY_BATCH_SIZE = 1000 DUMMY_TIMESTAMP = '0001-01-01:00:00:00' DUMMY_REQUEST_BODY = json.dumps({ 'start_index': DUMMY_START_INDEX, 'batch_size': DUMMY_BATCH_SIZE, 'timestamp': DUMMY_TIMESTAMP, }) DUMMY_SUCCESSES = ['0001', '0002', '0003'] DUMMY_FAILURES = [failure.Failure('0004', 'Error message')] DUMMY_SKIPPED = ['0005'] INSERT_URL = '/insert_items' DELETE_URL = '/delete_items' PREVENT_EXPIRING_URL = '/prevent_expiring_items'
def create_bigquery_row(self, data): return bigquery.Row( list(data.values()), {key: idx for idx, key in enumerate(data.keys())}, )
def generate_test_data( method: constants.Method, num_rows=1, remove_merchant_id=False ) -> Tuple[List[bigquery.Row], constants.Batch, constants.BatchIdToItemId, Dict[str, Any]]: """Generates a tuple containing a triplet of matching row, batch, response. Args: method: The API method for this batch (insert, delete) num_rows: The number of rows to generate remove_merchant_id: If true, set merchant_id to None Returns: A tuple containing row_data, batch, and response. The row_data represents data pulled directly from BigQuery in as Row objects. The batch data is a Content API request batch corresponding to the Row data but in JSON object form. The response is a JSON object which represents the expected response from the Content API call. """ rows = [] batch = {'entries': []} batch_id_to_item_id = dict() response = { u'kind': u'content#productsCustomBatchResponse', u'entries': [] } for batch_id in range(0, num_rows): merchant_id, item, api_item = generate_item_dict_api_pair() if remove_merchant_id: merchant_id = None rows.append( bigquery.Row( (merchant_id, item['item_id'], item['title'], item['description'], item['google_product_category'], item['product_types'], item['link'], item['image_link'], item['additional_image_link'], item['condition'], item['availability'], item['price'], item['brand'], item['gtin'], item['mpn'], item['shipping'], item['loyalty_points'], item['ads_redirect'], item['color'], item['size'], item['custom_label_0'], item['custom_label_1'], item['custom_label_2'], item['custom_label_3'], item['custom_label_4'], item['identifier_exists']), ROW_SCHEMA)) batch_id_to_item_id[batch_id] = item['item_id'] if method == constants.Method.INSERT: batch['entries'].append({ 'batchId': batch_id, 'merchantId': str(merchant_id), 'method': method.value, 'product': api_item }) response['entries'].append({ u'batchId': batch_id, u'kind': u'content#productsCustomBatchResponseEntry', u'product': { u'color': item['color'], u'offerId': item['item_id'], u'gtin': item['gtin'], u'googleProductCategory': item['google_product_category'], u'availability': item['availability'], u'targetCountry': constants.TARGET_COUNTRY, u'title': item['title'], u'item_id': '{}:{}:{}:{}'.format(api_item['channel'], api_item['contentLanguage'], api_item['targetCountry'], api_item['offerId']), u'customLabel1': item['custom_label_1'], u'price': { u'currency': constants.TARGET_CURRENCY, u'value': item['price'] }, u'channel': api_item['channel'], u'description': item['description'], u'contentLanguage': api_item['contentLanguage'], u'mpn': item['mpn'], u'brand': item['brand'], u'link': item['link'], u'adsRedirect': item['ads_redirect'], u'customLabel4': item['custom_label_4'], u'customLabel3': item['custom_label_3'], u'customLabel2': item['custom_label_2'], u'condition': item['condition'], u'customLabel0': item['custom_label_0'], u'kind': u'content#product', u'identifierExists': item['identifier_exists'], u'imageLink': item['image_link'], u'productTypes': [item['product_types']] } }) elif method == constants.Method.DELETE: json_snippet = { 'batchId': batch_id, 'merchantId': str(merchant_id), 'method': method.value, 'productId': '{}:{}:{}:{}'.format(api_item['channel'], api_item['contentLanguage'], api_item['targetCountry'], api_item['offerId']) } batch['entries'].append(json_snippet) response['entries'].append(json_snippet) return rows, batch, batch_id_to_item_id, response