def test_list_rows_page_size(bigquery_client: bigquery.Client, table_id: str):
num_items = 7
page_size = 3
num_pages, num_last_page = divmod(num_items, page_size)
to_insert = [{
"string_col": "item%d" % i,
"rowindex": i
} for i in range(num_items)]
bigquery_client.load_table_from_json(to_insert, table_id).result()
df = bigquery_client.list_rows(
table_id,
selected_fields=[
bigquery.SchemaField("string_col", enums.SqlTypeNames.STRING)
],
page_size=page_size,
)
pages = df.pages
for i in range(num_pages):
page = next(pages)
assert page.num_items == page_size
page = next(pages)
assert page.num_items == num_last_page
def write_data_to_bigquery(
client: bigquery.Client,
table_id: str,
rows_to_insert: List[Dict[str, str]],
json_schema: Dict[str, str],
):
bigquery_schema: List[
bigquery.SchemaField] = convert_json_schema_to_bigquery_schema(
json_schema)
complete_table_id = get_complete_table_id(table_id)
write_disposition = bigquery.WriteDisposition.WRITE_APPEND
schema_update_options = [bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION]
job_config = bigquery.LoadJobConfig(
schema=bigquery_schema,
write_disposition=write_disposition,
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
schema_update_options=schema_update_options,
)
# batch write
if len(rows_to_insert) > 0:
load_job = client.load_table_from_json(
rows_to_insert,
complete_table_id,
job_config=job_config,
)
load_job.result()
else:
print("uploading data row is empty")
