def create(self, table_id, schema):
""" Create a table in Google BigQuery given a table and schema
Parameters
----------
table : str
Name of table to be written
schema : str
Use the generate_bq_schema to generate your table schema from a
dataframe.
"""
from google.cloud.bigquery import SchemaField
from google.cloud.bigquery import Table
if self.exists(table_id):
raise TableCreationError(
"Table {0} already " "exists".format(table_id)
)
if not _Dataset(self.project_id, credentials=self.credentials).exists(
self.dataset_id
):
_Dataset(
self.project_id,
credentials=self.credentials,
location=self.location,
).create(self.dataset_id)
table_ref = self.client.dataset(self.dataset_id).table(table_id)
table = Table(table_ref)
# Manually create the schema objects, adding NULLABLE mode
# as a workaround for
# https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456
for field in schema["fields"]:
if "mode" not in field:
field["mode"] = "NULLABLE"
table.schema = [
SchemaField.from_api_repr(field) for field in schema["fields"]
]
try:
self.client.create_table(table)
except self.http_error as ex:
self.process_http_error(ex)
def update_columns(self, relation, columns):
if len(columns) == 0:
return
conn = self.connections.get_thread_connection()
table_ref = self.get_table_ref_from_relation(conn, relation)
table = conn.handle.get_table(table_ref)
new_schema = []
for bq_column in table.schema:
bq_column_dict = bq_column.to_api_repr()
new_bq_column_dict = self._update_column_dict(
bq_column_dict, columns)
new_schema.append(SchemaField.from_api_repr(new_bq_column_dict))
new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema)
conn.handle.update_table(new_table, ['schema'])
def update_column_descriptions(self, relation, columns):
if len(columns) == 0:
return
conn = self.connections.get_thread_connection()
table_ref = self.get_table_ref_from_relation(conn, relation)
table = conn.handle.get_table(table_ref)
new_schema = []
for column in table.schema:
if column.name in columns:
column_config = columns[column.name]
column_dict = column.to_api_repr()
column_dict['description'] = column_config.get('description')
column = SchemaField.from_api_repr(column_dict)
new_schema.append(column)
new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema)
conn.handle.update_table(new_table, ['schema'])
def create(self, table_id, schema):
"""Create a table in Google BigQuery given a table and schema
Parameters
----------
table : str
Name of table to be written
schema : str
Use the generate_bq_schema to generate your table schema from a
dataframe.
"""
from google.cloud.bigquery import SchemaField
from google.cloud.bigquery import Table
if self.exists(table_id):
raise TableCreationError(
"Table {0} already " "exists".format(table_id)
)
if not _Dataset(self.project_id, credentials=self.credentials).exists(
self.dataset_id
):
_Dataset(
self.project_id,
credentials=self.credentials,
location=self.location,
).create(self.dataset_id)
table_ref = self.client.dataset(self.dataset_id).table(table_id)
table = Table(table_ref)
schema = pandas_gbq.schema.add_default_nullable_mode(schema)
table.schema = [
SchemaField.from_api_repr(field) for field in schema["fields"]
]
try:
self.client.create_table(table)
except self.http_error as ex:
self.process_http_error(ex)
def create(self, table_id, schema):
""" Create a table in Google BigQuery given a table and schema
Parameters
----------
table : str
Name of table to be written
schema : str
Use the generate_bq_schema to generate your table schema from a
dataframe.
"""
from google.cloud.bigquery import SchemaField
from google.cloud.bigquery import Table
if self.exists(table_id):
raise TableCreationError("Table {0} already "
"exists".format(table_id))
if not _Dataset(self.project_id,
private_key=self.private_key).exists(self.dataset_id):
_Dataset(self.project_id,
private_key=self.private_key).create(self.dataset_id)
table_ref = self.client.dataset(self.dataset_id).table(table_id)
table = Table(table_ref)
for field in schema['fields']:
if 'mode' not in field:
field['mode'] = 'NULLABLE'
table.schema = [
SchemaField.from_api_repr(field)
for field in schema['fields']
]
try:
self.client.create_table(table)
except self.http_error as ex:
self.process_http_error(ex)
def alias_field(self, field, alias):
api_repr = field.to_api_repr()
api_repr['name'] = alias
return SchemaField.from_api_repr(api_repr)
