def create(self, table_id, schema): """ Create a table in Google BigQuery given a table and schema Parameters ---------- table : str Name of table to be written schema : str Use the generate_bq_schema to generate your table schema from a dataframe. """ from google.cloud.bigquery import SchemaField from google.cloud.bigquery import Table if self.exists(table_id): raise TableCreationError( "Table {0} already " "exists".format(table_id) ) if not _Dataset(self.project_id, credentials=self.credentials).exists( self.dataset_id ): _Dataset( self.project_id, credentials=self.credentials, location=self.location, ).create(self.dataset_id) table_ref = self.client.dataset(self.dataset_id).table(table_id) table = Table(table_ref) # Manually create the schema objects, adding NULLABLE mode # as a workaround for # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 for field in schema["fields"]: if "mode" not in field: field["mode"] = "NULLABLE" table.schema = [ SchemaField.from_api_repr(field) for field in schema["fields"] ] try: self.client.create_table(table) except self.http_error as ex: self.process_http_error(ex)
def update_columns(self, relation, columns): if len(columns) == 0: return conn = self.connections.get_thread_connection() table_ref = self.get_table_ref_from_relation(conn, relation) table = conn.handle.get_table(table_ref) new_schema = [] for bq_column in table.schema: bq_column_dict = bq_column.to_api_repr() new_bq_column_dict = self._update_column_dict( bq_column_dict, columns) new_schema.append(SchemaField.from_api_repr(new_bq_column_dict)) new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) conn.handle.update_table(new_table, ['schema'])
def update_column_descriptions(self, relation, columns): if len(columns) == 0: return conn = self.connections.get_thread_connection() table_ref = self.get_table_ref_from_relation(conn, relation) table = conn.handle.get_table(table_ref) new_schema = [] for column in table.schema: if column.name in columns: column_config = columns[column.name] column_dict = column.to_api_repr() column_dict['description'] = column_config.get('description') column = SchemaField.from_api_repr(column_dict) new_schema.append(column) new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema) conn.handle.update_table(new_table, ['schema'])
def create(self, table_id, schema): """Create a table in Google BigQuery given a table and schema Parameters ---------- table : str Name of table to be written schema : str Use the generate_bq_schema to generate your table schema from a dataframe. """ from google.cloud.bigquery import SchemaField from google.cloud.bigquery import Table if self.exists(table_id): raise TableCreationError( "Table {0} already " "exists".format(table_id) ) if not _Dataset(self.project_id, credentials=self.credentials).exists( self.dataset_id ): _Dataset( self.project_id, credentials=self.credentials, location=self.location, ).create(self.dataset_id) table_ref = self.client.dataset(self.dataset_id).table(table_id) table = Table(table_ref) schema = pandas_gbq.schema.add_default_nullable_mode(schema) table.schema = [ SchemaField.from_api_repr(field) for field in schema["fields"] ] try: self.client.create_table(table) except self.http_error as ex: self.process_http_error(ex)
def create(self, table_id, schema): """ Create a table in Google BigQuery given a table and schema Parameters ---------- table : str Name of table to be written schema : str Use the generate_bq_schema to generate your table schema from a dataframe. """ from google.cloud.bigquery import SchemaField from google.cloud.bigquery import Table if self.exists(table_id): raise TableCreationError("Table {0} already " "exists".format(table_id)) if not _Dataset(self.project_id, private_key=self.private_key).exists(self.dataset_id): _Dataset(self.project_id, private_key=self.private_key).create(self.dataset_id) table_ref = self.client.dataset(self.dataset_id).table(table_id) table = Table(table_ref) for field in schema['fields']: if 'mode' not in field: field['mode'] = 'NULLABLE' table.schema = [ SchemaField.from_api_repr(field) for field in schema['fields'] ] try: self.client.create_table(table) except self.http_error as ex: self.process_http_error(ex)
def alias_field(self, field, alias): api_repr = field.to_api_repr() api_repr['name'] = alias return SchemaField.from_api_repr(api_repr)