def load_task():
client = Client()
job_config = LoadJobConfig()
schema_path = os.path.join(
dags_folder,
'resources/stages/raw/schemas/{task}.json'.format(task=task))
job_config.schema = read_bigquery_schema_from_file(schema_path)
job_config.source_format = SourceFormat.CSV if file_format == 'csv' else SourceFormat.NEWLINE_DELIMITED_JSON
if file_format == 'csv':
job_config.skip_leading_rows = 1
job_config.write_disposition = 'WRITE_TRUNCATE'
job_config.allow_quoted_newlines = allow_quoted_newlines
job_config.ignore_unknown_values = True
export_location_uri = 'gs://{bucket}/export'.format(
bucket=output_bucket)
uri = '{export_location_uri}/{task}/*.{file_format}'.format(
export_location_uri=export_location_uri,
task=task,
file_format=file_format)
table_ref = client.dataset(dataset_name_raw).table(task)
load_job = client.load_table_from_uri(uri,
table_ref,
job_config=job_config)
submit_bigquery_job(load_job, job_config)
assert load_job.state == 'DONE'
def upload_tweets():
big_query_client = bigquery.Client.from_service_account_json('my-beam-project-b2834963a4ae.json')
dataset_ref = big_query_client.dataset('Tweets')
dataset = Dataset(dataset_ref)
dataset.description = 'This represents tweets of trending topics'
dataset = big_query_client.create_dataset(dataset)
SCHEMA = [
SchemaField('Tweets', 'STRING', mode='Nullable'),
]
table_ref = big_query_client.dataset('Tweets').table('tabletweet')
load_config = LoadJobConfig()
load_config.skip_leading_rows = 0
load_config.schema = SCHEMA
load_config.allow_quoted_newlines = True
load_config.ignore_unknown_values = False
load_config.max_bad_records = 200
with open('tweets.csv', 'rb') as readable:
big_query_client.load_table_from_file(
readable, table_ref, job_config=load_config)
print('tweets file uploaded to big query')
def _load_to_bq(self, client, dataset, table_name, table_schema,
table_config, key_props, metadata_columns, truncate, rows):
logger = self.logger
partition_field = table_config.get("partition_field", None)
cluster_fields = table_config.get("cluster_fields", None)
force_fields = table_config.get("force_fields", {})
schema = build_schema(table_schema,
key_properties=key_props,
add_metadata=metadata_columns,
force_fields=force_fields)
load_config = LoadJobConfig()
load_config.ignore_unknown_values = True
load_config.schema = schema
if partition_field:
load_config.time_partitioning = bigquery.table.TimePartitioning(
type_=bigquery.table.TimePartitioningType.DAY,
field=partition_field)
if cluster_fields:
load_config.clustering_fields = cluster_fields
load_config.source_format = SourceFormat.NEWLINE_DELIMITED_JSON
if truncate:
logger.info(f"Load {table_name} by FULL_TABLE (truncate)")
load_config.write_disposition = WriteDisposition.WRITE_TRUNCATE
else:
logger.info(f"Appending to {table_name}")
load_config.write_disposition = WriteDisposition.WRITE_APPEND
logger.info("loading {} to BigQuery".format(table_name))
load_job = None
try:
load_job = client.load_table_from_file(rows,
dataset.table(table_name),
job_config=load_config,
rewind=True)
logger.info("loading job {}".format(load_job.job_id))
job = load_job.result()
logger.info(job._properties)
return job
except google_exceptions.BadRequest as err:
logger.error("failed to load table {} from file: {}".format(
table_name, str(err)))
if load_job and load_job.errors:
reason = err.errors[0]["reason"]
messages = [f"{err['message']}" for err in load_job.errors]
logger.error("reason: {reason}, errors:\n{e}".format(
reason=reason, e="\n".join(messages)))
err.message = f"reason: {reason}, errors: {';'.join(messages)}"
raise err