def load_from_file(file_path, table):
table_ref = dataset.table(table)
load_config = LoadJobConfig()
load_config.autodetect = True
with open(file_path, 'rb') as readable:
result = client.load_table_from_file(readable, table_ref, job_config=load_config)
# return result
return table_ref.dataset_id + '.' + table_ref.table_id
def loadDataFromCSV(tablename, global_dataset_ref, filename):
schema = getTableSchema(tablename, global_dataset_ref)
table_ref = global_dataset_ref.table(tablename)
load_config = LoadJobConfig()
load_config.source_format = bigquery.SourceFormat.CSV
load_config.schema = schema
load_config.autodetect = True
load_config.allow_quoted_newlines = True
load_config.encoding = 'UTF-8'
try:
with open(filename, 'rb') as readable:
job = bigquery_client.load_table_from_file(readable,
table_ref,
location='US',
job_config=load_config)
except Exception as e:
print("Error")
print(e)
job.result()
print('Loaded {} rows into {}:{}.'.format(job.output_rows,
global_dataset_ref,
table_ref.table_id))
return
# Testing
# if __name__ == '__main__':
# datasetname = 'Testing'
# tablename = 'SOViews'
# sqlquery = '''SELECT CONCAT(
# 'https://stackoverflow.com/questions/',
# CAST(id as STRING)) as url,
# view_count
# FROM `bigquery-public-data.stackoverflow.posts_questions`
# WHERE tags like '%google-bigquery%'
# ORDER BY view_count DESC
# LIMIT 10'''
#createDataset(datasetname) #Successfully tested this code 2018-09-24
# global_dataset_ref = getDataset(datasetname) #Successfully tested this code 2018-09-24
#createTable(tablename, global_dataset_ref) #Successfully tested this code 2018-09-24
# getTable(tablename, global_dataset_ref) #Successfully tested this code 2018-09-24
# runBigQueryQuery(sqlquery) #Successfully tested this code 2018-09-24
#loadDataFromCSV(tablename, global_dataset_ref) #Successfully tested this code 2018-09-24