예제 #1
0
def load_from_file(file_path, table):
    table_ref = dataset.table(table)
    load_config = LoadJobConfig()
    load_config.autodetect = True

    with open(file_path, 'rb') as readable:
        result = client.load_table_from_file(readable, table_ref, job_config=load_config)

    # return result
    return table_ref.dataset_id + '.' + table_ref.table_id
예제 #2
0
def loadDataFromCSV(tablename, global_dataset_ref, filename):

    schema = getTableSchema(tablename, global_dataset_ref)

    table_ref = global_dataset_ref.table(tablename)

    load_config = LoadJobConfig()
    load_config.source_format = bigquery.SourceFormat.CSV
    load_config.schema = schema
    load_config.autodetect = True
    load_config.allow_quoted_newlines = True
    load_config.encoding = 'UTF-8'

    try:
        with open(filename, 'rb') as readable:
            job = bigquery_client.load_table_from_file(readable,
                                                       table_ref,
                                                       location='US',
                                                       job_config=load_config)
    except Exception as e:
        print("Error")
        print(e)

    job.result()

    print('Loaded {} rows into {}:{}.'.format(job.output_rows,
                                              global_dataset_ref,
                                              table_ref.table_id))

    return


# Testing
# if __name__ == '__main__':
# datasetname = 'Testing'
# tablename = 'SOViews'

# sqlquery = '''SELECT CONCAT(
#   'https://stackoverflow.com/questions/',
#   CAST(id as STRING)) as url,
# view_count
# FROM `bigquery-public-data.stackoverflow.posts_questions`
# WHERE tags like '%google-bigquery%'
# ORDER BY view_count DESC
# LIMIT 10'''

#createDataset(datasetname) #Successfully tested this code 2018-09-24
# global_dataset_ref = getDataset(datasetname) #Successfully tested this code 2018-09-24

#createTable(tablename, global_dataset_ref) #Successfully tested this code 2018-09-24
# getTable(tablename, global_dataset_ref) #Successfully tested this code 2018-09-24

# runBigQueryQuery(sqlquery) #Successfully tested this code 2018-09-24

#loadDataFromCSV(tablename, global_dataset_ref) #Successfully tested this code 2018-09-24