def getBucket():
credentials_file = 'K9Bucket-2-b6152a9b63fe.json'
project_name = 'k9bucket-2'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
buckets = project.list()[0]
return buckets
import gcs_client
credentials_file = 'HelloWorld-5b5ab5e9eba6.json'
project_name = 'helloworld-163119'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
buckets = project.list()
objects = buckets[0].list()
def buckets(project):
# Print buckets in the project
buckets = project.list()
print 'Buckets:\n\t-', '\n\t- '.join(map(str, buckets))
def bucketInfo(project,buckNo):
# Print some information from first bucket
buckets = project.list()
bucket = buckets[buckNo]
print 'Bucket %s is located in %s with storage class %s' % (bucket, bucket.location,bucket.storageClass)
def writeObject(project,buckNo,fileName,text):
# Writing Object
buckets = project.list()
bucket = buckets[buckNo]
with bucket.open(fileName, 'w') as obj:
obj.write(text + '\n')
with bucket.open(fileName) as obj:
from airflow.contrib.hooks.bigquery_hook import BigQueryHook
from airflow.hooks.postgres_hook import PostgresHook
from google.cloud import storage
from datetime import datetime, timedelta
from tempfile import NamedTemporaryFile
from io import BytesIO
import gcs_client
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = 'keys/airy-media-254122-973505938453.json'
CRED = gcs_client.Credentials(os.environ["GOOGLE_APPLICATION_CREDENTIALS"])
PROJECT = gcs_client.Project('airy-media-254122', CRED)
BUCKET = PROJECT.list()[0]
SQL_PATH = 'dags/extract_load_pipeline/sql/'
# loads postgres table, creates a table if it does not exist
def load_table(**kwargs):
conn = PostgresHook(postgres_conn_id='my_local_db').get_conn()
cursor = conn.cursor()
client = storage.Client()
bucket = client.get_bucket('airy-media-254122.appspot.com')
blob = bucket.get_blob('bq_bucket/bq_dataset.csv')
# create a temporary file and store csv into that to read
tempf = NamedTemporaryFile()
blob.download_to_filename(tempf.name)
query = "COPY airflow.austin_service_reports FROM '"+tempf.name+"' WITH (FORMAT csv)"