def robot_auth():
conn = base_hook('robot_auth')
auth = conn['host']
headers = {'content-type': 'application/json'}
data = {'username': conn['user'], 'password': conn['password']}
r = requests.post(auth, data=json.dumps(data), headers=headers, timeout=10)
token = r.json()['access_token']
Variable.set('robot_token', token)
def robot_out_of_stock(date):
conn = base_hook('robot_out_of_stock')
url = conn['host']
token = Variable.get('robot_token')
headers = {
'content-type': 'application/json',
'Authorization': 'JWT ' + token
}
data = {'date': date}
r = requests.get(url, data=json.dumps(data), headers=headers, timeout=10)
r = r.json()
if r['message'] == "No out_of_stock items for this date":
print(f'There is no out_of_stock items for {date}')
logging.info(f'There is no out_of_stock items for {date}')
client = InsecureClient('http://127.0.0.1:50070/', user='******')
with client.write(os.path.join('/bronze/out_of_stock',
date + '_out_of_stock.json'),
encoding='utf-8') as f:
json.dump(r, f)
logging.info('File loaded ' +
os.path.join('/bronze/out_of_stock', date +
'_out_of_stock.json'))
def gold_auth():
conn = base_hook('gold_auth')
gp_url = conn['host']
gp_properties = {"user": conn['user'], "password": conn['password']}
spark = SparkSession.builder\
.config('spark.driver.extraClassPath'\
, '/home/user/shared_folder/postgresql-42.2.20.jar')\
.master('local')\
.appName('gold_load')\
.getOrCreate()
return gp_url, gp_properties, spark
def get_tables():
pg_creds = base_hook('base_auth')
with psycopg2.connect(**pg_creds) as pg_connection:
cursor = pg_connection.cursor()
cursor.execute("""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'""")
tables = cursor.fetchall()
logging.info(f'Got all tables')
return tables
def load_to_bronze_psql(date, table):
pg_creds = base_hook('base_auth')
client = InsecureClient('http://127.0.0.1:50070/', user='******')
with psycopg2.connect(**pg_creds) as pg_connection:
cursor = pg_connection.cursor()
with client.write(
os.path.join('/bronze/postgres', date, table[0] + '.csv')
) as csv:
cursor.copy_expert(f'COPY {table[0]} to STDOUT WITH HEADER CSV', csv)
logging.info('File loaded ' + os.path.join('/bronze/postgres', date, table[0] + '.csv'))