import xml.etree.ElementTree as ET
import datetime
run_time_suffix = datetime.datetime.now()
run_time_suffix = run_time_suffix.strftime("%d%m%Y%H%M%S")
# Set the setup variables needed by CMLBootstrap
HOST = os.getenv("CDSW_API_URL").split(
":")[0] + "://" + os.getenv("CDSW_DOMAIN")
USERNAME = os.getenv("CDSW_PROJECT_URL").split(
"/")[6] # args.username # "vdibia"
API_KEY = os.getenv("CDSW_API_KEY")
PROJECT_NAME = os.getenv("CDSW_PROJECT")
# Instantiate API Wrapper
cml = CMLBootstrap(HOST, USERNAME, API_KEY, PROJECT_NAME)
# Set the STORAGE environment variable
try :
storage=os.environ["STORAGE"]
except:
tree = ET.parse('/etc/hadoop/conf/hive-site.xml')
root = tree.getroot()
for prop in root.findall('property'):
if prop.find('name').text == "hive.metastore.warehouse.dir":
storage = prop.find('value').text.split("/")[0] + "//" + prop.find('value').text.split("/")[2]
storage_environment_params = {"STORAGE":storage}
storage_environment = cml.create_environment_variable(storage_environment_params)
os.environ["STORAGE"] = storage
### Step 1: Install Requirements
!bash cdsw-build.sh
from cmlbootstrap import CMLBootstrap
import datetime
import os, time
HOST = os.getenv("CDSW_API_URL").split(":")[0] + "://" + os.getenv("CDSW_DOMAIN")
USERNAME = os.getenv("CDSW_PROJECT_URL").split("/")[6] # args.username # "vdibia"
API_KEY = os.getenv("CDSW_API_KEY")
PROJECT_NAME = os.getenv("CDSW_PROJECT")
# Instantiate API Wrapper
# Passing API key directly is better
cml = CMLBootstrap(HOST, USERNAME, os.environ["MY_API_KEY"], PROJECT_NAME)
# Get Project Details
project_details = cml.get_project({})
project_id = project_details["id"]
run_time_suffix = datetime.datetime.now()
run_time_suffix = run_time_suffix.strftime("%d%m%Y%H%M%S")
### Step 2: Run 00_bootstrap.py to create Spark table
exec(open("00_boostrap.py").read())
### Step 3: Run 01_ModelDevelopment.ipynb to develop a first baseline model
exec(open("01_A_ModelDevelopment.py").read())