import xml.etree.ElementTree as ET import datetime run_time_suffix = datetime.datetime.now() run_time_suffix = run_time_suffix.strftime("%d%m%Y%H%M%S") # Set the setup variables needed by CMLBootstrap HOST = os.getenv("CDSW_API_URL").split( ":")[0] + "://" + os.getenv("CDSW_DOMAIN") USERNAME = os.getenv("CDSW_PROJECT_URL").split( "/")[6] # args.username # "vdibia" API_KEY = os.getenv("CDSW_API_KEY") PROJECT_NAME = os.getenv("CDSW_PROJECT") # Instantiate API Wrapper cml = CMLBootstrap(HOST, USERNAME, API_KEY, PROJECT_NAME) # Set the STORAGE environment variable try : storage=os.environ["STORAGE"] except: tree = ET.parse('/etc/hadoop/conf/hive-site.xml') root = tree.getroot() for prop in root.findall('property'): if prop.find('name').text == "hive.metastore.warehouse.dir": storage = prop.find('value').text.split("/")[0] + "//" + prop.find('value').text.split("/")[2] storage_environment_params = {"STORAGE":storage} storage_environment = cml.create_environment_variable(storage_environment_params) os.environ["STORAGE"] = storage
### Step 1: Install Requirements !bash cdsw-build.sh from cmlbootstrap import CMLBootstrap import datetime import os, time HOST = os.getenv("CDSW_API_URL").split(":")[0] + "://" + os.getenv("CDSW_DOMAIN") USERNAME = os.getenv("CDSW_PROJECT_URL").split("/")[6] # args.username # "vdibia" API_KEY = os.getenv("CDSW_API_KEY") PROJECT_NAME = os.getenv("CDSW_PROJECT") # Instantiate API Wrapper # Passing API key directly is better cml = CMLBootstrap(HOST, USERNAME, os.environ["MY_API_KEY"], PROJECT_NAME) # Get Project Details project_details = cml.get_project({}) project_id = project_details["id"] run_time_suffix = datetime.datetime.now() run_time_suffix = run_time_suffix.strftime("%d%m%Y%H%M%S") ### Step 2: Run 00_bootstrap.py to create Spark table exec(open("00_boostrap.py").read()) ### Step 3: Run 01_ModelDevelopment.ipynb to develop a first baseline model exec(open("01_A_ModelDevelopment.py").read())