"report_attachments": [],
"success_recipients": [],
"failure_recipients": [],
"timeout_recipients": [],
"stopped_recipients": []
}
new_job = cml.create_job(create_jobs_params)
new_job_id = new_job["id"]
print("Created new job with jobid", new_job_id)
##
# Start a job
job_env_params = {}
start_job_params = {"environment": job_env_params}
job_status = cml.start_job(new_job_id, start_job_params)
print("Job started")
# Create model build script
cdsw_script = """#!/bin/bash
pip3 install -r requirements.txt"""
with open("cdsw-build.sh", 'w+') as f:
f.write(cdsw_script)
f.close()
os.chmod("cdsw-build.sh", 0o777)
# Get Default Engine Details
# Engine id is required for next step (create model)
default_engine_details = cml.get_default_engine({})
PROJECT_NAME = os.getenv("CDSW_PROJECT")
cml = CMLBootstrap(HOST, USERNAME, API_KEY, PROJECT_NAME)
latest_model = cml.get_model({
"id": model_id,
"latestModelDeployment": True,
"latestModelBuild": True
})
Model_CRN = latest_model["crn"]
Deployment_CRN = latest_model["latestModelDeployment"]["crn"]
# Read in the model metrics dict.
model_metrics = cdsw.read_metrics(model_crn=Model_CRN,
model_deployment_crn=Deployment_CRN)
# This is a handy way to unravel the dict into a big pandas dataframe.
metrics_df = pd.io.json.json_normalize(model_metrics["metrics"])
latest_aggregate_metric = metrics_df.dropna(
subset=["metrics.accuracy"]).sort_values(
'startTimeStampMs')[-1:]["metrics.accuracy"]
if latest_aggregate_metric.to_list()[0] < 0.6:
print("model is below threshold, retraining")
cml.start_job(job_id, {})
#TODO reploy new model
else:
print("model does not need to be retrained")
"success_recipients": [],
"failure_recipients": [],
"timeout_recipients": [],
"stopped_recipients": []
}
new_job = cml.create_job(create_jobs_params)
new_job_id = new_job["id"]
print("Created new job with jobid", new_job_id)
##
# Start the Training Job
job_env_params = {}
start_job_params = {"environment": job_env_params}
job_id = new_job_id
job_status = cml.start_job(job_id, start_job_params)
print("Training Job started")
# Stop a job
#job_dict = cml.start_job(job_id, start_job_params)
#cml.stop_job(job_id, start_job_params)
# Run experiment
run_experiment_params = {
"size": {
"id": 1,
"description": "1 vCPU / 2 GiB Memory",
"cpu": 1,
"memory": 2,
"route": "engine-profiles",
HOST = os.getenv("CDSW_API_URL").split(
":")[0] + "://" + os.getenv("CDSW_DOMAIN")
USERNAME = os.getenv("CDSW_PROJECT_URL").split(
"/")[6] # args.username # "vdibia"
API_KEY = os.getenv("CDSW_API_KEY")
PROJECT_NAME = os.getenv("CDSW_PROJECT")
cml = CMLBootstrap(HOST, USERNAME, API_KEY, PROJECT_NAME)
latest_model = cml.get_model({"id": model_id, "latestModelDeployment": True, "latestModelBuild": True})
Model_CRN = latest_model ["crn"]
Deployment_CRN = latest_model["latestModelDeployment"]["crn"]
# Read in the model metrics dict.
model_metrics = cdsw.read_metrics(model_crn=Model_CRN,model_deployment_crn=Deployment_CRN)
# This is a handy way to unravel the dict into a big pandas dataframe.
metrics_df = pd.io.json.json_normalize(model_metrics["metrics"])
latest_aggregate_metric = metrics_df.dropna(subset=["metrics.accuracy"]).sort_values('startTimeStampMs')[-1:]["metrics.accuracy"]
if latest_aggregate_metric.to_list()[0] < 0.6:
print("model is below threshold, retraining")
cml.start_job(244,{})
#TODO reploy new model
else:
print("model does not need to be retrained")
HOST = os.getenv("CDSW_API_URL").split(
":")[0] + "://" + os.getenv("CDSW_DOMAIN")
USERNAME = os.getenv("CDSW_PROJECT_URL").split(
"/")[6] # args.username # "vdibia"
API_KEY = os.getenv("CDSW_API_KEY")
PROJECT_NAME = os.getenv("CDSW_PROJECT")
cml = CMLBootstrap(HOST, USERNAME, API_KEY, PROJECT_NAME)
latest_model = cml.get_model({"id": model_id, "latestModelDeployment": True, "latestModelBuild": True})
Model_CRN = latest_model ["crn"]
Deployment_CRN = latest_model["latestModelDeployment"]["crn"]
# Read in the model metrics dict.
model_metrics = cdsw.read_metrics(model_crn=Model_CRN,model_deployment_crn=Deployment_CRN)
# This is a handy way to unravel the dict into a big pandas dataframe.
metrics_df = pd.io.json.json_normalize(model_metrics["metrics"])
latest_aggregate_metric = metrics_df.dropna(subset=["metrics.accuracy"]).sort_values('startTimeStampMs')[-1:]["metrics.accuracy"]
if latest_aggregate_metric.to_list()[0] < 0.6:
print("model is below threshold, retraining")
cml.start_job(68,{})
#TODO reploy new model
else:
print("model does not need to be retrained")