class MLHealth(object):
"""
This class provides APIs to access health metrics created for a workflow instance.
It includes stats and models.
"""
def __init__(self):
self._sc = None
self._eco_server = None
self._eco_port = None
self._db_host = None
self._db_port = None
self._mode = None
self._wf_id = None
self._agent_list = []
self._jvm_mlops = None
self._rest_helper = None
self._zk_host = None
self._token = None
@staticmethod
def _search_list_dict(kv, key, value):
for x in kv:
if x[key] == value:
return x
return None
def init(self,
sc=None,
wf_id=None,
eco_server=None,
eco_port=None,
db_host=None,
db_port=None,
zk_host=None,
token=None,
mode=MLOpsMode.PYSPARK):
"""
Perform initialization of the health library. eco and db configuration can be
set up using environment variables and hence are optional. This is true for the workflow
instance id as well. Currently, python and pyspark mode of operation are supported.
:param sc: optional spark context for pyspark jobs
:param wf_id: workflow instance id
:param eco_server: eco server host
:param eco_port: eco server port
:param db_host: stats db host
:param db_port: stats db port
:param zk_host: zookeeper host port string
:param token: authentication token
:param mode: python or pyspark
:return:
"""
self._sc = sc
no_zk = False
if zk_host is None:
if os.environ.get(Constants.MLOPS_ZK_HOST) is not None:
self._zk_host = os.environ[Constants.MLOPS_ZK_HOST]
else:
no_zk = True
else:
self._zk_host = zk_host
if token is None:
if os.environ.get(Constants.MLOPS_TOKEN) is not None:
self._token = os.environ[Constants.MLOPS_TOKEN]
else:
raise MLOpsException("Internal Error: No auth token provided")
else:
self._token = token
if no_zk is False:
# initialize zk connections and get active eco server
try:
zk = KazooClient(hosts=self._zk_host, read_only=True)
zk.start()
if zk.exists('/ECO/curator/activeHostPort'):
data, stat = zk.get("/ECO/curator/activeHostPort")
eco_host_port = data.decode("utf-8").split(':')
if len(eco_host_port) is 2:
self._eco_server = eco_host_port[0]
self._eco_port = eco_host_port[1]
else:
raise MLOpsException(
"Internal Error: Invalid zookeeper active server "
"entry")
else:
raise MLOpsException(
"Unable to connect to the active MLOps server")
zk.stop()
except Exception:
raise MLOpsException("Unable to locate active MLOps server")
# if eco server was found using zookeeper, then don't use the environment variable
if self._eco_server is None:
if eco_server is None:
if os.environ.get(Constants.MLOPS_ECO_HOST) is not None:
self._eco_server = os.environ[Constants.MLOPS_ECO_HOST]
else:
raise MLOpsException("MLOps server host not provided")
else:
self._eco_server = eco_server
if self._eco_port is None:
if eco_port is None:
if os.environ.get(Constants.MLOPS_ECO_PORT) is not None:
self._eco_port = os.environ[Constants.MLOPS_ECO_PORT]
else:
raise MLOpsException("MLOps server port not provided")
else:
self._eco_port = eco_port
if db_host is None:
if os.environ.get(Constants.MLOPS_TIMESERIES_DB_HOST) is not None:
self._db_host = os.environ[Constants.MLOPS_TIMESERIES_DB_HOST]
else:
raise MLOpsException("Database server host not provided")
else:
self._db_host = db_host
if db_port is None:
if os.environ.get(Constants.MLOPS_TIMESERIES_DB_PORT) is not None:
self._db_port = os.environ[Constants.MLOPS_TIMESERIES_DB_PORT]
else:
raise MLOpsException("Database server port not provided")
else:
self._db_port = db_port
if wf_id is None:
if os.environ.get(Constants.MLOPS_HEALTH_WF_ID) is not None:
self._wf_id = os.environ[Constants.MLOPS_HEALTH_WF_ID]
else:
raise MLOpsException("{} instance id not provided".format(
Constants.ION_LITERAL))
else:
self._wf_id = wf_id
self._mode = mode
self._rest_helper = MlOpsRestFactory().get_rest_helper(
MLOpsMode.STAND_ALONE)
self._rest_helper.init(self._eco_server, self._eco_port, self._token)
if sc is not None:
try:
import pyspark
import pyspark.mllib.common as ml
from pyspark.sql import SQLContext
import pyspark.sql.types
if not isinstance(sc, pyspark.context.SparkContext):
raise MLOpsException("sc argument is not pyspark context")
# initialize jvm to mlops
self._jvm_mlops = sc._jvm.org.mlpiper.mlops.MLOps
ping_val = 5
ping_ret = self._jvm_mlops.ping(ping_val)
if ping_ret != 5:
raise MLOpsException(
"Got unexpected value from MLOps.ping sent {} got {} ".
format(ping_val, ping_ret))
except Exception:
err = "Unable to access MLOps objects within the health program"
self._jvm_mlops = None
raise MLOpsException(err)
groups = self._rest_helper.get_groups()
agents = self._rest_helper.get_agents()
# based on ion / workflow run id, get ion description
wf_instances = self._rest_helper.get_workflow_instances()
wfi = MLHealth._search_list_dict(wf_instances, 'id', self._wf_id)
if wfi is None:
raise MLOpsException("Could not locate {} instance {}".format(
Constants.ION_LITERAL, self._wf_id))
# get agents/groups
node_info = wfi['pipelineInstanceIdToWfNode']
for id in node_info:
kv = node_info[id]
agents_in_group = MLHealth._search_list_dict(
groups, 'id', kv['groupId'])['agents']
if agents_in_group is None:
raise MLOpsException("Could not locate group {}".format(
kv['groupId']))
ptype = kv['pipelineType']
agent_addrs = []
for aig in agents_in_group:
agent_addr = MLHealth._search_list_dict(agents, 'id', aig)
if agent_addr is None:
raise MLOpsException(
"Could not locate agent {} in group description".
format(aig))
agent_addrs.append(agent_addr['address'])
self._agent_list.append((id, ptype, agent_addrs))
def _get_sparksql_context(self):
from pyspark.sql import SQLContext
return SQLContext.getOrCreate(self._sc)
def _convert_pd_to_df(self, pd_df):
sql_context = self._get_sparksql_context()
return sql_context.createDataFrame(pd_df)
def _get_model_pdf(self):
models = self._rest_helper.get_model_list()
return pd.read_json(json.dumps(models))
def _get_model_python(self, start_time, end_time):
mdf = self._get_model_pdf()
newdf = mdf[(mdf['workflowRunId'] == self._wf_id)
& (mdf['createdTimestamp'] >= start_time) &
(mdf['createdTimestamp'] <= end_time)]
if newdf.shape[0] == 0:
raise MLOpsException(
"No models found in time range {}:{} for instance {}".format(
start_time, end_time, self._wf_id))
output_df = newdf[['createdTimestamp', 'name', 'id']]
vals = newdf['id'].values
models = self._rest_helper.get_model_by_id(vals)
output_df = output_df.assign(model=models)
return output_df
def _get_model_pyspark(self, start_time, end_time):
df = self._get_model_python(start_time, end_time)
sdf = self._convert_pd_to_df(df)
return sdf
def get_model(self, start_time, end_time):
"""
Retrieve models from eco server based on start and end times. Use the workflow
instance id to retrieve only models that are specific to this job.
:param start_time: start time in milliseconds
:param end_time: end time in milliseconds
:return: spark or pandas dataframe based on mode with the models as a byte array
"""
if self._mode == MLOpsMode.PYSPARK:
return self._get_model_pyspark(start_time, end_time)
elif self._mode == MLOpsMode.PYTHON:
return self._get_model_python(start_time, end_time)
else:
raise MLOpsException("Invalid mode: [{}]".format(self._mode))
class MLOpsCtx(BaseObj):
"""
Provide context information for MLOps library.
This is an internal class which should not be exposed to users of MLOps.
The object contains the structure of the ION and groups and such.
"""
def __init__(self, config, mode=None):
"""
Perform initialization of the MLOpsCtx.
It expects configuration from the environment to arrive in the standard usage mode.
:param config: :class:`ConfigInfo` for this MLOps instantiation
:param mode: python or pyspark
:return:
: raises MLOpsException for invalid configurations
"""
super(MLOpsCtx, self).__init__(__name__)
self._info("MLOpsCtx __init__ called")
self._info("Config\n{}".format(config))
self._agent_list = []
self._rest_helper = None
self._ci = config
self._mode = mode
self._ion = None # Will contain an ION class corresponding to the active ION which this mlops is part of
self._ees_dict = {} # Will contain all ees defined with the agent inside
self._agents_dict = {}
self._rest_helper = MlOpsRestFactory().get_rest_helper(self._mode, self._ci.mlops_server, self._ci.mlops_port, self._ci.token)
if self._mode == MLOpsMode.AGENT or self._mode == MLOpsMode.REST_ACCUMULATOR:
# In agent mode, we talk with the agent and use the mlops prefix to the http requests
self._validate_config()
self._info("Agent mode")
self._rest_helper.set_prefix(Constants.URL_MLOPS_PREFIX)
json_dict = self._detect_ion_structure()
health_json_dict = self._fetch_health_thresholds()
self._detect_ees_and_agents()
self._build_ion_obj(json_dict)
self._build_health_obj(health_json_dict)
elif self._mode == MLOpsMode.ATTACH:
# In attach mode, we connect either to the ZK or to the server directly
if self._ci.zk_host:
self._detect_mlops_server_via_zk()
self._validate_config()
self._info("In pm mode - will try to connect to server")
ion_json_dict = self._detect_ion_structure()
health_json_dict = self._fetch_health_thresholds()
self._detect_ees_and_agents()
self._build_ion_obj(ion_json_dict)
self._build_health_obj(health_json_dict)
elif self._mode == MLOpsMode.STAND_ALONE:
# In stand alone mode, we do not have a valid ION structure
self._logger.info("In stand-alone mode: ctx data will not be available")
self._set_stand_alone_values()
else:
raise MLOpsException("Unsupported operation mode: {}".format(self._mode))
def _fetch_health_thresholds(self):
return self._rest_helper.get_health_thresholds(self._ci.ion_id)
def _set_stand_alone_values(self):
self._ion = ION()
self._ion.id = 1
self._ion.name = "ION_1"
def _validate_config(self):
"""
Validate that all config information is present
:return:
:raises MLOpsException for invalid configurations
"""
if self._ci.token is None:
raise MLOpsException("Internal Error: No auth token provided")
if self._ci.mlops_server is None or self._ci.mlops_port is None:
raise MLOpsException("MLOps server host or port were not provided")
if self._ci.ion_id is None:
MLOpsException("{} instance id not provided".format(Constants.ION_LITERAL))
def _detect_mlops_server_via_zk(self):
"""
Detect the active mlops server via the ZK
:return:
"""
zk = None
try:
zk = KazooClient(hosts=self._ci.zk_host, read_only=True)
zk.start()
if zk.exists(Constants.MLOPS_ZK_ACTIVE_HOST_PORT):
data, stat = zk.get(Constants.MLOPS_ZK_ACTIVE_HOST_PORT)
eco_host_port = data.decode("utf-8").split(':')
if len(eco_host_port) is 2:
self._ci.mlops_server = eco_host_port[0]
self._ci.mlops_port = eco_host_port[1]
else:
raise MLOpsException("Internal Error: Invalid zookeeper active server entry, host_port: {}"
.format(eco_host_port))
else:
raise MLOpsException("Unable to connect to the active MLOps server, zk_host: {}"
.format(self._ci.zk_host))
except Exception as e:
raise MLOpsException("{}, zk_host: {}".format(e, self._ci.zk_host))
finally:
if zk:
zk.stop()
@staticmethod
def _search_list_dict(kv, key, value):
for x in kv:
if x[key] == value:
return x
return None
def _detect_ion_structure(self):
"""
Detect the current ion structure (pipeline, groups, agents, etc.)
:return:
:raises MLOpsException if ION or other structures are not found
"""
self._info("Detecting {} structure".format(Constants.ION_LITERAL))
# This is the max number of retries to wait until the ION is running.
# The pipelineInstance part of the workflow description does not appear until the ION
# switches to RUNNING state. For this reason, the code loop until the pipelineInstances part
# appears in the JSON.
max_tries = Constants.WAIT_FOR_PIPELINE_INSTANCE_TO_APPEAR_TIMEOUT
wf_instance = {}
found = False
for idx in range(0, max_tries):
wf_instance = self._rest_helper.get_workflow_instance(self._ci.ion_id)
if wf_instance is None:
raise MLOpsException("Could not locate {} instance {}".format(
Constants.ION_LITERAL, self._ci.ion_id))
self._debug("{} status: {}".format(Constants.ION_LITERAL, wf_instance['status']))
if IONJsonConstants.PIPELINE_INSTANCES_SECTION in wf_instance:
found = True
break
self._info("Could not find {} in workflow json - try {}".format(
IONJsonConstants.PIPELINE_INSTANCES_SECTION, idx))
time.sleep(1)
if found is False:
raise MLOpsException("Could not find {} section in workflow information".format(
IONJsonConstants.PIPELINE_INSTANCES_SECTION))
ion_json_dict = wf_instance
self._debug("workflow: {}".format(ion_json_dict))
return ion_json_dict
def _detect_ees_and_agents(self):
ees_json_dict = self._rest_helper.get_ees()
agents_json_dict = self._rest_helper.get_agents()
self._debug("Agents JSON:\n{}\n\n".format(agents_json_dict))
self._debug("EEs JSON:\n{}\n\n".format(ees_json_dict))
# Generating a dict of all agents by ID
for agent_json in agents_json_dict:
agent_obj = Agent()
agent_obj.id = str(agent_json["id"])
agent_obj.hostname = str(agent_json["address"])
self._agents_dict[agent_obj.id] = agent_obj
for ee_json in ees_json_dict:
ee = EE()
ee.name = str(ee_json["name"])
ee.id = str(ee_json["id"])
ee.agent_id = str(ee_json["agentId"])
# get agent object we created above in the agent_dict
if ee.agent_id not in self._agents_dict:
raise MLOpsException("EE {} contains Agent {} which is not in global agent list".format(
ee.name, ee.agent_id))
agent_obj = self._agents_dict[ee.agent_id]
ee.agents.append(agent_obj)
ee.agent_by_id[ee.agent_id] = agent_obj
ee.agent_by_hostname[agent_obj.hostname] = agent_obj
self._ees_dict[ee.id] = ee
self._logger.info("EE:\n{}".format(ee))
def _build_ion_obj(self, ion_json_dict):
ion_builder = IONBuilder()
self._ion = ion_builder.build_from_dict(ion_json_dict)
# This info line is important so we can understand what was the ion structure if errors happens at customer site
self._info("{}:\n{}".format(Constants.ION_LITERAL, self._ion))
self._info("---------------------")
def _build_health_obj(self, health_json_dict):
self._ion.policy.set_thresholds(health_json_dict[IONJsonConstants.ION_GLOBAL_THRESHOLD_TAG], health_json_dict[IONJsonConstants.ION_CANARY_THRESHOLD_TAG])
def rest_helper(self):
return self._rest_helper
def ion(self):
"""
Return a copy of the ion object
:return: ION object
:rtype: ION
"""
return copy.deepcopy(self._ion)
def ion_id(self):
"""
Return the ION id
:return:
"""
return self._ion.id
def ion_node_id(self):
"""
Return the current node id this code is running in
:return: ION node id
"""
return self._ci.ion_node_id
def current_node(self):
if self._ci.ion_node_id not in self._ion.node_by_id:
raise MLOpsException("Current node id: [{}] is not detected in {}".format(
self._ci.ion_node_id, Constants.ION_LITERAL))
return self._ion.node_by_id[self._ci.ion_node_id]
def ion_name(self):
"""
Return the ION name
:return:
"""
return self._ion.name
def ion_policy(self):
"""
Return the ION policy
:return:
"""
return self._ion.policy
def ion_nodes(self):
"""
Return a list of ION components
:return:
"""
return copy.deepcopy(self._ion.nodes)
def get_ion_node(self, name):
"""
Return a component object given its name
:param name:
:return: Component object matching name (if found), None if not found
"""
self._debug("Getting {} component [{}]".format(Constants.ION_LITERAL, name))
self._debug("{} comps by name: {}".format(Constants.ION_LITERAL, self._ion.node_by_name))
if name in self._ion.node_by_name:
return self._ion.node_by_name[name]
return None
def get_ion_node_by_pipeline_instance(self, pipe_instance_id):
"""
Return a list of ion_nodes for the given pipeline instance.
:param pipe_instance_id: id of the pipeline
:return: ion_nodes or None if not found
"""
if pipe_instance_id in self._ion.node_by_pipe_inst_id:
return self._ion.node_by_pipe_inst_id[pipe_instance_id]
return None
def get_ion_node_agents(self, node):
"""
Return a list of agents for the given ion_node
:param node: a node within the ION
:return: List of agents for the given ion_node
:raises MLOpsException for invalid arguments
"""
# Getting by component name
if isinstance(node, six.string_types):
if node not in self._ion.node_by_name:
raise MLOpsException("Node {} is not part of current {}".format(
node, Constants.ION_LITERAL))
node_obj = self._ion.node_by_name[node]
if node_obj.ee_id not in self._ees_dict:
raise MLOpsException("Component {} had ee_id {} which is not part of valid ees".format(
node_obj.name, node_obj.ee_id))
ee_obj = self._ees_dict[node_obj.ee_id]
# Note: calling deepcopy in order for the user to get a copy of agents objects and not point to internal
# data
agent_list = copy.deepcopy(ee_obj.agents)
return agent_list
else:
raise MLOpsException("component argument should be component name (string)")
def get_agent_by_id(self, agent_id):
"""
Return agent object by ID, assuming the agent is part of the current ION
:param agent_id: Agent Id to search for
:type agent_id: str
:return: agent_id or None if not found
"""
if agent_id in self._agents_dict:
return self._agents_dict[agent_id]
return None
def done(self):
if self._rest_helper is not None:
self._rest_helper.done()