def get_ion_node_agents(self, node):
"""
Return a list of agents for the given ion_node
:param node: a node within the ION
:return: List of agents for the given ion_node
:raises MLOpsException for invalid arguments
"""
# Getting by component name
if isinstance(node, six.string_types):
if node not in self._ion.node_by_name:
raise MLOpsException("Node {} is not part of current {}".format(
node, Constants.ION_LITERAL))
node_obj = self._ion.node_by_name[node]
if node_obj.ee_id not in self._ees_dict:
raise MLOpsException("Component {} had ee_id {} which is not part of valid ees".format(
node_obj.name, node_obj.ee_id))
ee_obj = self._ees_dict[node_obj.ee_id]
# Note: calling deepcopy in order for the user to get a copy of agents objects and not point to internal
# data
agent_list = copy.deepcopy(ee_obj.agents)
return agent_list
else:
raise MLOpsException("component argument should be component name (string)")
def _detect_mlops_server_via_zk(self):
"""
Detect the active mlops server via the ZK
:return:
"""
zk = None
try:
zk = KazooClient(hosts=self._ci.zk_host, read_only=True)
zk.start()
if zk.exists(Constants.MLOPS_ZK_ACTIVE_HOST_PORT):
data, stat = zk.get(Constants.MLOPS_ZK_ACTIVE_HOST_PORT)
eco_host_port = data.decode("utf-8").split(':')
if len(eco_host_port) is 2:
self._ci.mlops_server = eco_host_port[0]
self._ci.mlops_port = eco_host_port[1]
else:
raise MLOpsException("Internal Error: Invalid zookeeper active server entry, host_port: {}"
.format(eco_host_port))
else:
raise MLOpsException("Unable to connect to the active MLOps server, zk_host: {}"
.format(self._ci.zk_host))
except Exception as e:
raise MLOpsException("{}, zk_host: {}".format(e, self._ci.zk_host))
finally:
if zk:
zk.stop()
def set_event(self, name, type=None, data=None, is_alert=False, timestamp=None):
"""
Generate an event which is sent to MLOps.
:param name: name for the event
:param type: type of :class:`Event`
:param data: Any python object, which will be serialized (pickle) and stored in the PM System. It can later
be fetched for further manipulation.
:param is_alert: Boolean indicating whether this event is an alert.
:param timestamp: optional datetime.datetime timestamp. If None, the current timestamp will be provided.
:return: The current mlops instance for further calls
:raises: MLOpsException
"""
self._verify_mlops_is_ready()
if name is None:
raise MLOpsException("Name argument must be a string or an event object")
if isinstance(name, six.string_types):
if type is None:
raise MLOpsException("Type of event can not be None")
event_obj = Event(label=name, event_type=type, description=None, data=data,
is_alert=is_alert, timestamp=timestamp)
elif isinstance(name, Event):
event_obj = name
else:
raise MLOpsException("Name argument can be a string or event object only")
if self._api_test_mode:
self._logger.info("API testing mode - returning without performing call")
return
self._event_broker.send_event(event_obj)
return self
def stat(self,
name,
data,
model_id,
category=None,
model=None,
model_stat=None):
if category in (StatCategory.CONFIG, StatCategory.TIME_SERIES):
self._logger.debug(
"{} stat called: name: {} data_type: {} class: {}".format(
Constants.OFFICIAL_NAME, name, type(data), category))
single_value = SingleValue().name(name).value(data).mode(category)
self.stat_object(single_value.get_mlops_stat(model_id))
elif category is StatCategory.INPUT:
self._logger.info(
"{} stat called: name: {} data_type: {} class: {}".format(
Constants.OFFICIAL_NAME, name, type(data), category))
# right now, only for ndarray or dataframe, we can generate input stats.
if isinstance(data, ndarray):
dims = data.shape
feature_length = 1
try:
feature_length = dims[1]
except Exception as e:
data = data.reshape(dims[0], -1)
pass
array_of_features = []
for i in range(feature_length):
array_of_features.append("c" + str(i))
features_values = data
features_names = array_of_features
elif isinstance(data, pd.DataFrame):
features_values = np.array(data.values)
features_names = list(data.columns)
else:
raise MLOpsException(
"stat_class: {} not supported yet for data: {}".format(
category, data))
self._logger.info("model_stat is: {}".format(model_stat))
PythonChannelHealth.generate_health_and_heatmap_stat(
stat_object_method=self.stat_object,
logger=self._logger,
features_values=features_values,
features_names=features_names,
model_stat=model_stat,
model_id=model_id,
num_bins=13)
else:
raise MLOpsException(
"stat_class: {} not supported yet".format(category))
def check_list_of_str(list_of_str, error_prefix="Columns names"):
if not isinstance(list_of_str, list):
raise MLOpsException(
"{} should be provided as a list".format(error_prefix))
if not all(isinstance(item, six.string_types) for item in list_of_str):
raise MLOpsException("{} should be strings".format(error_prefix))
def __init__(self,
label,
description=None,
data=None,
event_type=ReflexEvent.GenericEvent,
is_alert=False,
timestamp=None):
# TODO: make sure event_type is an event type and not something else
if not isinstance(label, six.string_types):
raise MLOpsException(
"Invalid 'label' type! Expecting string! label: " +
type(label))
if description is None:
description = ""
if not isinstance(description, six.string_types):
raise MLOpsException(
"Invalid 'description' type! Expecting string! desc: " +
description)
self.type = event_type
self.label = label
self.description = description
self.data = str(pickle.dumps(data))
self.is_alert = is_alert
self.timestamp = timestamp
def _assemble_nodes_and_agents(self, ion, node, agent):
# Assemble a list of nodes and for each node the list of agents.
agents_per_node = OrderedDict()
if node is not None:
node_obj = self.get_node(node)
if node_obj is None:
msg = "{} Node: [{}] is not present in {}".format(Constants.ION_LITERAL, node,
Constants.ION_LITERAL)
self._logger.error(msg)
raise MLOpsException(msg)
if agent is None:
agents_per_node[node_obj.name] = self.get_agents(node_obj.name)
else:
if isinstance(agent, six.string_types):
agent_id = agent
else:
agent_id = agent.id
agent_obj = self.get_agent(node_obj.name, agent_id)
if agent_obj is None:
raise MLOpsException("Agent: {} is not found in node {}".format(agent, node_obj.name))
agents_per_node[node_obj.name] = [agent_obj]
else:
node_list = ion.node_by_id.values()
for node_obj in node_list:
agents_per_node[node_obj.name] = self.get_agents(node_obj.name)
return agents_per_node
def post_model_as_file(self, model_file_path, params, metadata):
"""
Posts a file to the server
:param model_file_path: model file to upload
:param params: parameters dictionary
:param metadata: extended metadata(currently not used with rest connected)
:return: model_id
"""
if metadata and not isinstance(metadata, ModelMetadata):
raise MLOpsException(
"metadata argument must be a ModelMetadata object, got {}".
format(type(metadata)))
required_params = ["modelId"]
for param_name in required_params:
if param_name not in params:
raise MLOpsException(
'parameter {} is required for publishing model'.format(
param_name))
model_id = params["modelId"]
if metadata:
model_meta_file = os.path.join(self._meta_dir, model_id)
with open(model_meta_file, 'w') as outfile:
json.dump(metadata.to_dict(), outfile)
model_data_file = os.path.join(self._data_dir, model_id)
with open(model_data_file, "w+") as data_file:
# maybe should be changed to shutil.copyfile
data_file.write(open(model_file_path, "r").read())
return model_id
def __init__(self,
modelId,
name="",
model_format=ModelFormat.UNKNOWN,
description="",
user_defined="",
size=0):
if model_format and not isinstance(model_format, ModelFormat):
raise MLOpsException(
"model_format object must be an instance of ModelFormat class! provided: "
"{}, type: {}".format(model_format, type(model_format)))
if model_format == ModelFormat.UNKNOWN:
raise MLOpsException(
"model_format can not be {}. Did you forget to set a format for model?"
.format(model_format.value))
self.modelId = modelId
self.name = name
self.modelFormat = model_format
self.description = description
self.user_defined = user_defined
self.size = size
# these fields are set by MCenter server
self.source = ""
self.user = ""
self.state = ""
self.createdTimestamp = ""
self.workflowRunId = ""
def post_event(self, pipeline_inst_id, event):
"""
Post event using protobuf format
:param pipeline_inst_id: pipeline instance identifier
:param event: ReflexEvent based on protobuf
:return: Json response from agent
"""
url = build_url(self._mlops_server, self._mlops_port,
MLOpsRestHandles.EVENTS, pipeline_inst_id)
try:
payload = encoder(event)
headers = {"Content-Type": "application/json;charset=UTF-8"}
r = requests.post(url,
data=payload,
headers=headers,
cookies=self._return_cookie())
if r.ok:
return r.json()
else:
raise MLOpsException(
'Call {} with payload {} failed text:[{}]'.format(
url, event, r.text))
except requests.exceptions.ConnectionError as e:
self._error(e)
raise MLOpsException(
"Connection to MLOps agent [{}:{}] refused".format(
self._mlops_server, self._mlops_port))
except Exception as e:
raise MLOpsException('Call ' + str(url) + ' failed with error ' +
str(e))
def verify_table_from_list_of_lists(tbl_data):
"""
Verify that tbl_data is in the format expected
:param tbl_data:
:return: Throws an exception in case of badly formatted table data
"""
if not isinstance(tbl_data, list):
raise MLOpsException(
"Table data is not in the format of list of lists")
row_idx = 0
for item in tbl_data:
if not isinstance(item, list):
raise MLOpsException(
"Each item of tbl_data should be a list - row {} is not a list"
.format(row_idx))
row_idx += 1
if len(tbl_data) < 2:
raise MLOpsException(
"Table data must contain at least column names and one row of data"
)
nr_cols = len(tbl_data[0])
row_idx = 0
for row in tbl_data:
if len(row) != nr_cols:
raise MLOpsException(
"Row {} items number is {} which is not equal to number of columns provided {}"
.format(row_idx, len(row), nr_cols))
row_idx += 1
def feature_importance(self,
feature_importance_vector=None,
feature_names=None,
model=None,
df=None):
self._validate_specific_importance_inputs(df)
# Get the feature importance vector
if feature_importance_vector:
feature_importance_vector_final = feature_importance_vector
else:
try:
feature_importance_vector_final = model.feature_importances_
except Exception as e:
raise MLOpsException("Got an exception:{}".format(e))
if feature_names:
important_named_features = [[
name, feature_importance_vector_final[imp_idx]
] for imp_idx, name in enumerate(feature_names)]
return important_named_features
else:
try:
feature_names = df.columns[1:]
important_named_features = [[
name, feature_importance_vector_final[imp_idx]
] for imp_idx, name in enumerate(feature_names)]
return important_named_features
except Exception as e:
raise MLOpsException("Got an exception:{}".format(e))
def add_row(self, arg1, arg2=None):
"""
Add a row to the table.
:param arg1: Name of the row, or list of data items
:param arg2: List of data items (if argument 1 was provided)
:return: self
"""
row_name = ""
row_data = []
if isinstance(arg1, six.string_types):
# The case where we get the name of the row as the first argument
row_name = copy.deepcopy(arg1)
if arg2 is None:
raise MLOpsException("no data provided for row")
if not isinstance(arg2, list):
raise MLOpsException("Data should be provided as a list")
row_data = copy.deepcopy(arg2)
elif isinstance(arg1, list):
# The case where we get only data without the line/row name
row_data = copy.deepcopy(arg1)
else:
raise MLOpsException("Either provide row_name, data or just data")
if len(self._tbl_rows) > 0:
if len(self._tbl_rows[0]) != len(row_data):
raise MLOpsException(
"row length must be equal to length of previously provided rows"
)
self._tbl_rows.append(row_data)
self._rows_names.append(row_name)
return self
def verify_json_content(fname, expected_content):
first_line_dict = json.load(open(fname, 'r'))
for key in expected_content:
if key not in first_line_dict:
raise MLOpsException("File {} did not have key {}".format(
fname, key))
elif key == "Value":
expected_values = expected_content[key]
first_line_values = first_line_dict[key]
sub_keys = [
"Data", "GraphType", "Mode", "ShouldTransfer", "TransferType"
]
for sub_key in sub_keys:
if expected_values[sub_key] != first_line_values[sub_key]:
raise MLOpsException("{} did not match: {} {}".format(
sub_key, first_line_values[sub_key],
expected_values[sub_key]))
elif expected_content[key] != first_line_dict[key]:
raise MLOpsException(
"File {} had a value {} for key {} that did not match {}".
format(fname, first_line_dict[key], key,
expected_content[key]))
def verify_csv_content(fname, first_line, expected_content):
first_line = first_line.strip().split(", ")
expected_content = expected_content.strip().split(", ")
first_line_minus_json = first_line[1:4]
expected_content_minus_json = expected_content[1:4]
for i, (produced_elem, expected_elem) in enumerate(
zip(first_line_minus_json, expected_content_minus_json)):
if produced_elem != expected_elem:
raise MLOpsException(
"File {} has unexpected elem: {}. Expected {}".format(
fname, produced_elem, expected_elem))
first_line_json_str = ', '.join(first_line[4:]).replace("'", "\"")
expected_content_json_str = ', '.join(expected_content[4:]).replace(
"'", "\"")
first_line_json = json.loads(first_line_json_str)
expected_content_json = json.loads(expected_content_json_str)
if first_line_json != expected_content_json:
raise MLOpsException(
"File {} has unexpected json: {}. Expected {}".format(
fname, first_line_json, expected_content_json))
def annotate(self, label, x=None, y=None):
"""
Add an annotation to a point on a line
:param label: annotation to add
:param x: x value for the annotation
:param y: y value for the annotation
:return: self
"""
if self._x_series is None:
raise MLOpsException(
"Can not add annotations before setting x_series data")
if x is not None and y is not None:
raise MLOpsException("Annotation can be either x or y annotation")
if x is not None:
if not isinstance(x, (six.integer_types, float)):
raise MLOpsException("x argument should be a number")
self._x_annotations.append({"label": label, "value": x})
elif y is not None:
if not isinstance(y, (six.integer_types, float)):
raise MLOpsException("y argument should be a number")
self._y_annotations.append({"label": label, "value": y})
else:
raise MLOpsException("Annotation must provide an axis")
return self
def untar_timeline_capture(self):
"""
The function untars the timeline capture file to a local folder and saves the file names
into a list and the files themselves into a dict
:param self:
:return:
"""
try:
print("self._input_timeline_capture", self._input_timeline_capture)
print("self._tmpdir", self._tmpdir)
with tarfile.open(self._input_timeline_capture) as tar_obj:
tar_obj.extractall(self._tmpdir)
except Exception as e:
print("Unable to open the timeline capture file")
raise MLOpsException(e)
self._file_names = os.listdir(self._extracted_dir)
try:
for file_name in self._file_names:
if 'csv' in file_name:
with open(self._extracted_dir + file_name, 'r') as f:
reader = csv.reader(f)
parsed_list = list(reader)
self._timeline_capture[file_name] = parsed_list[1:-1]
self._timeline_capture[file_name].append(parsed_list[-1])
self._timeline_capture[file_name +
'header'] = parsed_list[0]
except Exception as err:
self._timeline_capture = {}
raise MLOpsException(err)
def check_vec_of_numbers(vec_of_numbers, error_prefix="Data"):
if not isinstance(vec_of_numbers, list):
raise MLOpsException("{} should be a list got {}".format(
error_prefix, type(vec_of_numbers)))
if not all(
isinstance(item, (six.integer_types, float))
for item in vec_of_numbers):
raise MLOpsException(
"{} should be a list of int or floats".format(error_prefix))
def stat(self,
name,
data,
modelId,
category=None,
model=None,
model_stat=None):
if category in (StatCategory.CONFIG, StatCategory.TIME_SERIES):
self._logger.info(
"{} stat called: name: {} data_type: {} class: {}".format(
Constants.OFFICIAL_NAME, name, type(data), category))
stat_mode, graph_type = self.resolve_type(data, category)
self._jvm_mlops.stat(name, data, modelId, graph_type, stat_mode)
elif category is StatCategory.INPUT:
if model_stat:
hist_rdd = self._sc.parallelize(model_stat)
else:
hist_rdd = self._sc.emptyRDD()
if isinstance(data, pyspark.rdd.PipelinedRDD):
self._jvm_mlops.inputStatsFromRDD(
name, modelId, ml._py2java(self._sc, data),
ml._py2java(self._sc, hist_rdd))
elif isinstance(data, pyspark.sql.DataFrame):
try:
self._logger.info(
"Spark ML is provided to help MCenter calculate Health!"
)
if model:
spark_ml_model = model._to_java()
self._jvm_mlops.inputStatsFromDataFrame(
name, modelId, ml._py2java(self._sc, data),
ml._py2java(self._sc, hist_rdd), spark_ml_model)
else:
self._jvm_mlops.inputStatsFromDataFrame(
name, modelId, ml._py2java(self._sc, data),
ml._py2java(self._sc, hist_rdd))
except Exception as e:
self._logger.info(
"model does not seem to have _to_java method \n{}".
format(e))
raise MLOpsException(
"Unable to convert from python to java {}".format(e))
else:
raise MLOpsException(
"Statistic type {} is not supported for data type {}".
format(StatCategory.INPUT, type(data)))
else:
raise MLOpsException(
"stat_class: {} not supported yet".format(category))
def post_model_as_file(self, model_file_path, params, metadata):
"""
Posts a file to the server
:param model_file_path: model file to upload
:param params: parameters dictionary
:param metadata: extended metadata(currently not used with rest connected)
:return: model_id
"""
if metadata and not isinstance(metadata, ModelMetadata):
raise MLOpsException(
"metadata argument must be a ModelMetadata object, got {}".
format(type(metadata)))
required_params = [
"modelName", "modelId", "format", "workflowInstanceId",
"pipelineInstanceId", "description"
]
for param_name in required_params:
if param_name not in params:
raise MLOpsException(
'parameter {} is required for publishing model'.format(
param_name))
url = build_url(self._mlops_server, self._mlops_port,
MLOpsRestHandles.MODELS, params["pipelineInstanceId"])
files = {
'file': ("file", open(model_file_path,
'rb'), "application/octet-stream")
}
try:
r = requests.post(url,
files=files,
params=params,
cookies=self._return_cookie())
if r.ok:
return r.json()
else:
raise MLOpsException(
'Call {} with filename {} failed text:[{}]'.format(
url, model_file_path, r.text))
except requests.exceptions.ConnectionError as e:
self._error(e)
raise MLOpsException(
"Connection to MLOps server [{}:{}] refused".format(
self._mlops_server, self._mlops_port))
except Exception as e:
raise MLOpsException('Call ' + str(url) + ' failed with error ' +
str(e))
def _validate_config(self):
"""
Validate that all config information is present
:return:
:raises MLOpsException for invalid configurations
"""
if self._ci.token is None:
raise MLOpsException("Internal Error: No auth token provided")
if self._ci.mlops_server is None or self._ci.mlops_port is None:
raise MLOpsException("MLOps server host or port were not provided")
if self._ci.ion_id is None:
MLOpsException("{} instance id not provided".format(Constants.ION_LITERAL))
def set_kpi(self, name, data, model_id, timestamp, units):
if not isinstance(name, six.string_types):
raise MLOpsException("name argument must be a string")
if not isinstance(data, (six.integer_types, float)):
raise MLOpsException("KPI data must be a number")
kpi_value = KpiValue(name, data, timestamp, units)
if self._api_test_mode:
self._logger.info("API testing mode - returning without performing call")
return
self._output_channel.stat_object(kpi_value.get_mlops_stat(model_id))
def post_model_as_file(self, model):
"""
Posts a file to the server
:param model: :class:`Model` object to publish
:return: model_id
"""
request_params = {
models.json_fields.MODEL_NAME_FIELD:
model.metadata.name,
models.json_fields.MODEL_ID_FIELD:
model.metadata.modelId,
models.json_fields.MODEL_FORMAT_FIELD:
model.metadata.modelFormat.value,
models.json_fields.MODEL_DESCRIPTION_FIELD:
model.metadata.description,
models.json_fields.MODEL_ANNOTATIONS_FIELD:
json.dumps(model.get_annotations()),
Constants.PIPELINE_INSTANCE_ID:
model._pipeline_instance_id
}
url = build_url(self._mlops_server, self._mlops_port,
MLOpsRestHandles.MODELS, model._pipeline_instance_id)
files = {
'file': ("file", open(model._path_to_publish,
'rb'), "application/octet-stream")
}
try:
r = requests.post(url,
files=files,
params=request_params,
cookies=self._return_cookie())
if r.ok:
return r.json()
else:
raise MLOpsException(
'Call {} with filename {} failed text:[{}]'.format(
url, model_file_path, r.text))
except requests.exceptions.ConnectionError as e:
self._error(e)
raise MLOpsException(
"Connection to MLOps server [{}:{}] refused".format(
self._mlops_server, self._mlops_port))
except Exception as e:
raise MLOpsException('Call ' + str(url) + ' failed with error ' +
str(e))
def _validate_specific_importance_inputs(self, model, df):
"""
:param model: sparkML pipeline model
:param df: sparkML dataframe
:return:
"""
if PipelineModel is not None and not isinstance(model, PipelineModel):
raise MLOpsException(
"Got an Exception. should be a pipeline model")
if df is not None and not isinstance(df, DataFrame):
raise MLOpsException(
"Got an Exception. should be a sparkML dataframe")
def __init__(self, stats_helper, rest_helper, name, model_format, description, id=None):
super(Model, self).__init__(rest_helper, id)
model_id = self.get_id()
if model_id is None or not isinstance(model_id, six.string_types) or model_id == "":
raise MLOpsException('model id must be non zero valid string type, received: {}'.format(model_id))
self.model_path = None
self.metadata = ModelMetadata(self.get_id(), name, model_format, description)
if stats_helper and not isinstance(stats_helper, StatsHelper):
raise MLOpsException("stats_helper object must be an instance of StatsHelper class")
self._stats_helper = stats_helper
self._pipeline_instance_id = None
self._path_to_publish = None
def labels(self, label_list):
"""
Set the labels for this multi-line graph
:param label_list: List of strings to use as labels
:return: self
"""
if not isinstance(label_list, list):
raise MLOpsException("Labels should be provided as a list")
if not all(isinstance(item, six.string_types) for item in label_list):
raise MLOpsException("Labels should be strings")
self._label_list = copy.deepcopy(label_list)
return self
def value(self):
if self._graph_type == StatGraphType.LINEGRAPH:
return {self._key: self._value}
else:
raise MLOpsException(
"Value graph type is not supported! key: {}, value: {}, graph_type: {}"
.format(self._key, self._value, self._graph_type))
def post_stat(self, pipeline_inst_id, stat):
"""
Post stat to agent using json formatting
:param pipeline_inst_id: pipeline instance identifier
:param stat: stat in json format
:return: Json response from agent
"""
if pipeline_inst_id is None:
self._error("Missing pipeline instance id cannot post stat")
return
url = self.url_post_stat(pipeline_inst_id)
try:
payload = encoder(stat)
headers = {"Content-Type": "application/json;charset=UTF-8"}
r = requests.post(url,
data=payload,
headers=headers,
cookies=self._return_cookie())
if r.ok:
return r.json()
else:
self._error('Call {} with payload {} failed: text:[{}]'.format(
url, stat, r.text))
except requests.exceptions.ConnectionError as e:
self._error(e)
raise MLOpsConnectionException(
"Connection to MLOps agent [{}:{}] refused; {}".format(
self._mlops_server, self._mlops_port, e))
except Exception as e:
raise MLOpsException('Call ' + str(url) + ' failed with error ' +
str(e))
def get_model_by_id(self, model, download=False):
"""
Return the model with this id
:param model: the model id
:param download: Boolean, whether to download the model
:return: return model as a bytearray
:raises: MLOpsException
"""
self._verify_mlops_is_ready()
if isinstance(model, six.string_types):
model_id = model
elif isinstance(model, Model):
model_id = model.id
else:
raise MLOpsException("model parameter can be either a string or of class Model: got [{}]".format(
type(model)))
if self._api_test_mode:
self._logger.info("API testing mode - returning without performing call - in {}".format(
inspect.stack()[0][3]))
model_filter = ModelFilter()
model_filter.id = model_id
model_df = self._model_helper.get_models_dataframe(model_filter=model_filter, download=download)
return model_df
def get_mlops_stat(self, model_id):
if self._x_series is None:
raise MLOpsException("No x_series was provided")
if len(self._y_series) == 0:
raise MLOpsException("At leat one y_series should be provided")
tbl_data = self._to_dict()
mlops_stat = MLOpsStat(name=self._name,
stat_type=InfoType_pb2.General,
graph_type=StatGraphType.GENERAL_GRAPH,
mode=StatsMode.Instant,
data=tbl_data,
model_id=model_id)
return mlops_stat
