def test_opaque():
test_cycle = _get_curr_test_cycle()
print("OPAQUE-TEST: {}".format(test_cycle))
obj = {"1": "aaaaaa", "2": 33}
stat_name = "opq-1"
opq = Opaque().name(stat_name).data(obj)
mlops.set_stat(opq)
print("Done reporting opaque statistics")
time.sleep(10)
now = datetime.utcnow()
last_day = (now - timedelta(hours=24))
df = mlops.get_stats(name=stat_name,
start_time=last_day,
end_time=datetime.utcnow(),
mlapp_node=None,
agent=None)
assert len(df) >= (
test_cycle + 1), "Got: {} lines in df, expecting at least: {}".format(
len(df), 1)
for idx in range(0, test_cycle + 1):
data = df.iloc[idx]["data"]
assert data["1"] == obj["1"]
assert data["2"] == obj["2"]
def test_multiline():
print("Testing multiline")
data = [[5, 15, 20], [55, 155, 255], [75, 175, 275]]
columns = ["a", "b", "c"]
expected_df = pd.DataFrame(data, columns=columns)
stat_name = "stat-multi-line-test"
# Multi line graphs
for row in data:
mlt = MultiLineGraph().name(stat_name).labels(columns).data(row)
mlops.set_stat(mlt)
time.sleep(10)
agent_list = _get_my_agents()
now = datetime.utcnow()
last_hour = (now - timedelta(hours=1))
df = mlops.get_stats(name=stat_name,
mlapp_node=mlops.get_current_node().name,
agent=agent_list[0].id,
start_time=last_hour,
end_time=now)
print("Got multiline df\n", df)
df_tail = pd.DataFrame(df.tail(len(data))).reset_index(drop=True)
print("df_tail:\n", df_tail)
df_only_cols = df_tail[columns]
print("Tail of multiline df + only relevant cols\n", df_only_cols)
print("Expected:\n", expected_df)
# We expect the tail (last rows) of the result df to be like the expected df.
assert expected_df.equals(
df_only_cols
), "Expected multiline is not equal to obtained\n-------\n{}\n-------\n{}\n".format(
expected_df, df_only_cols)
def test_kpi_basic():
sec_now = int(time.time())
sec_4h_ago = sec_now - (3600 * 4)
kpi_window_start = sec_4h_ago
val = 3.56
nr_kpi_point = 10
kpi_name = "test-kpi-1"
kpi_window_end = kpi_window_start
for i in range(nr_kpi_point):
mlops.kpi(kpi_name, val, kpi_window_end, KpiValue.TIME_SEC)
kpi_window_end += 1
val += 1
time.sleep(5)
agent_list = _get_my_agents()
kpi_datetime_start = datetime.utcfromtimestamp(kpi_window_start)
kpi_datetime_end = datetime.utcfromtimestamp(kpi_window_end)
print("datetime start: {}".format(kpi_datetime_start))
print("datetime end: {}".format(kpi_datetime_end))
df = mlops.get_stats(stat_name=kpi_name,
ion_component=mlops.ion_node_id,
agent=agent_list[0].id,
start_time=kpi_datetime_start,
end_time=kpi_datetime_end)
print(df)
if len(df) != nr_kpi_point:
raise Exception("Got: {} kpi points, expecting: {}".format(
len(df), nr_kpi_point))
def test_stats_basic():
# Adding multiple points (to see a graph in the ui), expecting each run to generate 8 points
mlops.stat("stat1", 1.0, st.TIME_SERIES)
mlops.stat("stat1", 3.0, st.TIME_SERIES)
mlops.stat("stat1", 4.0, st.TIME_SERIES)
mlops.stat("stat1", 2.0, st.TIME_SERIES)
mlops.stat("stat1", 5.0, st.TIME_SERIES)
mlops.stat("stat1", 6.0, st.TIME_SERIES)
mlops.stat("stat1", 7.0, st.TIME_SERIES)
mlops.stat("stat1", 8.0, st.TIME_SERIES)
print("Done reporting statistics")
time.sleep(10)
print("MyION = {}".format(mlops.ion_id))
now = datetime.utcnow()
last_hour = (now - timedelta(hours=1))
print("Hour before: {}".format(last_hour))
print("Now: {}".format(now))
agent_list = _get_my_agents()
df = mlops.get_stats(stat_name="stat1",
ion_component=mlops.ion_node_id,
agent=agent_list[0].id,
start_time=last_hour,
end_time=now)
print("Got stat1 statistic\n", df)
def _get_curr_test_cycle():
now = datetime.utcnow()
last_4hour = (now - timedelta(hours=4))
df = mlops.get_stats(name=E2EConstants.E2E_RUN_STAT,
mlapp_node=None,
agent=None,
start_time=last_4hour,
end_time=now)
return len(df)
def test_model_stats():
print("MODEL-STATS-TEST")
now = datetime.utcnow()
last_10_hour = (now - timedelta(hours=10))
models_df = mlops.get_models_by_time(start_time=last_10_hour, end_time=now)
print(models_df)
if len(models_df) == 0:
print("No models yet - skipping for now")
return
# Taking the last model generated
tail_df = models_df.tail(1)
print("tail_models_stat:{}".format(tail_df))
model_id = tail_df.iloc[0]['id']
print("model_id: [{}]".format(model_id))
is_model_stats_reported_df = mlops.get_stats(
name=E2EConstants.MODEL_STATS_REPORTED_STAT_NAME,
mlapp_node=None,
agent=None,
start_time=last_10_hour,
end_time=now)
# TODO: we need to somehow indicate that this test was actually skipped and not passing
if len(is_model_stats_reported_df) > 0:
# Only in this case we check the model statistics
model_stats = mlops.get_data_distribution_stats(model_id)
assert len(model_stats) > 0
print("model_stats:\n{}".format(model_stats))
# Getting the attributes reported in the histograms and comparing to the expected list of attributes
existing_attributes_set = set(model_stats['stat_name'].tolist())
diff_set = existing_attributes_set ^ E2EConstants.MODEL_STATS_EXPECTED_ATTR_SET
print("existing attr: {}".format(existing_attributes_set))
print("expected attr: {}".format(
E2EConstants.MODEL_STATS_EXPECTED_ATTR_SET))
print("diff_set: {}".format(diff_set))
assert len(diff_set) == 0
def test_get_stats_api():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
pm._set_api_test_mode()
now = datetime.utcnow()
last_hour = (now - timedelta(hours=1))
with pytest.raises(MLOpsException):
pm.get_stats(name=None,
mlapp_node="33",
agent=1,
start_time=None,
end_time=None)
with pytest.raises(MLOpsException):
pm.get_stats(name="stat_1",
mlapp_node="33",
agent="aaa",
start_time=None,
end_time=None)
with pytest.raises(MLOpsException):
pm.get_stats(name="stat_1",
mlapp_node="33",
agent="aaa",
start_time=last_hour,
end_time=None)
with pytest.raises(MLOpsException):
pm.get_stats(name="stat_1",
mlapp_node="33",
agent="aaa",
start_time=None,
end_time=now)
with pytest.raises(MLOpsException):
pm.get_stats(name="stat_1",
mlapp_node="33",
agent=1,
start_time=last_hour,
end_time=now)
with pytest.raises(MLOpsException):
pm.get_stats(name="stat_1",
mlapp_node=None,
agent=None,
start_time=now,
end_time=last_hour)
agent_obj = Agent()
pm.get_stats(name="stat_1",
mlapp_node="1",
agent=agent_obj,
start_time=last_hour,
end_time=now)
with pytest.raises(MLOpsException):
pm.get_stats(name="stat_1",
mlapp_node=None,
agent=agent_obj,
start_time=last_hour,
end_time=now)
pm.get_stats(name="stat_1",
mlapp_node=None,
agent=None,
start_time=last_hour,
end_time=now)
pm.done()
def main():
options = parse_args()
token = options.token
sc = SlackClient(token)
is_alert = options.alert == 'True'
chid = get_channel_id(sc, options.channel_name)
if chid is None:
print("Channel <{}> not found".format(options.channel_name))
return
mlops.init()
last_time = -1
try:
now = datetime.utcnow()
hour_ago = (now - timedelta(hours=1))
node = mlops.get_current_node()
agents = get_my_agents()
df = mlops.get_stats(name="last_time_slack",
mlapp_node=node.name,
agent=agents[0].id,
start_time=hour_ago,
end_time=now)
if df.empty is False:
last_time = df.iloc[[-1]].value.values[0]
print("Last message retrieved at {} msec".format(last_time))
except Exception:
print(traceback.format_exc())
pass
if last_time == -1:
evts = mlops.get_events(is_alert=is_alert)
else:
# expects in sec, convert msec to sec
utc_last = datetime.utcfromtimestamp(last_time / 1000)
evts = mlops.get_events(start_time=utc_last,
end_time=datetime.utcnow(),
is_alert=is_alert)
if evts is None or len(evts) == 0:
print("No events yet found in the time range")
return
last_row = evts.iloc[[-1]]
if last_row.created.values[0] == last_time:
print("No new events since {} msec".format(
datetime.utcfromtimestamp(last_time / 1000)))
return
else:
print("New events last {} msec now {} msec".format(
datetime.utcfromtimestamp(last_time / 1000),
datetime.utcfromtimestamp(last_row.created.values[0] / 1000)))
text = """
[
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "TEXTKEY"
},
"accessory": {
"type": "image",
"image_url": "https://media.licdn.com/dms/image/C4E0BAQEHfXCM5IqzeQ/company-logo_400_400/0?e=1559174400&v=beta&t=GxnMnlESAz0XDg9et80kGOrTV6mg0NpxdnttKYs63Jo",
"alt_text": "ParallelM MCenter"
}
}
]"""
for index, row in evts.iterrows():
#Sample event
#{
# "clearedTimestamp":0,
# "created":1550812504925,
# "createdBy":"admin",
# "createdTimestamp":1550812504056,
# "deletedTimestamp":0,
# "description":"KB",
# "eventType":"ModelAccepted",
# "host":"daenerys-c28",
# "id":"9c9ec480-d950-4d4a-acab-774c685c5aa0",
# "ionName":"kab slack",
# "modelId":"469d47bf-554b-4279-81c9-8dcfb6039494",
# "msgType":"UNKNOWN",
# "name":"event-2756",
# "pipelineInstanceId":"2e0208c1-85c3-41c8-9f68-e2a175d6033d",
# "raiseAlert":false,
# "reviewedBy":null,
# "sequence":2756,
# "state":null,
# "stateDescription":null,
# "type":"ModelAccepted",
# "workflowRunId":"1c75a622-05c9-481f-ae87-33a379800b52",
# "node":"2"
#}
jsons = json.loads(row.to_json())
if is_alert and jsons['raiseAlert'] is False:
continue
if jsons['raiseAlert']:
alert_message = ":warning:"
else:
alert_message = ":ok_hand:"
text_message =\
"*MLApp Name*: {}\n"\
"*Description*: {}\n"\
"*Event Type*: {}\n"\
"*Message Type*: {}\n"\
"*Alert*: {}\n"\
"*Created At* :clock12: : {}\n"\
"*Host*: {}\n".format(
jsons['ionName'],
jsons['description'],
jsons['eventType'],
jsons['msgType'],
alert_message,
datetime.utcfromtimestamp(jsons['created']/1000),
jsons['host'])
if jsons['eventType'] == "ModelAccepted":
text_message = text_message +\
"*Model ID*: {}\n".format(jsons['modelId'])
ret = sc.api_call('chat.postMessage',
channel=chid,
blocks=text.replace("TEXTKEY", text_message))
#TODO: Fix this bug - opaques dont work on spark
#last_time_new = Opaque().name("last_time_slack").data(last_row.created.values[0])
mlops.set_stat("last_time_slack", last_row.created.values[0].item())
mlops.done()
def test_stats_basic():
now = datetime.utcnow()
last_hour = (now - timedelta(hours=1))
df = mlops.get_stats("non-existing-stat__AAA",
mlapp_node=None,
agent=None,
start_time=last_hour,
end_time=now)
assert len(df) == 0
# Adding multiple points (to see a graph in the ui), expecting each run to generate 8 points
mlops.set_stat("stat1", 1.0)
mlops.set_stat("stat1", 3.0)
mlops.set_stat("stat1", 4.0)
mlops.set_stat("stat1", 5.0)
mlops.set_stat("stat1", 6.0)
mlops.set_stat("stat1", 2.0)
mlops.set_stat("stat1", 7.0)
mlops.set_stat("stat1", 8.0)
print("Done reporting statistics")
time.sleep(10)
print("MyION = {}".format(mlops.get_mlapp_id()))
print("Hour before: {}".format(last_hour))
print("Now: {}".format(now))
agent_list = _get_my_agents()
now = datetime.utcnow()
last_hour = (now - timedelta(hours=1))
last_day = (now - timedelta(hours=24))
# A test for a big time window
df = mlops.get_stats(name="stat1",
mlapp_node=mlops.get_current_node().name,
agent=agent_list[0].id,
start_time=last_day,
end_time=now)
assert len(df) >= 8, "Got: {} lines in df, expecting at least: {}".format(
len(df), 8)
df = mlops.get_stats(name="stat1",
mlapp_node=mlops.get_current_node().name,
agent=agent_list[0].id,
start_time=last_hour,
end_time=now)
assert len(df) >= 8, "Got: {} lines in df, expecting at least: {}".format(
len(df), 8)
print("Got stat1 statistic\n", df)
# Another check with the agent object
df = mlops.get_stats(name="stat1",
mlapp_node=mlops.get_current_node().name,
agent=agent_list[0],
start_time=last_hour,
end_time=now)
print("Got stat1 statistic_2\n", df)
# Another with node equal to none and agent equal to none
df = mlops.get_stats(name="stat1",
mlapp_node=None,
agent=None,
start_time=last_hour,
end_time=now)
print("Got stat1 statistic_3\n", df)
nodes_in_stats = df["node"].tolist()
if len(nodes_in_stats) > 8:
print("case_with_stats_from_2_nodes")
set_nodes = set(nodes_in_stats)
assert len(set_nodes) > 1
def canary_comparator(options, start_time, end_time, mode):
sc = None
if mode == RunModes.PYSPARK:
from pyspark import SparkContext
sc = SparkContext(appName="canary-comparator")
mlops.init(sc)
elif mode == RunModes.PYTHON:
mlops.init()
else:
raise Exception("Invalid mode " + mode)
not_enough_data = False
# Following are main and canary component names
main_prediction_component_name = options.nodeA
canary_prediction_component_name = options.nodeB
main_stat_name = options.predictionHistogramA
canary_stat_name = options.predictionHistogramB
main_agent = utils._get_agent_id(main_prediction_component_name,
options.agentA)
canary_agent = utils._get_agent_id(canary_prediction_component_name,
options.agentB)
if main_agent is None or canary_agent is None:
print("Invalid agent provided {} or {}".format(options.agentA,
options.agentB))
mlops.system_alert(
"PyException",
"Invalid Agent {} or {}".format(options.agentA, options.agentB))
return
try:
main_data_frame = mlops.get_stats(
name=main_stat_name,
mlapp_node=main_prediction_component_name,
agent=main_agent,
start_time=start_time,
end_time=end_time)
canary_data_frame = mlops.get_stats(
name=canary_stat_name,
mlapp_node=canary_prediction_component_name,
agent=canary_agent,
start_time=start_time,
end_time=end_time)
main_pdf = pd.DataFrame(main_data_frame)
canary_pdf = pd.DataFrame(canary_data_frame)
try:
row1 = main_pdf.tail(1).iloc[0]
row2 = canary_pdf.tail(1).iloc[0]
except Exception as e:
not_enough_data = True
print("Not enough histograms produced in pipelines")
raise ValueError("Not enough data to compare")
if row1['hist_type'] != row2['hist_type']:
raise ValueError(
'Canary and Main pipelines dont produce histograms' +
'of same type {} != {}'.format(row1['hist_type'],
row2['hist_type']))
if row1['hist_type'] == 'continuous':
rmse = _compare_cont_hist(row1['bin_edges'], row2['bin_edges'],
row1['hist_values'], row2['hist_values'])
gg2 = MultiGraph().name("Prediction Histograms").set_categorical()
gg2.x_title("Predictions")
gg2.y_title("Normalized Frequency")
gg2.add_series(label="Main",
x=[float(x) for x in row1['bin_edges']][:-1],
y=[y for y in row1['hist_values']])
gg2.add_series(label="Canary",
x=[float(x) for x in row2['bin_edges']][:-1],
y=[y for y in row2['hist_values']])
mlops.set_stat(gg2)
bar1 = BarGraph().name("Main Pipeline").cols([
"{} to {}".format(x, y)
for (x, y) in pairwise(row1['bin_edges'])
]).data([x for x in row1['hist_values']])
mlops.set_stat(bar1)
bar2 = BarGraph().name("Canary Pipeline").cols([
"{} to {}".format(x, y)
for (x, y) in pairwise(row2['bin_edges'])
]).data([x for x in row2['hist_values']])
mlops.set_stat(bar2)
elif row1['hist_type'] == 'categorical':
rmse = _compare_cat_hist(row1['bin_edges'], row2['bin_edges'],
row1['hist_values'], row2['hist_values'])
gg2 = MultiGraph().name("Prediction Histograms").set_categorical()
gg2.x_title("Predictions")
gg2.y_title("Normalized Frequency")
gg2.add_series(label="Main",
x=row1['bin_edges'],
y=[y for y in row1['hist_values']])
gg2.add_series(label="Canary",
x=row2['bin_edges'],
y=[y for y in row2['hist_values']])
mlops.set_stat(gg2)
bar1 = BarGraph().name("Main Pipeline").cols([
"{}".format(x) for x in row1['bin_edges']
]).data([x for x in row1['hist_values']])
mlops.set_stat(bar1)
bar2 = BarGraph().name("Canary Pipeline").cols([
"{}".format(x) for x in row2['bin_edges']
]).data([x for x in row2['hist_values']])
mlops.set_stat(bar2)
else:
raise ValueError('Invalid histogram type: {}'.format(
row1['hist_type']))
mlops.set_stat("RMSE", rmse, st.TIME_SERIES)
print("mlops policy {}".format(mlops.mlapp_policy))
if mlops.mlapp_policy.canary_threshold is None:
print("Canary health threshold not set")
raise ValueError("Canary health threshold not set in config")
# Following code perform comparison between the histograms.
# Here you can insert your own code
if rmse > mlops.mlapp_policy.canary_threshold:
print("Canary Alert {} > {}".format(
rmse, mlops.mlapp_policy.canary_threshold))
mlops.event(
CanaryAlert(label="CanaryAlert",
is_healthy=False,
score=rmse,
threshold=mlops.mlapp_policy.canary_threshold))
else:
print("Data matches {}".format(rmse))
mlops.event(
CanaryAlert(label="CanaryAlert",
is_healthy=True,
score=rmse,
threshold=mlops.mlapp_policy.canary_threshold))
except Exception as e:
if not_enough_data is False:
print("Got exception while getting stats: {}".format(e))
mlops.system_alert(
"PyException",
"Got exception {}".format(traceback.format_exc()))
if mode == RunModes.PYSPARK:
sc.stop()
mlops.done()
def ab_test(options, start_time, end_time, mode):
sc = None
if mode == RunModes.PYSPARK:
from pyspark import SparkContext
sc = SparkContext(appName="pm-ab-testing")
pm.init(sc)
elif mode == RunModes.PYTHON:
pm.init()
else:
raise Exception("Invalid mode " + mode)
not_enough_data = False
# Following are a and b component names
a_prediction_component_name = options.nodeA
b_prediction_component_name = options.nodeB
conv_a_stat_name = options.conversionsA
conv_b_stat_name = options.conversionsB
samples_a_stat_name = options.samplesA
samples_b_stat_name = options.samplesB
a_agent = utils._get_agent_id(a_prediction_component_name, options.agentA)
b_agent = utils._get_agent_id(b_prediction_component_name, options.agentB)
if a_agent is None or b_agent is None:
print("Invalid agent provided {} or {}".format(options.agentA, options.agentB))
pm.system_alert("PyException",
"Invalid Agent {} or {}".format(options.agentA, options.agentB))
return
try:
a_samples = pm.get_stats(name=samples_a_stat_name, mlapp_node=a_prediction_component_name,
agent=a_agent, start_time=start_time,
end_time=end_time)
b_samples = pm.get_stats(name=samples_b_stat_name, mlapp_node=b_prediction_component_name,
agent=b_agent, start_time=start_time,
end_time=end_time)
a_samples_pdf = pd.DataFrame(a_samples)
b_samples_pdf = pd.DataFrame(b_samples)
try:
rowa1 = int(a_samples_pdf.tail(1)['value'])
rowb1 = int(b_samples_pdf.tail(1)['value'])
except Exception as e:
not_enough_data = True
print("Not enough samples stats produced in pipelines")
raise ValueError("Not enough data to compare")
a_conv = pm.get_stats(name=conv_a_stat_name, mlapp_node=a_prediction_component_name,
agent=a_agent, start_time=start_time,
end_time=end_time)
b_conv = pm.get_stats(name=conv_b_stat_name, mlapp_node=b_prediction_component_name,
agent=b_agent, start_time=start_time,
end_time=end_time)
a_conv_pdf = pd.DataFrame(a_conv)
b_conv_pdf = pd.DataFrame(b_conv)
try:
rowa2 = int(a_conv_pdf.tail(1)['value'])
rowb2 = int(b_conv_pdf.tail(1)['value'])
except Exception as e:
not_enough_data = True
print("Not enough conversion stats produced in pipelines")
raise ValueError("Not enough data to compare")
abHealth = statsCalculator()
abHealth.exptOutcome(float(rowa1), float(rowa2), float(rowb1), float(rowb2),
options.confidence)
confidence = abHealth.calConfidence()
out = abHealth.calSuccess(options.confidence)
# calculate conversion rate
convA = float(rowa2) / float(rowa1)
convB = float(rowb2) / float(rowb1)
if convA != 0.0:
relUplift = (convB - convA) / (convA)
else:
relUplift = convB
relUplift = relUplift * 100
# AB Graphs
ab = MultiGraph().name("AB").set_continuous()
ab.x_title("Conversion Rate (%)")
ab.y_title(" ")
# normalizing x and y axis for A for display
dist_a_norm_x = [a_x * 100.0 / rowa1 for a_x in abHealth._distControl[0].tolist()]
dist_a_norm_y = [a_y * rowa1 / 100.0 for a_y in abHealth._distControl[1].tolist()]
ab.add_series(label="A", x=dist_a_norm_x, y=dist_a_norm_y)
# normalizing x and y axis for B for display
dist_b_norm_x = [b_x * 100.0 / rowb1 for b_x in abHealth._distB[0].tolist()]
dist_b_norm_y = [b_y * rowb1 / 100.0 for b_y in abHealth._distB[1].tolist()]
ab.add_series(label="B", x=dist_b_norm_x, y=dist_b_norm_y)
# annotate confidence line on normalized x-axis
ab.annotate(label="{} %".format(options.confidence),
x=abHealth._verticalLine * 100.0 / rowa1)
# for not overriding it in display
# annotate CR line on normalized x-axis
if convA != convB:
ab.annotate(label="CR A {}".format(convA * 100.0), x=convA * 100.0)
ab.annotate(label="CR B {}".format(convB * 100.0), x=convB * 100.0)
else:
ab.annotate(label="CR A & B {}".format(convA * 100.0), x=convA * 100.0)
pm.set_stat(ab)
# conversion rate
cols = ["A", "B"]
mlt = MultiLineGraph().name("ConversionRate").labels(cols).data(
[convA * 100.0, convB * 100.0])
pm.set_stat(mlt)
# emit table with all stats
tbl2 = Table().name("AB Stats").cols(
["Samples Processed", "Conversions", "Conversion Rate (%)",
"Improvement (%)", "Chance to beat baseline (%)"])
tbl2.add_row(options.champion,
[str(rowa1), str(rowa2), "{0:.2f}".format(convA * 100), "-", "-"])
tbl2.add_row(options.challenger, [str(rowb1), str(rowb2), "{0:.2f}".format(convB * 100),
"{0:.2f}".format(relUplift),
"{0:.2f}".format(confidence)])
pm.set_stat(tbl2)
# set cookie
tbl = Table().name("cookie").cols(["uplift", "champion", "challenger",
"conversionA", "conversionB", "realUplift", "success",
"confidence", "realConfidence"])
tbl.add_row("1", [str(options.uplift), options.champion, options.challenger,
"{0:.2f}".format(convA * 100), "{0:.2f}".format(convB * 100),
"{0:.2f}".format(abHealth._uplift), str(out), str(options.confidence),
"{0:.2f}".format(abHealth.calConfidence())])
pm.set_stat(tbl)
if out == True:
pm.data_alert("DataAlert", "AB Test Success zScore {}".format(abHealth._zScore))
pm.set_stat("Success", 1, st.TIME_SERIES)
else:
pm.set_stat("Success", 0, st.TIME_SERIES)
except Exception as e:
if not_enough_data is False:
print("Got exception while getting stats: {}".format(e))
pm.system_alert("PyException", "Got exception {}".format(e))
if mode == RunModes.PYSPARK:
sc.stop()
pm.done()