def load_oob_model():
encoded_model = demisto_ml.load_oob(OUT_OF_THE_BOX_MODEL_PATH)
res = demisto.executeCommand('createMLModel', {'modelData': encoded_model,
'modelName': OUT_OF_THE_BOX_MODEL_NAME,
'modelLabels': ['legit', 'spam', 'malicious'],
'modelOverride': 'true'})
if is_error(res):
return_error(get_error(res))
with open(EVALUATION_PATH, 'r') as json_file:
data = json.load(json_file)
y_test = data['y_true']
y_pred = data['y_pred']
res = demisto.executeCommand('GetMLModelEvaluation', {'yTrue': json.dumps(y_test),
'yPred': json.dumps(y_pred),
'targetPrecision': str(0),
'targetRecall': str(0),
'detailedOutput': 'false'
})
if is_error(res):
return_error(get_error(res))
confusion_matrix = json.loads(res[0]['Contents']['csr_matrix_at_threshold'])
confusion_matrix_no_all = {k: v for k, v in confusion_matrix.items() if k != 'All'}
confusion_matrix_no_all = {k: {sub_k: sub_v for sub_k, sub_v in v.items() if sub_k != 'All'}
for k, v in confusion_matrix_no_all.items()}
res = demisto.executeCommand('evaluateMLModel',
{'modelConfusionMatrix': confusion_matrix_no_all,
'modelName': OUT_OF_THE_BOX_MODEL_NAME})
if is_error(res):
return_error(get_error(res))
def load_oob_model():
try:
encoded_model = demisto_ml.load_oob(OUT_OF_THE_BOX_MODEL_PATH)
except Exception:
return_error(traceback.format_exc())
res = demisto.executeCommand('createMLModel', {'modelData': encoded_model.decode('utf8'),
'modelName': OUT_OF_THE_BOX_MODEL_NAME,
'modelLabels': ['Malicious', 'Non-Malicious'],
'modelOverride': 'true',
'modelType': 'torch',
'modelExtraInfo': {'threshold': THRESHOLD}
})
if is_error(res):
return_error(get_error(res))
with open(EVALUATION_PATH, 'r') as json_file:
data = json.load(json_file)
y_test = data['YTrue']
y_pred = data['YPred']
y_pred_prob = data['YPredProb']
y_pred_evaluation = [{pred: prob} for pred, prob in zip(y_pred, y_pred_prob)]
res = demisto.executeCommand('GetMLModelEvaluation', {'yTrue': json.dumps(y_test),
'yPred': json.dumps(y_pred_evaluation),
'targetPrecision': str(0.85),
'targetRecall': str(0),
'detailedOutput': 'true'
})
if is_error(res):
return_error(get_error(res))
confusion_matrix = json.loads(res[0]['Contents']['csr_matrix_at_threshold'])
confusion_matrix_no_all = {k: v for k, v in confusion_matrix.items() if k != 'All'}
confusion_matrix_no_all = {k: {sub_k: sub_v for sub_k, sub_v in v.items() if sub_k != 'All'}
for k, v in confusion_matrix_no_all.items()}
res = demisto.executeCommand('evaluateMLModel',
{'modelConfusionMatrix': confusion_matrix_no_all,
'modelName': OUT_OF_THE_BOX_MODEL_NAME,
'modelEvaluationVectors': {'Ypred': y_pred,
'Ytrue': y_test,
'YpredProb': y_pred_prob
},
'modelConfidenceThreshold': THRESHOLD,
'modelTargetPrecision': TARGET_PRECISION
})
if is_error(res):
return_error(get_error(res))