def __init__(self, imbalance_upsampling=None, class_weight=None, method=None, random_state=1, log=None):
"""
Construtor
:param imbalance_upsampling: Use upsampling to compensate imbalanced dataset
:param class_weight: Use class_weight to compensate imbalanced dataset
:param method: [Optional] Ensemble method
:param random_state: Random state
:param log: Log
"""
MlModelCommon.__init__(self,
imbalance_upsampling=imbalance_upsampling,
class_weight=class_weight,
method=method,
log=log)
#
# GaussianNB does not support class_weight
#
if method == "Bagging":
model = GaussianNB()
self.ensemble_method = BaggingClassifier(base_estimator=model,
n_estimators=100,
random_state=random_state)
elif method == "Adaptive Boosting":
model = GaussianNB()
self.ensemble_method = AdaBoostClassifier(base_estimator=model,
n_estimators=100,
random_state=random_state)
else:
self.ensemble_method = None
GaussianNB.__init__(self)
def __init__(self,
imbalance_upsampling=None,
class_weight=None,
method=None,
random_state=10,
log=None):
MlModelCommon.__init__(self,
imbalance_upsampling=imbalance_upsampling,
class_weight=class_weight,
method=method,
log=log)
if method == "Bagging":
model = DecisionTreeClassifier(class_weight=class_weight,
min_samples_split=20,
random_state=99)
self.ensemble_method = BaggingClassifier(base_estimator=model,
n_estimators=10,
random_state=random_state)
elif method == "Adaptive Boosting":
model = DecisionTreeClassifier(class_weight=class_weight,
min_samples_split=20,
random_state=99)
self.ensemble_method = AdaBoostClassifier(
base_estimator=model,
n_estimators=50,
random_state=random_state)
else:
self.ensemble_method = None
DecisionTreeClassifier.__init__(self,
class_weight=class_weight,
min_samples_split=20,
random_state=99)
def __init__(self,
imbalance_upsampling=None,
class_weight=None,
method=None,
c=100.0,
random_state=1,
log=None):
MlModelCommon.__init__(self,
imbalance_upsampling=imbalance_upsampling,
class_weight=class_weight,
method=method,
log=log)
if method == "Bagging":
model = BernoulliNB()
self.ensemble_method = BaggingClassifier(base_estimator=model,
n_estimators=10,
random_state=random_state)
elif method == "Adaptive Boosting":
model = BernoulliNB()
self.ensemble_method = AdaBoostClassifier(
base_estimator=model,
n_estimators=10,
random_state=random_state)
else:
#
# BernoulliNB does not support class_weight
#
BernoulliNB.__init__(self)
self.ensemble_method = None
def __init__(self, strategy="stratified", log=None):
"""
:param self:
:param strategy: Possible choices
"stratified": Predict by respecting the training set's class distribution
"most_frequent": Always predict the majority
"prior": Maximize the lass prior
"uniform": Generates uniform prediction randomly
"constant": Predict a given value always
:param log:
:return:
"""
MlModelCommon.__init__(self, log=log)
DummyClassifier.__init__(self, strategy=strategy)
def __init__(self,
imbalance_upsampling=None,
class_weight=None,
method=None,
c=100.0,
random_state=1,
log=None):
"""
Initialize the model
:param imbalance_upsampling: Using upsampling to compensate imbalanced dataset
:param class_weight: It can be None, "balanced", or a dict. Used for imbalance class
:param method: Optional ensemble method
:param c: Not supported yet.
:param random_state: Random state
:param log: log
"""
self.c = c
self.random_state = random_state
MlModelCommon.__init__(self,
imbalance_upsampling=imbalance_upsampling,
class_weight=class_weight,
method=method,
log=log)
if method == "Bagging":
model = LgRegression(C=c,
class_weight=class_weight,
random_state=random_state)
self.ensemble_method = BaggingClassifier(base_estimator=model,
n_estimators=200,
random_state=random_state)
elif method == "Adaptive Boosting":
model = LgRegression(C=c,
class_weight=class_weight,
random_state=random_state)
self.ensemble_method = AdaBoostClassifier(
base_estimator=model,
n_estimators=200,
random_state=random_state)
else:
self.ensemble_method = None
LgRegression.__init__(self,
C=c,
random_state=random_state,
class_weight=class_weight)
def __init__(self,
imbalance_upsampling=None,
class_weight=None,
random_state=1,
n_neighbors=5,
method=None,
log=None):
"""
:param imbalance_upsampling: Use upsampling to compensate imbalance
:param class_weight: Use class_weight to compensate imbalance
:param random_state: Random state
:param n_neighbors: Number of neighbor samples to use
:param method: Ensemble method
:param log: Log
"""
MlModelCommon.__init__(self,
imbalance_upsampling=imbalance_upsampling,
class_weight=class_weight,
method=method,
log=log)
#
# class_weight is not supported for KNN.
#
if method == "Bagging":
model = KNeighborsClassifier(n_neighbors=n_neighbors,
metric="minkowski")
self.ensemble_method = BaggingClassifier(base_estimator=model,
n_estimators=10,
random_state=random_state)
elif method == "Adaptive Boosting":
model = KNeighborsClassifier(n_neighbors=n_neighbors,
metric="minkowski")
self.ensemble_method = AdaBoostClassifier(
base_estimator=model,
n_estimators=10,
random_state=random_state)
else:
self.ensemble_method = None
KNeighborsClassifier.__init__(self,
n_neighbors=n_neighbors,
metric="minkowski")
def __init__(self,
imbalance_upsampling=None,
class_weight=None,
method=None,
log=None):
MlModelCommon.__init__(self,
imbalance_upsampling=imbalance_upsampling,
class_weight=class_weight,
method=method,
log=log)
#
# Random forest is a special case of bagging of
# decision tree. Might not make sense to
# add ensemble method.
#
self.ensemble_method = None
RandomForestClassifier.__init__(self,
class_weight=class_weight,
n_estimators=100,
random_state=99)
def view_model(args):
"""
Show the summary of a saved model file.
This method is call when user use command line util to view summary of
a saved model file.
:param args: Command line arguments
:return:
"""
file_name = args.input
file_exits = os.path.exists(file_name)
if file_exits:
#
# Deserialize model
#
model = MlModelCommon.load_from_file(file_name)
#
# Output the information of this model
#
show_model_summary(model, os.path.abspath(file_name))
else:
LOG.error("Model file {} does not exist.".format(file_name))
def _ml_predict_function(self, event, *args, **kwargs):
"""
This function takes input of incident id and saved model name. It will
1. Retrieve the corresponding incident from Resilient server
2. Read the saved model
3. Use the model to make a prediction for this incident.
:param event:
:param args:
:param kwargs:
:return:
"""
try:
# Get the function parameters:
ml_incident_id = kwargs.get("ml_incident_id") # number
ml_model_file = kwargs.get("ml_model_file") # text
log = logging.getLogger(__name__)
log.info("ml_incident_id: %s", ml_incident_id)
filename = resilient_utils.extract_filename(ml_model_file)
if not filename:
err_str = "No valid model file found in {}".format(ml_model_file)
log.error(err_str)
raise Exception(err_str)
log.info("ml_model_file: %s", filename)
yield StatusMessage("starting...")
inc = self.rest_client().get("/incidents/{}".format(str(ml_incident_id)))
log.debug("Incident {id}: {dic}".format(id=str(ml_incident_id),
dic=inc))
#
# Read model folder from app.config
#
model_folder = self.opts[APP_CONFIG_SECTION].get(MODEL_DIR_KEY, None)
selected_model = os.path.join(model_folder, filename)
if selected_model:
model = MlModelCommon.load_from_file(selected_model)
else:
raise ValueError("active_model not defined in app.config")
model.log = log
mapping = resilient_utils.get_field_def(self.rest_client(),
model.config.predict_field,
"incident")
prediction = ""
if model:
prediction = model.predict_map(inc, mapping)
else:
raise Exception("Failed to load model file: {}".format(selected_model))
results = {
"prediction": prediction
}
log.info("Result: {}".format(results))
yield StatusMessage("done...")
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception as e:
log.exception(str(e))
yield FunctionError()