def train(cls, spark, **args): categorical_colnames = args["categorical_colnames"] numerical_colnames = args["numerical_colnames"] sdf = args["sdf"] entity_profile_sdf = sdf if sdf.count() > 0: kmean_preprocess_model = KMeansPipeline.train( spark=spark, sdf=entity_profile_sdf, cat_colnames=categorical_colnames, num_colnames=numerical_colnames) centroids = np.array(kmean_preprocess_model.pipeline_model. stages[-1].clusterCenters()) result_cluster_sdf = kmean_preprocess_model.pipeline_model.transform( entity_profile_sdf).persist() result_score_sdf = result_cluster_sdf.withColumn( "sed", udf_calculate_SED(centroids)(col("features"))).persist() kmean_postprocess_model = MinMaxPipeline.train( spark=spark, result_score_sdf=result_score_sdf, colnames=["sed"]) else: kmean_preprocess_model = None kmean_postprocess_model = None return ClusterProfileModel(kmeans_model=kmean_preprocess_model, minmax_model=kmean_postprocess_model)
def score(self, sdf): if sdf.rdd.isEmpty(): return None if self.kmeans_model != None and self.minmax_model != None: result_cluster_sdf = self.kmeans_model.pipeline_model.transform( sdf) centroids = np.array( self.kmeans_model.pipeline_model.stages[-1].clusterCenters()) result_sed_sdf = result_cluster_sdf.withColumn( "sed", udf_calculate_SED(centroids)(col("features"))) result_scaler_sdf = self.minmax_model.pipeline_model.transform( result_sed_sdf).persist() def format_anomaly_score(value): val = float(value[0]) * 100 if val > 100: return 100.0 if val < 0: return 0.0 return val udf_format_anomaly_score = udf(format_anomaly_score, FloatType()) result_score_sdf = result_scaler_sdf.withColumn( "PAS", udf_format_anomaly_score("scaled_sed")) else: result_score_sdf = sdf.withColumn("PAS", lit(0.0)) return result_score_sdf
def train(cls, spark, data_store, **args): categorical_colnames = args["cat_colnames"] numerical_colnames = args["num_colnames"] sdf = args["sdf"] event_sdf = sdf if sdf is None: event_sdf = data_store.read_spark_df_from_data_store(**args) kmean_preprocess_model = None kmean_postprocess_model = None if not event_sdf.rdd.isEmpty(): kmean_preprocess_model = KMeansPipeline.train( spark=spark, sdf=event_sdf, cat_colnames=categorical_colnames, num_colnames=numerical_colnames) centroids = np.array(kmean_preprocess_model.pipeline_model. stages[-1].clusterCenters()) result_cluster_sdf = kmean_preprocess_model.pipeline_model.transform( event_sdf).persist() result_score_sdf = result_cluster_sdf.withColumn( "sed", udf_calculate_SED(centroids)(col("features"))).persist() kmean_postprocess_model = MinMaxPipeline.train( spark=spark, result_score_sdf=result_score_sdf, colnames=["sed"]) return ClusterEventModel(kmeans_model=kmean_preprocess_model, minmax_model=kmean_postprocess_model)