def train(self, df): df = self.build_features_vectors(df) scaler = StandardScaler() scaler.setInputCol(self.features_values_column) scaler.setOutputCol(self.features_values_scaled) scaler.setWithMean(self.scaler_with_mean) scaler.setWithStd(self.scaler_with_std) self.scaler_model = scaler.fit(df) df = self.scaler_model.transform(df).persist( StorageLevelFactory.get_storage_level(self.storage_level)) if len(self.categorical_features): self._create_indexes(df) self._add_categorical_features(df, self.features_values_scaled) iforest = IForest( featuresCol=self.features_values_scaled, predictionCol=self.prediction_column, # anomalyScore=self.score_column, numTrees=self.num_trees, maxSamples=self.max_samples, maxFeatures=self.max_features, maxDepth=self.max_depth, contamination=self.contamination, bootstrap=self.bootstrap, approxQuantileRelativeError=self. approximate_quantile_relative_error, # numCategoricalFeatures=len(self.categorical_features) ) iforest.setSeed(self.seed) params = {'threshold': self.threshold} self.iforest_model = iforest.fit(df, params) df.unpersist()
imputer.setInputCols(["total_bedrooms", "bedrooms_per_room"]) imputer.setOutputCols(["out_total_bedrooms", "out_bedrooms_per_room"]) imputedHousing = imputer.setStrategy('median').setMissingValue(414).fit(renamedHousing).transform(renamedHousing) imputedHousing = imputedHousing.drop('total_bedrooms').drop('bedrooms_per_room') for c in imputedHousing.columns: print(c, " has null values : ", imputedHousing.filter(imputedHousing[c].isNull()).count()) colNum_to_scale = ['longitude', 'latitude', 'housing_median_age', 'total_rooms', 'population', 'households', 'median_income', 'rooms_per_household','population_per_household','out_total_bedrooms','out_bedrooms_per_room'] va = VectorAssembler().setInputCols(colNum_to_scale).setOutputCol('features') featuredHousing = va.transform(imputedHousing) featuredHousing.show() scaler = StandardScaler(withMean=True, withStd=True) scaler.setInputCol("features").setOutputCol("scaled_features") scaledHousing = scaler.fit(featuredHousing).transform(featuredHousing) scaledHousing.select('scaled_features').show() # 3-2 bu tai ming bai zhe li????????????????????????????? distinct = renamedHousing.select('ocean_proximity').distinct().collect() print(distinct) renamedHousing.agg(countDistinct("ocean_proximity")).show() indexer = StringIndexer().setInputCol('ocean_proximity').setOutputCol('idx_ocean_proximity') idxHousing = indexer.fit(renamedHousing).transform(renamedHousing) idxHousing.show() encoder = OneHotEncoder().setInputCol('idx_ocean_proximity').setOutputCol('one_hot_ocean_proximity') ohHousing = encoder.fit(idxHousing).transform(idxHousing) ohHousing.show()