def encode_courses_and_instructors(df): course_string_indexer = StringIndexer(inputCol="course", outputCol="course_index") instructor_string_indexer = StringIndexer(inputCol="instructor", outputCol="instructor_index") encoder = OneHotEncoderEstimator(inputCols=["course_index", "instructor_index"], outputCols=["course_vec", "instructor_vec"]) encoder.setDropLast(False) pipeline = Pipeline(stages=[course_string_indexer, instructor_string_indexer]) indexed_data = pipeline.fit(df).transform(df) # Make a table mapping course index and course name courses_table = indexed_data.select('course_index', 'course').distinct() # Make a table mapping instructor index and instructor name instructors_table = indexed_data.select('instructor_index', 'instructor').distinct() indexed_data = indexed_data.select('course_index', 'instructor_index', 'term', 'year', "cat1", "cat2", "cat3", "cat4", "cat5", "cat6", "cat7", "cat8", "cat9", "num_invited", "num_responded") return indexed_data, courses_table, instructors_table
dataset_df = dataset_df.withColumn('model_type', sf.lit(1)) # list of categorical features for further hot-encoding cat_features = [ "age_bin", "WorkClass", "fnlwgt_bin", "Education", "edu_num_bin", "MaritalStatus", "Occupation", "Relationship", "Race", "Gender", "cap_gain_bin", "CapitalLoss", "hours_per_w_bin", "NativeCountry" ] # hot encoding categorical features for feature in cat_features: string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index") encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"]) encoder.setDropLast(False) stages += [string_indexer, encoder] assembler_inputs = [feature + "_vec" for feature in cat_features] assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs") stages += [assembler] assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features") label_indexer = StringIndexer(inputCol=label, outputCol=label + "_idx") stages += [assembler_final] stages += [label_indexer] pipeline = Pipeline(stages=stages) pipeline_model = pipeline.fit(dataset_df) dataset_transformed = pipeline_model.transform(dataset_df) cat_dict = [] decode_dict = {}
def test_attr_spark(self): conf = SparkConf().setAppName("toy_test").setMaster('local[2]') num_partitions = 2 enumerator = "join" model_type = "regression" label = 'target' sparkContext = SparkContext(conf=conf) sqlContext = SQLContext(sparkContext) train_df = sqlContext.read.csv("toy_train.csv", header='true', inferSchema='true') test_df = sqlContext.read.csv("toy.csv", header='true', inferSchema='true') # initializing stages of main transformation pipeline stages = [] # list of categorical features for further hot-encoding cat_features = ['a', 'b', 'c'] for feature in cat_features: string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index").setHandleInvalid("skip") encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"]) encoder.setDropLast(False) stages += [string_indexer, encoder] assembler_inputs = [feature + "_vec" for feature in cat_features] assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs") stages += [assembler] assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features") stages += [assembler_final] pipeline = Pipeline(stages=stages) train_pipeline_model = pipeline.fit(train_df) test_pipeline_model = pipeline.fit(test_df) train_df_transformed = train_pipeline_model.transform(train_df) test_df_transformed = test_pipeline_model.transform(test_df) train_df_transformed = train_df_transformed.withColumn('model_type', sf.lit(0)) test_df_transformed = test_df_transformed.withColumn('model_type', sf.lit(0)) decode_dict = {} counter = 0 cat = 0 for feature in cat_features: colIdx = test_df_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap() colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])} for item in colIdx: decode_dict[counter] = (cat, item, colIdx[item], counter) counter = counter + 1 cat = cat + 1 train_df_transform_fin = train_df_transformed.select('features', label, 'model_type') test_df_transform_fin = test_df_transformed.select('features', label, 'model_type') lr = LinearRegression(featuresCol='features', labelCol=label, maxIter=10, regParam=0.0, elasticNetParam=0.8) lr_model = lr.fit(train_df_transform_fin) eval = lr_model.evaluate(test_df_transform_fin) f_l2 = eval.meanSquaredError pred = eval.predictions pred_df_fin = pred.withColumn('error', spark_utils.calc_loss(pred[label], pred['prediction'], pred['model_type'])) predictions = pred_df_fin.select('features', 'error').repartition(num_partitions) converter = IndexToString(inputCol='features', outputCol='cats') all_features = list(decode_dict) predictions = predictions.collect() spark_join = spark_slicer.parallel_process(all_features, predictions, f_l2, sparkContext, debug=self.debug, alpha=self.alpha, k=self.k, w=self.w, loss_type=self.loss_type, enumerator="join") spark_union = spark_union_slicer.process(all_features, predictions, f_l2, sparkContext, debug=self.debug, alpha=self.alpha, k=self.k, w=self.w, loss_type=self.loss_type, enumerator="union") self.assertEqual(3, len(spark_join.slices)) print("check1") self.assertEqual(spark_join.min_score, spark_union.min_score) print("check2") self.assertEqual(spark_join.keys, spark_union.keys) print("check3") self.assertEqual(len(spark_join.slices), len(spark_union.slices)) print("check4") idx = -1 for sliced in spark_join.slices: idx += 1 self.assertEqual(sliced.score, spark_union.slices[idx].score) print("check5")