def test_vector_slicer(self):
data = self.spark.createDataFrame(
[(Vectors.dense([-2.0, 2.3, 0.0, 0.0, 1.0]), ),
(Vectors.dense([0.0, 0.0, 0.0, 0.0, 0.0]), ),
(Vectors.dense([0.6, -1.1, -3.0, 4.5, 3.3]), )], ["features"])
model = VectorSlicer(inputCol="features",
outputCol="sliced",
indices=[1, 4])
feature_count = data.first()[0].array.size
model_onnx = convert_sparkml(
model, 'Sparkml VectorSlicer',
[('features', FloatTensorType([None, feature_count]))])
self.assertTrue(model_onnx is not None)
# run the model
predicted = model.transform(data)
expected = predicted.toPandas().sliced.apply(
lambda x: pandas.Series(x.toArray())).values.astype(numpy.float32)
data_np = data.toPandas().features.apply(
lambda x: pandas.Series(x.toArray())).values.astype(numpy.float32)
paths = save_data_models(data_np,
expected,
model,
model_onnx,
basename="SparkmlVectorSlicer")
onnx_model_path = paths[-1]
output, output_shapes = run_onnx_model(['sliced'], data_np,
onnx_model_path)
compare_results(expected, output, decimal=5)
def predict_prob(lrModelPath, test_data):
lrModel = LogisticRegressionModel.load(lrModelPath)
predictions = lrModel.transform(test_data)
result = predictions.select(['_c0', '_c1', '_c2', 'probability'])
print('*************** result **************')
print(result.show(5))
# result.write.csv('file:///opt/int_group/result123')
vs = VectorSlicer(inputCol="probability", outputCol="prob_1", indices=[1])
prob_1 = vs.transform(result)
print('*************** prob_1 **************')
print(prob_1.show(5))
result_prob1 = prob_1.select(['_c0', '_c1', '_c2', 'prob_1'])
print('*************** result_prob1 **************')
print(result_prob1.show(5))
new_result_prob1 = result_prob1.select([
'_c0', '_c1', '_c2',
result_prob1['prob_1'].cast('string').alias('prob_1_str')
])
print('*************** new_result_prob1 **************')
print(new_result_prob1.show(10))
print(new_result_prob1)
# find null rows
final_null_rows = new_result_prob1.filter(new_result_prob1._c0.isNull() | new_result_prob1._c1.isNull()\
| new_result_prob1._c2.isNull() | new_result_prob1.prob_1_str.isNull())
print('########### find null rows #############')
final_null_rows.show(100)
def slice_win_source_to(source, destination):
df_w = spark.read.parquet(os.path.join("datasets", source))
for j in range(8):
slicer = VectorSlicer(inputCol="f"+str(j), outputCol="f_sl"+str(j), indices=[i for i in range(50,76)])
df_w = slicer.transform(df_w).drop("f"+str(j))
cols = ["f_sl"+str(i) for i in range(8)]
assembler = VectorAssembler(inputCols=cols, outputCol="f")
df_w = assembler.transform(df_w)
df_w.write.mode("overwrite").parquet(os.path.join("datasets", destination))
df_w.printSchema()
def predict_prob(lrModelPath, test_data):
lrModel = LogisticRegressionModel.load(lrModelPath)
predictions = lrModel.transform(test_data)
result = predictions.select(['_c0', '_c1', '_c2', 'probability'])
print('*************** result **************')
print(result.show(5))
# result.write.csv('file:///opt/int_group/result123')
vs = VectorSlicer(inputCol="probability", outputCol="prob_1", indices=[1])
prob_1 = vs.transform(result)
print('*************** prob_1 **************')
print(prob_1.show(5))
result_prob1 = prob_1.select(['_c0', '_c1', '_c2', 'prob_1'])
print('*************** result_prob1 **************')
print(result_prob1.show(5))
# for i in range(800, 802):
# g = i / 1000
# h = g + 0.001
# sqlTrans = SQLTransformer(statement="SELECT _c0, _c1, _c2, prob_1[0] AS prob FROM __THIS__ WHERE prob_1[0] < h AND prob_1[0] >= g")
# dd = sqlTrans.transform(result_prob1)
# dd.write.csv('file:///opt/int_group/sql_test')
new_result_prob1 = result_prob1.select([
'_c0', '_c1', '_c2',
result_prob1['prob_1'].cast('string').alias('prob_1_str')
])
print('*************** new_result_prob1 **************')
print(new_result_prob1.show(5))
print(new_result_prob1)
dd = new_result_prob1.head(1000)
dd_df = spark.createDataFrame(dd)
dd_df.write.csv('file:///opt/int_group/head_1kw_test')
# for i in [1,2,3,4,5]:
# dd = new_result_prob1.head(i)
# dd_df = spark.createDataFrame(dd)
# dd_df.write.csv('file:///opt/int_group/head_test', mode='append')
# DataFrame[_c0: string, _c1: string, _c2: string, prob_1_str: string]
###
'''
Error:
Exception: Python in worker has different version 2.7 than that in driver 3.6,
PySpark cannot run with different minor versions.Please check environment variables PYSPARK_PYTHON and
PYSPARK_DRIVER_PYTHON are correctly set.
'''
# new_result_prob1.toPandas().to_csv('file:///opt/int_group/result.csv')
# new_result_prob1.toPandas().to_csv('hdfs://bcg/opt/int_group/result/result.csv')
###
'''
def _evaluate(self, dataset, metric="AP"):
def precision(y_true, y_scores, k):
act_set = set(y_true)
pred_set = set(y_scores[:k])
result = len(act_set & pred_set) / float(k)
return result
def recall(y_true, y_scores, k):
act_set = set(y_true)
pred_set = set(y_scores[:k])
result = len(act_set & pred_set) / float(len(act_set))
return result
neg_slicer = VectorSlicer(inputCol="probability",
outputCol="0_prob",
indices=[0])
pos_slicer = VectorSlicer(inputCol="probability",
outputCol="1_prob",
indices=[1])
output_stg1 = neg_slicer.transform(dataset)
output = pos_slicer.transform(output_stg1)
Ranked_prediction = output.sort(col("1_prob").desc())
y_true = Ranked_prediction.select("label").rdd.flatMap(
lambda x: x).collect()
y_scores = Ranked_prediction.select("prediction").rdd.flatMap(
lambda x: x).collect()
score = 0
if metric == "AP":
score = average_precision_score(y_true, y_scores)
elif metric == "P100":
score = precision(y_true, y_scores, 100)
return score
def predict_prob(lrModelPath, test_data):
lrModel = LogisticRegressionModel.load(lrModelPath)
predictions = lrModel.transform(test_data)
result = predictions.select(['_c0', '_c1', '_c2', 'probability'])
print('*************** result **************')
print(result.show(5))
# result.write.csv('file:///opt/int_group/result123')
vs = VectorSlicer(inputCol="probability", outputCol="prob_1", indices=[1])
prob_1 = vs.transform(result)
print('*************** prob_1 **************')
print(prob_1.show(5))
result_prob1 = prob_1.select(['_c0', '_c1', '_c2', 'prob_1'])
print('*************** result_prob1 **************')
print(result_prob1.show(5))
new_result_prob1 = result_prob1.select(['_c0', '_c1', '_c2', result_prob1['prob_1'].cast('string').alias('prob_1_str')])
print('*************** new_result_prob1 **************')
print(new_result_prob1.show(10))
print(new_result_prob1)
new_result_prob1 = new_result_prob1.na.drop()
# print('######################### printing num - {} #########################'.format(new_path))
new_result_prob1.write.csv('/opt/int_group/hanmo.wang/3key_all_v3', nullValue=None, mode='append')
def predict_prob(lrModel, test_data):
# lrModel = LogisticRegressionModel.load(lrModelPath)
predictions = lrModel.transform(test_data)
result = predictions.select(['id', 'name', 'cell', 'probability'])
print('*************** result **************')
print(result.show(5))
vs = VectorSlicer(inputCol="probability", outputCol="prob_1", indices=[1])
prob_1 = vs.transform(result)
print('*************** prob_1 **************')
print(prob_1.show(5))
result_prob1 = prob_1.select(['id', 'name', 'cell', 'prob_1'])
print('*************** result_prob1 **************')
print(result_prob1.show(5))
new_result_prob1 = result_prob1.select([
'id', 'name', 'cell',
result_prob1['prob_1'].cast('string').alias('prob_1_str')
])
print('*************** new_result_prob1 **************')
print(new_result_prob1.show(5))
print(new_result_prob1)
return new_result_prob1
from pyspark.ml.linalg import Vectors
from pyspark.sql.types import Row
df = spark.createDataFrame([
Row(userFeatures=Vectors.sparse(3, {
0: -2.0,
1: 2.3
})),
Row(userFeatures=Vectors.dense([-2.0, 2.3, 0.0]))
])
slicer = VectorSlicer(inputCol="userFeatures",
outputCol="features",
indices=[1])
output = slicer.transform(df)
output.select("userFeatures", "features").show()
# COMMAND ----------
###R formula selects columns specified by R formula
from pyspark.ml.feature import RFormula
dataset = spark.createDataFrame([(7, "US", 18, 1.0), (8, "CA", 12, 0.0),
(9, "NZ", 15, 0.0)],
["id", "country", "hour", "clicked"])
formula = RFormula(formula="clicked ~ country + hour",
featuresCol="features",
labelCol="label")
#
from __future__ import print_function
# $example on$
from pyspark.ml.feature import VectorSlicer
from pyspark.mllib.linalg import Vectors
from pyspark.sql.types import Row
# $example off$
from pyspark.sql import SparkSession
if __name__ == "__main__":
spark = SparkSession\
.builder\
.appName("VectorSlicerExample")\
.getOrCreate()
# $example on$
df = spark.createDataFrame([
Row(userFeatures=Vectors.sparse(3, {0: -2.0, 1: 2.3}),),
Row(userFeatures=Vectors.dense([-2.0, 2.3, 0.0]),)])
slicer = VectorSlicer(inputCol="userFeatures", outputCol="features", indices=[1])
output = slicer.transform(df)
output.select("userFeatures", "features").show()
# $example off$
spark.stop()
result_df = chisq_selector.fit(scaled_df).transform(scaled_df)
result_df.select("selected_features").display()
# COMMAND ----------
# MAGIC %md Feature Selection using VectorSclicer
# COMMAND ----------
from pyspark.ml.feature import VectorSlicer
vec_slicer = VectorSlicer(inputCol="scaled_features",
outputCol="selected_features",
indices=[1])
result_df = vec_slicer.transform(scaled_df)
result_df.select("scaled_features", "selected_features").display()
# COMMAND ----------
# MAGIC %md ###Delta Lake as Feature Store
# COMMAND ----------
spark.sql("CREATE DATABASE IF NOT EXISTS feature_store ")
(result_df.write.format("delta").mode("overwrite").option(
"location",
"/FileStore/shared_uploads/delta/retail_features.delta").saveAsTable(
"feature_store.retail_features"))
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, LSTM, GRU, CuDNNGRU, RNN, ConvLSTM2D, Conv1D, Reshape, MaxPooling1D, SimpleRNNCell, Flatten
from keras.callbacks import Callback
from keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt
import config
from sklearn.metrics import mean_absolute_error
import pandas
import csv
from pyspark.ml.feature import VectorSlicer
spark = config.get_config()
df_f = spark.read.parquet(os.path.join("datasets", "train.vector.fbin.2.parquet"))
slicer = VectorSlicer(inputCol='f', outputCol="fsl", indices=[i for i in range(0,26,2)])
df_f = slicer.transform(df_f).drop('f')
df_y = spark.read.parquet(os.path.join("datasets", "train.target.parquet"))
df_f = df_f.selectExpr("*").drop("_c0")
df_y = df_y.selectExpr("seg AS seg2", "y as label").drop("seg")
df_train = df_f
df_train = df_train.join(df_y, df_train.seg.cast(IntegerType()) == df_y.seg2.cast(IntegerType())).drop("seg2")
df_train.printSchema()
n_dim = 13 #26 99 119 99+14 20+26 14+26
# vect_cols = ["f"]
# vectorAssembler = VectorAssembler(inputCols=vect_cols, outputCol="features")
# df_train = vectorAssembler.transform(df_train)
testset,trainset = Data.randomSplit([0.3,0.7], seed=25)
print("Training Dataset Count: " + str(trainset.count()))
print("Test Dataset Count: " + str(testset.count()))
### GENERALIZED LINEAR REGRESSION FOR FEATURE SELECTION
from pyspark.ml.regression import GeneralizedLinearRegression
glr = GeneralizedLinearRegression(predictionCol="Predicted_median", labelCol="label", featuresCol="features",family="binomial", link="logit", maxIter=10,regParam=0.01)
model = glr.fit(Data)
summary = model.summary
print("Coefficient Standard Errors: " + str(summary.coefficientStandardErrors))
print("P Values: " + str(summary.pValues))
#Removing all the columns that had a p-value above 0.05
vs = VectorSlicer(inputCol="features", outputCol="selected_features", indices=[0,2,9,18,21,23,24,26,27,28,31,32,37,41])
Training_set= vs.transform(trainset)
Test_set = vs.transform(testset)
#### LOGISTIC REGRESSION
logReg = LogisticRegression(predictionCol="Predicted_median", labelCol="label", featuresCol="features", maxIter=20,regParam=0.01, elasticNetParam=0.8, family="binomial")
logReg_model = logReg.fit(Training_set)
trainingSummary = logReg_model.summary
roc = trainingSummary.roc.toPandas()
print('Training set ROC: ' + str(trainingSummary.areaUnderROC))
predictions = logReg_model.transform(Test_set)
predictions.select('features', 'label', 'rawPrediction', 'Predicted_median', 'probability').show(10)
evaluator = BinaryClassificationEvaluator()
print("Test_SET (Area Under ROC): " + str(evaluator.evaluate(predictions, {evaluator.metricName: "areaUnderROC"})))
### GRADIENT BOOSTING
def slice_source_to(source, destination):
slicer = VectorSlicer(inputCol="f_t1", outputCol="f", indices=[i for i in range(50,76)])
df_v = spark.read.parquet(os.path.join("datasets", source))
df_v = df_v.selectExpr("*", "f AS f_t1").drop("_c0").drop("f")
df_v = slicer.transform(df_v).drop("f_t1")
df_v.write.mode("overwrite").parquet(os.path.join("datasets", destination))
rdd = sc.textFile("/hive/warehouse/wlcredit.db/t_credit_feature_merge/ds=" + today + "_cms1234_anf")
rdd1 = rdd.map(lambda x:x.split("\001")[0] + " " + x.split("\001")[1])
rdd1.saveAsTextFile("/user/wrt/credit/allexample.libsvm")
data_svm_sql = sqlContext.read.format("libsvm").load("/user/wrt/credit/allexample.libsvm")
data_svm = data_svm_sql.map(lambda row:LabeledPoint(int(row.label),row.features))
features = data_svm.map(lambda x: x.features)
stat = Statistics.colStats(features)
coverage = (stat.numNonzeros()/stat.count()).tolist()
std = numpy.sqrt(stat.variance()).tolist()
features_nums = data_svm.map(lambda x: x.features.size).take(1)[0]
features_arr = range(0, features_nums)
re = zip(zip(coverage,std),features_arr)
filteredIndexes = map(lambda m: m[1],filter(lambda a:a[0][0] >=0.005,re))
slicer = VectorSlicer(inputCol="features", outputCol="featuresFiltered", indices=filteredIndexes)
output_df = slicer.transform(data_svm_sql)
data_svm_filtered = output_df.select("label","featuresFiltered")
data_svm_labelpoint = data_svm_filtered.map(lambda row:LabeledPoint(int(row.label),row.featuresFiltered))
MLUtils.saveAsLibSVMFile(data_svm_labelpoint,"/user/wrt/credit/allexample_filter.libsvm")
rdd_r = sc.textFile("/user/wrt/credit/allexample_filter.libsvm")\
.map(lambda x :x.split()[0].split('.')[0] + '\001' + ' '.join(x.split()[1:]))
rdd_r.saveAsTextFile("/user/wrt/credit/allexample_filter_telindex_features")
feature_raw = sc.textFile("/hive/warehouse/wlcredit.db/t_wrt_credit_all_features_name/ds=" + today + "_cms1234_anf")\
.map(lambda x:valid_jsontxt(x.split("\t")[0])).collect()
fea_all_index = []
j = 1
for i in filteredIndexes:
fea_all_index.append(feature_raw[i] + "\t" + str(j))
j += 1
sc.parallelize(fea_all_index).saveAsTextFile('/user/wrt/temp/filter_feature_name')
# MAGIC After comparing the results of four tested models, we can see that we are getting quite satisfying results with Random Forest, even for the small number of trees. Random Forest is performing much better than the other tested algorithms. In that manner, we will try to select only the most relevant features, and try to lower the dimensionality of the problem. <br>
# MAGIC After evaluating the results of Random Forest, we have identified the importance of the used features:
# MAGIC SparseVector(17, {0: 0.0589, 1: 0.0454, 2: 0.028, 3: 0.0834, 4: 0.1145, 5: 0.0469, 6: 0.0257, 7: 0.019, 8: 0.0159, 9: 0.003, 10: 0.0059, 11: 0.0009, 12: 0.0524, 13: 0.0211, 14: 0.0021, 15: 0.2103, 16: 0.2664}).<br>
# MAGIC Here we are going to lower the dimensionality of the problem based on feature importance, and using Vector Slicer Feature Selecion. Let's test how our datasets behave with less features, and try to make more robust model while using smaller number of features.
# COMMAND ----------
from pyspark.ml.feature import VectorSlicer
from pyspark.ml.linalg import Vectors
from pyspark.sql.types import Row
slicer = VectorSlicer(inputCol="features",
outputCol="selectedFeatures").setIndices([3, 4, 15, 16])
# We are using the same datasets as for the other algorithms
output = slicer.transform(transformed)
otestData = slicer.transform(testData)
otrainData = slicer.transform(trainingData)
# Let's make our model
rf = RandomForestClassifier(labelCol="label",
featuresCol="selectedFeatures",
numTrees=10)
# Chain indexers and forest in a Pipeline
pipeline = Pipeline(stages=[labelIndexer, rf])
# Train model. This also runs the indexers.
model = pipeline.fit(otrainData)
# Make predictions.