def test_list_string(self):
for labels in [
np.array(['a', u'b']), ['a', u'b'],
np.array(['a', 'b'])
]:
idx_to_string = IndexToString(labels=labels)
self.assertListEqual(idx_to_string.getLabels(), ['a', 'b'])
self.assertRaises(TypeError, lambda: IndexToString(labels=['a', 2]))
def index_to_string(df, input_cols, output_cols=None, columns=None, **kargs):
"""
Maps a column of indices back to a new column of corresponding string values. The index-string mapping is
either from the ML attributes of the input column, or from user-supplied labels (which take precedence over
ML attributes).
:param df: Dataframe to be transformed.
:param input_cols: Columns to be indexed.
:param output_cols: Column where the output is going to be saved.
:param columns:
:return: Dataframe with indexed columns.
"""
df_actual = df
if columns is None:
input_cols = parse_columns(df, input_cols)
if output_cols is None:
output_cols = [name_col(input_col, "index_to_string") for input_col in input_cols]
output_cols = get_output_cols(input_cols, output_cols)
else:
input_cols, output_cols = zip(*columns)
indexers = [IndexToString(inputCol=input_col, outputCol=output_col, **kargs) for input_col, output_col
in zip(list(set(input_cols)), list(set(output_cols)))]
pipeline = Pipeline(stages=indexers)
df = pipeline.fit(df).transform(df)
df = df.preserve_meta(df_actual, Actions.INDEX_TO_STRING.value, output_cols)
return df
def _transform(self, df) -> DataFrame:
"""
:param df: A pyspark.sql.dataframe.DataFrame
"""
# Apply string indexer
for in_col, out_col in zip(self.inputCols, self.outputCols):
self.__logger().info("Applying StringIndexer on col {}".format(in_col))
df = self.dict_indexers[in_col]['indexer'].transform(df)
n_to_keep = self.dict_indexers[in_col]['n_to_keep']
# If elements occur below (threshold * number of rows), replace them with n_to_keep.
this_meta = df.select(out_col).schema.fields[0].metadata
if n_to_keep != len(this_meta['ml_attr']['vals']):
this_meta['ml_attr']['vals'] = this_meta['ml_attr']['vals'][0:(n_to_keep + 1)]
this_meta['ml_attr']['vals'][n_to_keep] = self.groupText
self.__logger().info("Truncating number of categories of {} at {}".format(in_col, n_to_keep))
df = df.withColumn(out_col,
F.when(F.col(out_col) >= n_to_keep, F.lit(n_to_keep)).otherwise(
F.col(out_col)))
# add the new indexed column with correct metadata, remove original indexed column.
df = df.withColumn(out_col,
with_meta(F.col(out_col), "", this_meta))
if not self.returnIndexed:
for output_col in self.outputCols:
df = df.withColumnRenamed(output_col, output_col + '_temp')
df = IndexToString(inputCol=output_col + '_temp', outputCol=output_col).transform(df)
df = df.drop(output_col + '_temp')
if self.dropInputCols:
df = df.drop(*self.inputCols)
return df
def DecisionTree():
IrisData = spark.sparkContext.textFile("file:///home/unbroken/MyFiles/Work/Programming/Spark/DecisionTree/Iris.txt")\
.map(lambda line: line.split(',')).map(lambda p: Row(**f(p))).toDF()
IrisData.createOrReplaceTempView("iris")
df = spark.sql("select * from iris")
labelIndexer = StringIndexer(inputCol='label',
outputCol='labelIndex').fit(IrisData)
featureIndexer = VectorIndexer(
inputCol='feature',
outputCol='indexFeature').setMaxCategories(4).fit(IrisData)
labelConverter = IndexToString(inputCol='prediction',
outputCol='predictionLabel').setLabels(
labelIndexer.labels)
trainningData, testingData = IrisData.randomSplit([0.7, 0.3])
dtClassifier = DecisionTreeClassifier().setLabelCol(
'labelIndex').setFeaturesCol('indexFeature')
pipelineClassifier = Pipeline().setStages(
[labelIndexer, featureIndexer, dtClassifier, labelConverter])
modelClassifier = pipelineClassifier.fit(trainningData)
prediction = modelClassifier.transform(testingData)
print(prediction.show())
evaluator = MulticlassClassificationEvaluator().setLabelCol(
'labelIndex').setPredictionCol('prediction').setMetricName("accuracy")
accuracy = evaluator.evaluate(prediction)
print(accuracy)
treeModelClassifier = modelClassifier.stages[2]
print("Learned classification tree model:\n" +
str(treeModelClassifier.toDebugString))
def recommend_users(spark,
input_user,
input_video,
model,
user_indexer,
video_indexer,
user_language,
video_language,
num_recommend=20,
is_show=True):
"""Use als model to recommend for users
Args:
spark: Spark session
input_user:
input_video:
model: ALS model
user_indexer:
video_indexer:
num_recommend: The maximum number of recommendation videos
is_show: If true, the data would be shown
user_language: The language of user
video_language: The language of video
"""
# Recommend for all users
userRecs = model.recommendForAllUsers(num_recommend)
# Turn index back string
indexer_user = IndexToString(inputCol=input_user + "_index",
outputCol=input_user,
labels=user_indexer.labels)
index_user = indexer_user.transform(userRecs)
video_labels = array(*[lit(x) for x in video_indexer.labels])
recommendations = array(*[
struct(
video_labels[col("recommendations")[i][
input_video + "_index"]].alias(input_video),
col("recommendations")[i][input_rating])
for i in range(num_recommend)
])
recs = index_user.withColumn("recommendations", recommendations).select(
input_user, "recommendations")
explode_recs = recs.select(input_user, explode("recommendations").alias("recommendation")).\
select(input_user, "recommendation.*").\
select(input_user, input_video, col("col2").alias("score"))
# Keep user and video have same language
user_label = read_data_hive(spark, [input_user, user_language], is_show)
video_label = read_data_hive(spark, [input_video, video_language, is_show])
explode_recs_filter = explode_recs.join(user_label, input_user,
"inner").join(
video_label, input_video,
"inner")
explode_recs_filter = explode_recs_filter.filter(
explode_recs_filter[user_language] ==
explode_recs_filter[video_language])
if is_show:
explode_recs_filter.show(20)
explode_recs_filter.registerTempTable("temp_table")
def createPipeline(irisData, lrElasticNetParam, lrRegParam):
'''Creates a pipeline for coverting the data into features and label with the required format
Args: irisData - Input data for the feature and label processing
lrElasticNetParam - ElasticNet parameter of LR, 0-L2 penalty and 1-L1 penalty
lrRegParam - Regularization parameter
'''
strIndexer = StringIndexer().setInputCol('species').setOutputCol(
'label').fit(irisData)
va = VectorAssembler(inputCols=[
'sepal_length', 'sepal_width', 'petal_length', 'petal_width'
],
outputCol='vec_features')
ss = StandardScaler().setInputCol(
va.getOutputCol()).setOutputCol('features').fit(va.transform(irisData))
lr = LogisticRegression().setFeaturesCol('features')
labelConverter = IndexToString(inputCol='prediction',
outputCol='predictedLabel',
labels=strIndexer.labels)
stages = [strIndexer, va, ss, lr, labelConverter]
pipeline = Pipeline().setStages(stages)
params = ParamGridBuilder().addGrid(lr.elasticNetParam,
lrElasticNetParam).addGrid(
lr.regParam, lrRegParam).build()
evaluator = MulticlassClassificationEvaluator(labelCol='label',
predictionCol='prediction',
metricName=lrMetric)
return pipeline, params, evaluator
def _feature_transform(df):
string_indexers = list()
label_converter = None
feature_names = []
for field in df.schema.fields:
is_label = field.metadata and field.metadata['label']
if is_label:
if not isinstance(field.dataType, NumericType):
string_indexer = StringIndexer(inputCol=field.name,
outputCol="label").fit(df)
label_converter = IndexToString(inputCol="prediction",
outputCol="predictedLabel",
labels=string_indexer.labels)
string_indexers.append(string_indexer)
else:
df.withColumnRenamed(field.name, "label")
elif not isinstance(field.dataType, NumericType):
feature_name = field.name + "_idx"
string_indexer = StringIndexer(inputCol=field.name,
outputCol=feature_name).fit(df)
string_indexers.append(string_indexer)
feature_names.append(feature_name)
else:
feature_names.append(field.name)
return feature_names, label_converter, string_indexers
def RunRandomForest(tf, ctx):
sqlContext = SQLContext(ctx)
rdd = tf.map(parseForRandomForest)
# The schema is encoded in a string.
schema = ['genre', 'track_id', 'features']
# Apply the schema to the RDD.
songDF = sqlContext.createDataFrame(rdd, schema)
# Register the DataFrame as a table.
songDF.registerTempTable("genclass")
labelIndexer = StringIndexer().setInputCol("genre").setOutputCol("indexedLabel").fit(songDF)
trainingData, testData = songDF.randomSplit([0.8, 0.2])
labelConverter = IndexToString().setInputCol("prediction").setOutputCol("predictedLabel").setLabels(labelIndexer.labels)
rfc = RandomForestClassifier().setMaxDepth(10).setNumTrees(2).setLabelCol("indexedLabel").setFeaturesCol("features")
#rfc = SVMModel([.5, 10, 20], 5)
#rfc = LogisticRegression(maxIter=10, regParam=0.01).setLabelCol("indexedLabel").setFeaturesCol("features")
pipeline = Pipeline(stages=[labelIndexer, rfc, labelConverter])
model = pipeline.fit(trainingData)
predictions = model.transform(testData)
predictions.show()
evaluator = MulticlassClassificationEvaluator().setLabelCol("indexedLabel").setPredictionCol("prediction").setMetricName("precision")
accuracy = evaluator.evaluate(predictions)
print 'Accuracy of RandomForest = ', accuracy * 100
print "Test Error = ", (1.0 - accuracy) * 100
def trainModel(self, trainingData):
""" Ham huan luyen du lieu
Mac dinh training toan bo du lieu trong dataset splitratio 100% training, 0% testing
"""
# Chuyen toan bo nhan thanh so neu chua chuyen
# trainingData.select("label").groupBy("label").count().show()
labelIndexer = StringIndexer(
inputCol="label", outputCol="indexedLabel").fit(trainingData)
# Chuyen toan bo gia tri thuoc tinh thanh so neu chua chuyen
featureIndexer = VectorIndexer(inputCol="features",
outputCol="indexedFeatures",
maxCategories=4).fit(trainingData)
# Khai bao thuat toan RandomForest
rf = RandomForestClassifier(labelCol="indexedLabel",
featuresCol="indexedFeatures",
numTrees=30,
maxDepth=5,
maxBins=32,
seed=None,
impurity="gini")
# Chuyen nhan du doan duoc tu dang so ve dang ban dau,
labelConverter = IndexToString(inputCol="prediction",
outputCol="predictedLabel",
labels=labelIndexer.labels)
# Hop nhat tat ca cac buoc thanh mot luong duy nhat pipeline
pipeline = Pipeline(
stages=[labelIndexer, featureIndexer, rf, labelConverter])
# Train model qua pipeline
model = pipeline.fit(trainingData)
model.write().overwrite().save(os.path.join(self.modelpath,
"detector"))
return model
def trainModel(self, trainingData):
""" Ham huan luyen du lieu
Mac dinh training toan bo du lieu trong dataset splitratio 100% training, 0% testing
"""
labelIndexer = StringIndexer(
inputCol="label", outputCol="indexedLabel").fit(trainingData)
featureIndexer = VectorIndexer(inputCol="features",
outputCol="indexedFeatures",
maxCategories=4).fit(trainingData)
rf = RandomForestClassifier(labelCol="indexedLabel",
featuresCol="indexedFeatures",
numTrees=30,
maxDepth=5,
maxBins=32,
seed=None,
impurity="gini")
labelConverter = IndexToString(inputCol="prediction",
outputCol="predictedLabel",
labels=labelIndexer.labels)
pipeline = Pipeline(
stages=[labelIndexer, featureIndexer, rf, labelConverter])
model = pipeline.fit(trainingData)
model.write().overwrite().save(os.path.join(self.modelpath,
"detector"))
return model
def test_index_to_string(self):
original_data = self.spark.createDataFrame([(0, "a"), (1, "b"),
(2, "c"), (3, "a"),
(4, "a"), (5, "c")],
["id", "category"])
string_indexer = StringIndexer(inputCol="category",
outputCol="categoryIndex")
string_indexer_model = string_indexer.fit(original_data)
data = string_indexer_model.transform(original_data)
model = IndexToString(inputCol="categoryIndex",
outputCol="originalCategory",
labels=['A', 'B', 'C'])
# the input name should match that of what IndexToString.inputCol
model_onnx = convert_sparkml(
model, 'Sparkml IndexToString',
[('categoryIndex', Int64TensorType([1, 1]))])
self.assertTrue(model_onnx is not None)
# run the model
predicted = model.transform(data)
expected = predicted.select("originalCategory").toPandas().values
data_np = data.select('categoryIndex').toPandas().values.astype(
numpy.int64)
paths = save_data_models(data_np,
expected,
model,
model_onnx,
basename="SparkmlIndexToString")
onnx_model_path = paths[3]
output, output_shapes = run_onnx_model(['originalCategory'], data_np,
onnx_model_path)
compare_results(expected, output, decimal=5)
def prepare(self):
data = (self.spark_session.read.format(self.data_format)
.load(self.data_file))
labelIndexer = StringIndexer(
inputCol="label", outputCol="indexedLabel").fit(data)
featureIndexer = (VectorIndexer(inputCol="features",
outputCol="indexedFeatures",
maxCategories=self.max_categories)
.fit(data))
self.train_data, self.valid_data = data.randomSplit([0.8, 0.2])
if self.model_builder.__name__ == 'DecisionTreeClassifier':
classifier = self.model_builder(labelCol="indexedLabel",
featuresCol="indexedFeatures")
elif self.model_builder.__name__ == 'RandomForestClassifier':
classifier = self.model_builder(labelCol="indexedLabel",
featuresCol="indexedFeatures",
numTrees=self.num_trees)
labelConverter = IndexToString(inputCol="prediction",
outputCol="predictedLabel",
labels=labelIndexer.labels)
elif self.model_builder.__name__ == 'GBTClassifier':
classifier = self.model_builder(labelCol="indexedLabel",
featuresCol="indexedFeatures",
maxIter=self.max_iter)
if self.model_builder.__name__ == 'RandomForestClassifier':
self.pipeline = Pipeline(stages=[labelIndexer,
featureIndexer,
classifier,
labelConverter])
else:
self.pipeline = Pipeline(stages=[labelIndexer,
featureIndexer,
classifier])
def my_transform(rdd):
with open("./index2whiskey1.json", mode="r", encoding="utf-8") as f:
whiskey_list = list(json.loads(f.read()).values())
model = ALSModel.load("hdfs://master/ALSModel1/")
spark = SparkSession.builder.appName('sql coming~').getOrCreate()
whiskey = rdd.map(lambda x: Row(whiskeyId=int(x[1]), user_name=x[0]))
whiskey_df = spark.createDataFrame(whiskey)
predict = model.recommendForItemSubset(whiskey_df, 1)
df_user = predict.select(
predict.whiskeyId,
predict.recommendations[0].userId.alias("userId"),
)
df_whiskey = model.recommendForUserSubset(df_user, 5)
result_df = df_user.join(df_whiskey, on=['userId'], how='left')
result_df = result_df.join(whiskey_df, on=['whiskeyId'], how='left')
result_df = result_df.select("user_name",
result_df["recommendations"][0].whiskeyId.alias("whiskeyId1"), \
result_df["recommendations"][1].whiskeyId.alias("whiskeyId2"), \
result_df["recommendations"][2].whiskeyId.alias("whiskeyId3"), \
result_df["recommendations"][3].whiskeyId.alias("whiskeyId4"), \
result_df["recommendations"][4].whiskeyId.alias("whiskeyId5") \
)
whiskeyId1converter = IndexToString(inputCol="whiskeyId1",
outputCol="whiskey1",
labels=whiskey_list)
whiskeyId2converter = IndexToString(inputCol="whiskeyId2",
outputCol="whiskey2",
labels=whiskey_list)
whiskeyId3converter = IndexToString(inputCol="whiskeyId3",
outputCol="whiskey3",
labels=whiskey_list)
whiskeyId4converter = IndexToString(inputCol="whiskeyId4",
outputCol="whiskey4",
labels=whiskey_list)
whiskeyId5converter = IndexToString(inputCol="whiskeyId5",
outputCol="whiskey5",
labels=whiskey_list)
result_df = whiskeyId1converter.transform(result_df)
result_df = whiskeyId2converter.transform(result_df)
result_df = whiskeyId3converter.transform(result_df)
result_df = whiskeyId4converter.transform(result_df)
result_df = whiskeyId5converter.transform(result_df)
return result_df.rdd
def Customer_List(model, user):
# Create a dataset with distinct Customers as one column and the asin as another column
Customer = data_train.select("userid").distinct().withColumn("item", lit(user))
# # convert index back to original CustomerID
userconverter = IndexToString(inputCol="userid", outputCol="List of Customers")
userString = userconverter.transform(Customer)
userString.drop("userid").drop("item").show()
def main(train_x,
train_y,
test_x,
test_y=None,
idf=False,
ngram=1,
base='gs',
asm=False):
# Load : DF[id, url, features, label?]
# The DataFrames only have a labels column if labels are given.
# We drop the text, since Naive Bayes doesn't use it and we already have all the tokens
kind = 'asm' if asm else 'bytes'
train = elizabeth.load(train_x, train_y, base=base, kind=kind).drop('text')
test = elizabeth.load(test_x, test_y, base=base, kind=kind).drop('text')
# convert the string labels to numeric indices
# the handleInvalid param allows the label indexer to deal with labels that weren't seen during fitting
label_indexer = StringIndexer(inputCol='label',
outputCol='indexedLabel',
handleInvalid="skip")
label_indexer = label_indexer.fit(train)
train = label_indexer.transform(train)
# the test set won't always have labels
if test_y is not None:
test = label_indexer.transform(test)
index_labeller = IndexToString(inputCol='prediction',
outputCol='predictedClass',
labels=label_indexer.labels)
# Train the preprocessor and transform the data.
prep = elizabeth.Preprocessor()
prep.add(NGram(n=int(ngram)))
prep.add(CountVectorizer())
if idf: prep.add(IDF())
train = prep.fit(train)
test = prep.transform(test)
# Naive Bayes : DF[id, url, text, features, label?, rawPrediction, probability, prediction]
nb = NaiveBayes(labelCol='indexedLabel').fit(train)
test = nb.transform(test)
test = index_labeller.transform(
test) # DF[id, url, ... prediction, predictedClass]
# If labels are given for the test set, print a score.s
if test_y:
test = test.orderBy(test.id)
test = test.withColumn(
'correct', (test.label == test.predictedClass).cast('double'))
test = test.select(avg(test.correct))
print(test.show())
# If no labels are given for the test set, print predictions.
else:
test = test.orderBy(test.id).select(test.predictedClass)
test = test.rdd.map(lambda row: int(row.predictedClass))
test = test.toLocalIterator()
print(*test, sep='\n')
def exec_method(spark, data):
# Index labels, adding metadata to the label column.
# Fit on whole dataset to include all labels in index.
labelIndexer = StringIndexer(inputCol="label",
outputCol="indexedLabel").fit(data)
# Automatically identify categorical features, and index them.
# Set maxCategories so features with > 4 distinct values are treated as continuous.
featureIndexer = \
VectorIndexer(inputCol="features", outputCol="indexedFeatures", maxCategories=4).fit(data)
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Convert indexed labels back to original labels.
labelConverter = IndexToString(inputCol="prediction",
outputCol="predictedLabel",
labels=labelIndexer.labels)
# dtree
rf = RandomForestClassifier(labelCol="indexedLabel",
featuresCol="indexedFeatures")
pipeline = Pipeline(
stages=[labelIndexer, featureIndexer, rf, labelConverter])
model = pipeline.fit(trainingData)
# gbt
# gbt = GBTClassifier(labelCol="indexedLabel", featuresCol="indexedFeatures", maxIter=10)
# pipeline = Pipeline(stages=[labelIndexer, featureIndexer, gbt, labelConverter])
# model = pipeline.fit(trainingData)
# Make predictions.
predictions = model.transform(testData)
# Select example rows to display.
predictions.select("predictedLabel", "label", "features").show(5)
print("predictedLabel=1: " +
str(predictions.filter("predictedLabel=1.0").count()))
print("label=1: " + str(predictions.filter("label=1.0").count()))
print("total: " + str(predictions.count()))
# Select (prediction, true label) and compute test error
evaluator = MulticlassClassificationEvaluator(labelCol="indexedLabel",
predictionCol="prediction",
metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test Error = %g" % (1.0 - accuracy))
rfModel = model.stages[2]
print(rfModel) # summary only
# $example off$
print("model detail\n:" + rfModel.toDebugString)
spark.stop()
def run_randomforest(df):
(train_data, test_data) = df.randomSplit([0.8, 0.2], 1234)
labelIndexer = StringIndexer(
inputCol="quality", outputCol="indexedLabel"
).fit(
df
) # Identify and index labels that could be fit through classification pipeline
assembler = VectorAssembler(
inputCols=['temp', 'pressure', 'duration'],
outputCol="features").setHandleInvalid(
"skip"
) # Incorporate all input fields as vector for classificaion pipeline
# scaler = StandardScaler(inputCol="features_assembler", outputCol="features") # Scale input fields using standard scale
labelConverter = IndexToString(
inputCol="prediction",
outputCol="predicted_quality",
labels=labelIndexer.labels
) # Convert/Lookup prediction label index to actual label
numTreesList = [10, 25, 50]
maxDepthList = [3, 10, 5]
for numTrees, maxDepth in [(numTrees, maxDepth)
for numTrees in numTreesList
for maxDepth in maxDepthList]:
params = {
"numTrees": numTrees,
"maxDepth": maxDepth,
"model": "RandomForest"
}
params.update({
"model_data_date":
model_data_date['start_date'] + ' - ' + model_data_date['end_date']
})
if run_exists(mlflow_exp_id, params):
print("Trees: %s, Depth: %s, Run already exists" %
(numTrees, maxDepth))
else:
rf = RandomForestClassifier(labelCol="indexedLabel",
featuresCol="features",
numTrees=numTrees,
maxDepth=maxDepth,
seed=512)
model, predictions, accuracy, ml_run_info = randomforest_model(
[labelIndexer, assembler, rf, labelConverter], params,
train_data, test_data)
print("Trees: %s, Depth: %s, Accuracy: %s\n" %
(numTrees, maxDepth, accuracy))
mlflow_search_query = "params.model = 'RandomForest' and params.model_data_date = '" + model_data_date[
'start_date'] + ' - ' + model_data_date['end_date'] + "'"
return best_run(mlflow_exp_id, mlflow_search_query)
def pipeline_for_feature_cols(input_df_with_non_features):
string_indexer = StringIndexer(
inputCol="category", outputCol="label").fit(input_df_with_non_features)
return Pipeline(stages=[
string_indexer,
VectorAssembler(inputCols=["id"], outputCol="features"),
LogisticRegression(),
IndexToString(inputCol="prediction",
outputCol="originalLabel",
labels=string_indexer.labels),
])
def __predict_sentiment(self, schema):
"""Gets predictions for a given tweet formatted RDD
Returns: an RDD with format {"sentiment":"POSITIVE"}
"""
test = self.pipelineFit.transform(schema)
predict = self.model.transform(test)
converter = IndexToString(inputCol="prediction",
outputCol="predicted_label",
labels=self.pipelineFit.stages[3].labels)
converted = converter.transform(predict)
return converted.select("predicted_label").collect()[0].asDict()
def perceptron_random_forest_train(sp_session, df_p):
data = df_p
# Index labels, adding metadata to the label column.
# Fit on whole dataset to include all labels in index.
labelIndexer = StringIndexer(inputCol="label",
outputCol="indexedLabel").fit(data)
# Automatically identify categorical features, and index them.
# Set maxCategories so features with > 4 distinct values are treated as continuous.
featureIndexer =\
VectorIndexer(inputCol="features", outputCol="indexedFeatures", maxCategories=4).fit(data)
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
rf = RandomForestClassifier(labelCol="indexedLabel",
featuresCol="indexedFeatures",
numTrees=10)
#rf = RandomForestClassifier(labelCol="label", featuresCol="features", numTrees=10)
# Convert indexed labels back to original labels.
labelConverter = IndexToString(inputCol="prediction",
outputCol="predictedLabel",
labels=labelIndexer.labels)
# Chain indexers and forest in a Pipeline
pipeline = Pipeline(
stages=[labelIndexer, featureIndexer, rf, labelConverter])
# Train model. This also runs the indexers.
# Train on complete data
model = pipeline.fit(data)
##model = rf.fit(trainingData)
#Save the model
model.save("ml_randomforest.model")
# Make predictions.
predictions = model.transform(testData)
# Select example rows to display.
predictions.select("predictedLabel", "label", "features").show(5)
# Select (prediction, true label) and compute test error
evaluator = MulticlassClassificationEvaluator(labelCol="indexedLabel",
predictionCol="prediction",
metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test Error = %g" % (1.0 - accuracy))
rfModel = model.stages[2]
print(rfModel) # summary only
def run_xgboost(df):
(train_data, test_data) = df.randomSplit([0.8, 0.2])
labelIndexer = StringIndexer(
inputCol="quality", outputCol="indexedLabel"
).fit(
df
) # Identify and index labels that could be fit through classification pipeline
assembler = VectorAssembler(
inputCols=['temp', 'pressure', 'duration'],
outputCol="features_assembler").setHandleInvalid(
"skip"
) # Incorporate all input fields as vector for classificaion pipeline
scaler = StandardScaler(
inputCol="features_assembler",
outputCol="features") # Scale input fields using standard scale
labelConverter = IndexToString(
inputCol="prediction",
outputCol="predicted_quality",
labels=labelIndexer.labels
) # Convert/Lookup prediction label index to actual label
numTreesList = [10, 25, 50]
learningRateList = [.1, .2, .3]
for numTrees, learningRate in [(numTrees, learningRate)
for numTrees in numTreesList
for learningRate in learningRateList]:
params = {
"numTrees": numTrees,
"learningRate": learningRate,
"model": "XGBoost"
}
params.update(model_data_date)
if run_exists(mlflow_exp_id, params):
print("Trees: %s, learning Rate: %s, Run already exists" %
(numTrees, learningRate))
else:
xgboost = XGBoostEstimator(labelCol="indexedLabel",
featuresCol="features",
eta=learningRate,
maxDepth=maxDepth)
model, predictions, accuracy, ml_run_info = xgboost_model(
[labelIndexer, assembler, scaler, rf, labelConverter], params,
train_data, test_data)
print("Trees: %s, learning Rate: %s, Accuracy: %s\n" %
(numTrees, learningRate, accuracy))
mlflow_search_query = "params.model = 'XGBoost' and params.model_data_date = '" + model_data_date[
'model_data_date'] + "'"
return best_run(mlflow_exp_id, mlflow_search_query)
def index2string(df,columns:list,param):
for column in columns:
column_new=column+"_str"
print(f"index2string {column} to {column_new}")
labels=param[column+"_labels"]
model=IndexToString(inputCol=column,outputCol=column_new,labels=labels)
df=model.transform(df)\
.withColumn(column,col(column_new))\
.drop(column_new)
return df
def indexToString(infoData):
stringIndexerPath = infoData.get(mc.INDEXERPATH)
inverterColm = infoData.get(mc.COLMTOINVERT)
dataset = infoData.get(mc.DATASET)
stringIndexer = StringIndexerModel.load(stringIndexerPath)
inverter = IndexToString(inputCol=inverterColm, outputCol=mc.DMXINVERTEDCOLM,
labels=stringIndexer.labels)
dataset = inverter.transform(dataset)
#drop the indexed colm and rename the new unindexed colm with the actual one
dataset = dataset.drop(inverterColm)
dataset = dataset.withColumnRenamed(mc.DMXINVERTEDCOLM, inverterColm)
return dataset
def test_index_to_string_throws(self):
original_data = self.spark.createDataFrame(
[(0, "a"), (1, "b"), (2, "c"), (3, "a"), (4, "a"), (5, "c")],
["id", "category"])
string_indexer = StringIndexer(inputCol="category", outputCol="categoryIndex")
string_indexer_model = string_indexer.fit(original_data)
data = string_indexer_model.transform(original_data)
model = IndexToString(inputCol="categoryIndex", outputCol="originalCategory")
# the input name should match that of what IndexToString.inputCol
model_onnx = None
with pytest.raises(SparkMlConversionError):
model_onnx = convert_sparkml(model, 'Sparkml IndexToString', [('categoryIndex', Int64TensorType([1, 1]))])
def multinomial_lr_with_index_to_string_stage_pipeline(
multinomial_df_with_string_labels):
string_indexer = StringIndexer(
inputCol="category",
outputCol="label").fit(multinomial_df_with_string_labels)
return Pipeline(stages=[
string_indexer,
VectorAssembler(inputCols=["id"], outputCol="features"),
LogisticRegression(),
IndexToString(inputCol="prediction",
outputCol="originalLabel",
labels=string_indexer.labels),
])
def random_decision_tree_usecase():
spark = getSparkSession()
# Load and parse the data file, converting it to a DataFrame.
data = spark.read.format("libsvm").load("../data/lib_svm.txt")
# Index labels, adding metadata to the label column.
# Fit on whole dataset to include all labels in index.
labelIndexer = StringIndexer(inputCol="label",
outputCol="indexedLabel").fit(data)
# Automatically identify categorical features, and index them.
# Set maxCategories so features with > 4 distinct values are treated as continuous.
featureIndexer = \
VectorIndexer(inputCol="features", outputCol="indexedFeatures", maxCategories=4).fit(data)
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
rf = RandomForestClassifier(labelCol="indexedLabel",
featuresCol="indexedFeatures",
numTrees=10)
# Convert indexed labels back to original labels.
labelConverter = IndexToString(inputCol="prediction",
outputCol="predictedLabel",
labels=labelIndexer.labels)
# Chain indexers and forest in a Pipeline
pipeline = Pipeline(
stages=[labelIndexer, featureIndexer, rf, labelConverter])
# Train model. This also runs the indexers.
model = pipeline.fit(trainingData)
# Make predictions.
predictions = model.transform(testData)
# Select example rows to display.
predictions.select("predictedLabel", "label", "features").show(5)
# Select (prediction, true label) and compute test error
evaluator = MulticlassClassificationEvaluator(labelCol="indexedLabel",
predictionCol="prediction",
metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test Error = %g" % (1.0 - accuracy))
rfModel = model.stages[2]
print(rfModel) # summary only
def keywords_naive_bayes_classifier(sc, training_path, number_of_features,
train_path_parquet, eval_path_parquet,
save, path_to_save):
sqlContext = SQLContext(sc)
# if classification_type == 'home_pages':
# classifier_utils.prepare_input_for_home_page_classifier(
# sc,sqlContext,training_path,evaluation_rdd,prepare_training_input,train_path_parquet,eval_path_parquet)
# elif classification_type == 'cluster_pages':
# classifier_utils.prepare_input_for_cluster_page_classifier(
# sc,sqlContext,training_path,evaluation_rdd,prepare_training_input,train_path_parquet,eval_path_parquet)
schemaTrain = sqlContext.read.load(train_path_parquet)
schemaEval = sqlContext.read.load(eval_path_parquet)
categoryIndexer = StringIndexer(inputCol="category", outputCol="label")
tokenizer = Tokenizer(inputCol="text", outputCol="words")
hashingTF = HashingTF(inputCol="words",
outputCol="features",
numFeatures=number_of_features)
nb = NaiveBayes(smoothing=1.0, modelType="multinomial")
categoryConverter = IndexToString(inputCol="label", outputCol="pred")
pipeline = Pipeline(
stages=[categoryIndexer, tokenizer, hashingTF, nb, categoryConverter])
model = pipeline.fit(schemaTrain)
pr = model.transform(schemaEval)
classifier_utils.print_metrics(pr, 'label', 'prediction')
output = pr.rdd
# if classification_type == 'home_pages':
# output = output.filter(lambda row: row.prediction == 1.0)
# if classification_type == 'cluster_pages':
output = output.map(
lambda row: {
'url': row.url,
'cluster_label': row.cluster_label,
'probability': row.probability,
'prediction': row.prediction,
'pred': row.pred
})
# |category|cluster_label|domain|referring_url|text|url|label|words|features| rawPrediction|probability|prediction|pred
rdd_utils.save_rdd(output, save, path_to_save)
return output
def train_test(self, df):
df = self.dropNonTCPUDP(df)
catCols = []
numCols = ['avg_ipt', 'bytes_in', 'bytes_out', 'entropy', 'total_entropy', 'num_pkts_out', 'num_pkts_in', 'duration']
labelCol = 'label'
data = self.get_dummy(df, catCols, numCols, labelCol)
data.show()
labelIndexer = StringIndexer(inputCol='label',
outputCol='indexedLabel').fit(data)
labelIndexer.transform(data)
featureIndexer = VectorIndexer(inputCol="features", \
outputCol="indexedFeatures").fit(data)
featureIndexer.transform(data)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
trainingData.cache()
# trainingData.repartition(200)
testData.cache()
# testData.repartition(200)
trainingData.show(5,False)
testData.show(5,False)
rf = RandomForestClassifier(featuresCol='indexedFeatures', labelCol='indexedLabel')
gbt = GBTClassifier(featuresCol='indexedFeatures', labelCol='indexedLabel')
logr = LogisticRegression(featuresCol='indexedFeatures', labelCol='indexedLabel')
# Convert indexed labels back to original labels.
labelConverter = IndexToString(inputCol="prediction", outputCol="predictedLabel",
labels=labelIndexer.labels)
pipeline = Pipeline(stages=[labelIndexer, featureIndexer, gbt, labelConverter])
model = pipeline.fit(trainingData)
predictions = model.transform(testData)
# Select example rows to display.
predictions.select("features","label","predictedLabel", "prediction")
# Select (prediction, true label) and compute test error
print(self.getTestError(predictions))
self.printMetrics(predictions)
# print(self.ExtractFeatureImp(model.stages[-2].featureImportances, testData, "features"))
return model
def main():
sensor_data_df = spark.read.parquet('sensor_data_ts')
# creating ML pipelines for classification and regression problems
sensor_data_df = sensor_data_df.select(
sensor_data_df['datetime'], sensor_data_df['message_code_name'],
sensor_data_df['H2S'], sensor_data_df['CO'], sensor_data_df['LEL'],
sensor_data_df['O2']).orderBy(sensor_data_df['datetime'].asc())
condition = (sensor_data_df['datetime'] < datetime(2020, 10, 1))
train_df = sensor_data_df.where(condition)
test_df = sensor_data_df.where(~condition).cache()
""" CLASSIFICATION PROBLEM FOR GAS EVENT IDENTIFICATION"""
classification_set = train_df.select(
train_df['message_code_name'].alias('target'), train_df['H2S'],
train_df['CO'], train_df['LEL'], train_df['O2'])
train_set, validation_set = classification_set.randomSplit([0.75, 0.25])
train_set.cache()
validation_set.cache()
# message_code_name is the target column representing gas events (Normal, GasHighAlarm, GasLowAlarm, GasAlarm)
# sql_transformer_statement = "SELECT datetime ,H2S, LEL, O2, CO, message_code_name AS target " \
# # "FROM __THIS__"
# sql_transformer = SQLTransformer(statement=sql_transformer_statement_2)
assemble_features = VectorAssembler(inputCols=['H2S', 'LEL', 'O2', 'CO'],
outputCol='features')
target_indexer = StringIndexer(inputCol='target',
outputCol='label').fit(train_set)
label_converter = IndexToString(inputCol='label',
outputCol='predicted_target')
print(target_indexer.labels)
classifier = MultilayerPerceptronClassifier(layers=[4, 20, 4])
pipeline = Pipeline(stages=[
assemble_features, target_indexer, classifier, label_converter
])
model = pipeline.fit(train_set)
predictions = model.transform(validation_set)
predictions.select(['target', 'predicted_target']).show(30)
evaluator = MulticlassClassificationEvaluator(predictionCol='prediction',
labelCol='label')
prediction_score = evaluator.evaluate(predictions)
print('Gas Events Prediction Score: ', prediction_score)
model.write().overwrite().save('mlp_model')
def predict(rdd, bestModel):
if (not rdd.isEmpty()):
df = sqlContext.createDataFrame(rdd).toDF("descr")
predictions = bestModel.transform(df)
converter = IndexToString(inputCol="prediction",
outputCol="label",
labels=bestModel.stages[0].labels)
labelReverse = converter.transform(predictions)
print("predictions for tweet:")
print(
labelReverse.select("features", "probability", "prediction",
"label").show())
labelReverse.write.mode('append').parquet(
'hdfs:///predictions/tweets_predictions.parquet')
else:
print("No data received")