def test_model_onehot_encoder(self):
encoder = OneHotEncoderEstimator(inputCols=['index'],
outputCols=['indexVec'])
data = self.spark.createDataFrame([(0.0, ), (1.0, ), (2.0, ), (2.0, ),
(0.0, ), (2.0, )], ['index'])
model = encoder.fit(data)
model_onnx = convert_sparkml(model, 'Sparkml OneHotEncoder',
[('index', FloatTensorType([1, 1]))])
self.assertTrue(model_onnx is not None)
self.assertTrue(model_onnx.graph.node is not None)
# run the model
predicted = model.transform(data)
data_np = data.select("index").toPandas().values.astype(numpy.float32)
predicted_np = predicted.select("indexVec").toPandas().indexVec.apply(
lambda x: x.toArray().tolist()).values
expected = numpy.asarray(
[x + [0] if numpy.amax(x) == 1 else x + [1] for x in predicted_np])
paths = save_data_models(data_np,
expected,
model,
model_onnx,
basename="SparkmlOneHotEncoder")
onnx_model_path = paths[3]
output, output_shapes = run_onnx_model(['indexVec'], data_np,
onnx_model_path)
compare_results(expected, output, decimal=5)
def encoding2(df, incol, outcol):
encoder = OneHotEncoderEstimator(inputCols=[incol], outputCols=[outcol])
encoder = encoder.fit(df)
df = encoder.transform(df)
return df #, encoder
def preprocess(data_frame):
category_columns = [
"workclass", "education", "marital_status", "occupation",
"relationship", "race", "sex", "native_country", "income"
]
index_columns = [col + "_index" for col in category_columns]
vec_columns = [col + "_vec" for col in category_columns]
for col in category_columns:
stringIndexer = StringIndexer(inputCol=col,
outputCol=col + "_index",
handleInvalid='error')
model = stringIndexer.fit(data_frame)
data_frame = model.transform(data_frame)
data_frame = data_frame.drop(col)
index_columns.pop(-1)
vec_columns.pop(-1)
ohe = OneHotEncoderEstimator(inputCols=index_columns,
outputCols=vec_columns)
ohe_model = ohe.fit(data_frame)
ohe_df = ohe_model.transform(data_frame)
ohe_df = ohe_df.drop(*index_columns)
# ohe_df.show()
cols = ohe_df.columns
cols.remove("income_index")
vector_assembler = VectorAssembler(inputCols=cols, outputCol="features")
vdata_frame = vector_assembler.transform(ohe_df)
vdata_frame = vdata_frame.drop(*cols)
# vdata_frame.show()
print("Preprocess input data correctly.")
return vdata_frame
def encoding(i, df, col):
encoder = OneHotEncoderEstimator(inputCols=[col],
outputCols=["p" + str(i)])
encoder = encoder.fit(df)
df = encoder.transform(df)
return df #, encoder
def OneHotEncode(df, columns):
from pyspark.ml.feature import OneHotEncoderEstimator
df1 = df
for col in columns:
encoder = OneHotEncoderEstimator(inputCols=[col + '_Index'], outputCols=[col+'_Vector'])
df1 = encoder.fit(df1).transform(df1).drop(col + '_Index')
print('---> ', type(df1[col+'_Vector']), df1.columns, df1.take(1))
return df1
def oneHotEncoderExample(movieSamples):
samplesWithIdNumber = movieSamples.withColumn(
"movieIdNumber",
F.col("movieId").cast(IntegerType()))
encoder = OneHotEncoderEstimator(inputCols=["movieIdNumber"],
outputCols=['movieIdVector'],
dropLast=False)
oneHotEncoderSamples = encoder.fit(samplesWithIdNumber).transform(
samplesWithIdNumber)
oneHotEncoderSamples.printSchema()
oneHotEncoderSamples.show(10)
def onehot(self):
from pyspark.ml.feature import OneHotEncoderEstimator
df = self.session.createDataFrame([(0.0, 1.0), (1.0, 0.0), (2.0, 1.0),
(0.0, 2.0), (0.0, 1.0), (2.0, 0.0)],
["categoryIndex1", "categoryIndex2"])
encoder = OneHotEncoderEstimator(
inputCols=["categoryIndex1", "categoryIndex2"],
outputCols=["categoryVec1", "categoryVec2"])
model = encoder.fit(df)
encoded = model.transform(df)
encoded.show()
def main(train_data_folder, model_path, onehot_path):
#Starting session
spark = SparkSession.builder.appName('BigDataML').getOrCreate()
spark.sparkContext.setLogLevel("ERROR")
#Loading data
data = spark.read.parquet(train_data_folder)
data = data.dropna(how='any')
encoder = OneHotEncoderEstimator(inputCols=["station_index"],
outputCols=["station_vector"])
encoder_model = encoder.fit(data)
data = encoder_model.transform(data)
encoder_model.write().overwrite().save(onehot_path)
def get_df(reps=REPS):
def make_rows(i):
f = float(i)
return [
("assert", Vectors.sparse(3, [0, 1, 2], [1, 2, 3]), 0,
Vectors.dense([f + 0.1, f + 1.1])),
("require", Vectors.sparse(3, {1: 2}), 1,
Vectors.dense([f + 2.2, f + 3.2])),
]
rows = []
for i in range(reps):
rows.extend(make_rows(i))
df = sc.parallelize(rows).toDF(["word", "vector", "more", "vorpal"])
ohe = OneHotEncoderEstimator(inputCols=['more'], outputCols=['more__ohe'])
return ohe.fit(df).transform(df)
def encode_using_one_hot(df, column_name):
'''
Transforms a df at a particular column_name by converting all unique categories to a vector as they get processed.
ie. If values in column are {a,b,b,c,d} => {<1.0,0,0>, <0,1.0,0>, ... } (Good for non-ordinal categories)
'''
indexed_name = 'index_'+column_name
vectored_name = 'vec_'+column_name
df = StringIndexer(inputCol=column_name, outputCol=indexed_name,
handleInvalid="skip").fit(df).transform(df)
encoder = OneHotEncoderEstimator(
inputCols=[indexed_name], outputCols=[vectored_name])
model = encoder.fit(df)
df = model.transform(df)
df = df.drop(indexed_name)
df = df.drop(column_name)
df = df.withColumnRenamed(vectored_name, column_name)
return df
def test_model_onehot_encoder(self):
import numpy
encoder = OneHotEncoderEstimator(inputCols=['index'],
outputCols=['indexVec'])
data = self.spark.createDataFrame([(0.0, ), (1.0, ), (2.0, ), (2.0, ),
(0.0, ), (2.0, )], ['index'])
model = encoder.fit(data)
model_onnx = convert_sparkml(model, 'Sparkml OneHotEncoder',
[('index', FloatTensorType([1, 1]))])
self.assertTrue(model_onnx is not None)
self.assertTrue(model_onnx.graph.node is not None)
# run the model
predicted = model.transform(data)
data_np = data.select("index").toPandas().values.astype(numpy.float32)
predicted_np = predicted.select("indexVec").toPandas().indexVec.apply(
lambda x: x.toArray().tolist()).values
predicted_shifted = numpy.asarray(
[x + [0] if numpy.amax(x) == 1 else x + [1] for x in predicted_np])
dump_data_and_sparkml_model(data_np,
predicted_shifted,
model,
model_onnx,
basename="SparkmlOneHotEncoder")
def train_model(args):
# do not run this test for pytorch lightning below min supported verson
import pytorch_lightning as pl
if LooseVersion(pl.__version__) < LooseVersion(MIN_PL_VERSION):
print("Skip test for pytorch_ligthning=={}, min support version is {}".format(pl.__version__, MIN_PL_VERSION))
return
# Initialize SparkSession
conf = SparkConf().setAppName('pytorch_spark_mnist').set('spark.sql.shuffle.partitions', '16')
if args.master:
conf.setMaster(args.master)
elif args.num_proc:
conf.setMaster('local[{}]'.format(args.num_proc))
spark = SparkSession.builder.config(conf=conf).getOrCreate()
# Setup our store for intermediate data
store = Store.create(args.work_dir)
# Download MNIST dataset
data_url = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/mnist.bz2'
libsvm_path = os.path.join(args.data_dir, 'mnist.bz2')
if not os.path.exists(libsvm_path):
subprocess.check_output(['wget', data_url, '-O', libsvm_path])
# Load dataset into a Spark DataFrame
df = spark.read.format('libsvm') \
.option('numFeatures', '784') \
.load(libsvm_path)
# One-hot encode labels into SparseVectors
encoder = OneHotEncoder(inputCols=['label'],
outputCols=['label_vec'],
dropLast=False)
model = encoder.fit(df)
train_df = model.transform(df)
# Train/test split
train_df, test_df = train_df.randomSplit([0.9, 0.1])
# Define the PyTorch model without any Horovod-specific parameters
class Net(LightningModule):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.conv2_drop = nn.Dropout2d()
self.fc1 = nn.Linear(320, 50)
self.fc2 = nn.Linear(50, 10)
def forward(self, x):
x = x.float().reshape((-1, 1, 28, 28))
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.log_softmax(x, -1)
def configure_optimizers(self):
return optim.SGD(self.parameters(), lr=0.01, momentum=0.5)
def training_step(self, batch, batch_idx):
if batch_idx == 0:
print(f"training data batch size: {batch['label'].shape}")
x, y = batch['features'], batch['label']
y_hat = self(x)
loss = F.nll_loss(y_hat, y.long())
self.log('train_loss', loss)
return loss
def validation_step(self, batch, batch_idx):
if batch_idx == 0:
print(f"validation data batch size: {batch['label'].shape}")
x, y = batch['features'], batch['label']
y_hat = self(x)
loss = F.nll_loss(y_hat, y.long())
self.log('val_loss', loss)
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() if len(outputs) > 0 else float('inf')
self.log('avg_val_loss', avg_loss)
model = Net()
# Train a Horovod Spark Estimator on the DataFrame
backend = SparkBackend(num_proc=args.num_proc,
stdout=sys.stdout, stderr=sys.stderr,
prefix_output_with_timestamp=True)
from pytorch_lightning.callbacks import Callback
epochs = args.epochs
class MyDummyCallback(Callback):
def __init__(self):
self.epcoh_end_counter = 0
self.train_epcoh_end_counter = 0
self.validation_epoch_end_counter = 0
def on_init_start(self, trainer):
print('Starting to init trainer!')
def on_init_end(self, trainer):
print('Trainer is initialized.')
def on_epoch_end(self, trainer, model):
print('A train or eval epoch ended.')
self.epcoh_end_counter += 1
def on_train_epoch_end(self, trainer, model, unused=None):
print('A train epoch ended.')
self.train_epcoh_end_counter += 1
def on_validation_epoch_end(self, trainer, model, unused=None):
print('A val epoch ended.')
self.validation_epoch_end_counter += 1
def on_train_end(self, trainer, model):
print("Training ends:"
f"epcoh_end_counter={self.epcoh_end_counter}, "
f"train_epcoh_end_counter={self.train_epcoh_end_counter}, "
f"validation_epoch_end_counter={self.validation_epoch_end_counter} \n")
assert self.train_epcoh_end_counter <= epochs
assert self.epcoh_end_counter == self.train_epcoh_end_counter + self.validation_epoch_end_counter
callbacks = [MyDummyCallback()]
# added EarlyStopping and ModelCheckpoint
from pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint
callbacks.append(ModelCheckpoint(monitor='val_loss', mode="min",
save_top_k=1, verbose=True))
from pytorch_lightning.callbacks.early_stopping import EarlyStopping
callbacks.append(EarlyStopping(monitor='val_loss',
min_delta=0.001,
patience=3,
verbose=True,
mode='min'))
torch_estimator = hvd.TorchEstimator(backend=backend,
store=store,
model=model,
input_shapes=[[-1, 1, 28, 28]],
feature_cols=['features'],
label_cols=['label'],
batch_size=args.batch_size,
epochs=args.epochs,
validation=0.1,
verbose=1,
callbacks=callbacks,
profiler="simple" if args.enable_profiler else None)
torch_model = torch_estimator.fit(train_df).setOutputCols(['label_prob'])
# Evaluate the model on the held-out test DataFrame
pred_df = torch_model.transform(test_df)
argmax = udf(lambda v: float(np.argmax(v)), returnType=T.DoubleType())
pred_df = pred_df.withColumn('label_pred', argmax(pred_df.label_prob))
evaluator = MulticlassClassificationEvaluator(predictionCol='label_pred', labelCol='label', metricName='accuracy')
print('Test accuracy:', evaluator.evaluate(pred_df))
spark.stop()
data = data.select('Var1', 'Var2', 'Var3', 'Var4', 'Var5', 'Var6', 'Var7',
'Var8', 'NVVar1', 'NVVar2', 'NVVar3', 'NVVar4',
'Cat1_index', 'Cat3_index', 'Cat6_index', 'Cat8_index',
'Cat9_index', 'Cat10_index', 'Cat11_index', 'Cat12_index',
'Calendar_Year', 'Model_Year', 'Claim_Amount')
category = [
'Cat1_index', 'Cat3_index', 'Cat6_index', 'Cat8_index', 'Cat9_index',
'Cat10_index', 'Cat11_index', 'Cat12_index'
]
new_cat = []
for col in category:
name = col.replace('_index', '_vec')
new_cat.append(name)
num_column.append(name)
encoder = OneHotEncoderEstimator(inputCols=category, outputCols=new_cat)
one_model = encoder.fit(data)
data = one_model.transform(data)
#question 1.3 handle the unbalanced data
data = data.withColumn('weight',
when((data['Claim_Amount'] != 0), 0.99).otherwise(0.01))
#allocate label for each row
data = data.withColumn('type',
when((data['Claim_Amount'] != 0), 1).otherwise(0))
#feature choose and PCA
data = data.select('Var1', 'Var2', 'Var3', 'Var4', 'Var5', 'Var6', 'Var7',
'Var8', 'NVVar1', 'NVVar2', 'NVVar3', 'NVVar4', 'Cat1_vec',
'Cat3_vec', 'Cat6_vec', 'Cat8_vec', 'Cat9_vec', 'Cat10_vec',
'Cat11_vec', 'Cat12_vec', 'Calendar_Year', 'Model_Year',
'Claim_Amount', 'weight', 'type')
# LGA — La Guardia Airport (New York)
# SMF — Sacramento
# SJC — San Jose
# TUS — Tucson International Airport
# OGG — Kahului (Hawaii)
# Obviously this is only a small subset of airports. Nevertheless, since this is a categorical variable, it needs to be one-hot encoded before it can be used in a regression model.
# The data are in a variable called flights. You have already used a string indexer to create a column of indexed values corresponding to the strings in org.
# Note:: You might find it useful to revise the slides from the lessons in the Slides panel next to the IPython Shell.
# Instructions
# 100 XP
# Import the one-hot encoder class.
# Create an one-hot encoder instance, naming the output column 'org_dummy'.
# Apply the one-hot encoder to the flights data.
# Generate a summary of the mapping from categorical values to binary encoded dummy variables. Include only unique values and order by org_idx.
# Import the one hot encoder class
from pyspark.ml.feature import OneHotEncoderEstimator
# Create an instance of the one hot encoder
onehot = OneHotEncoderEstimator(inputCols=['org_idx'], outputCols=['org_dummy'])
# Apply the one hot encoder to the flights data
onehot = onehot.fit(flights)
flights_onehot = onehot.transform(flights)
# Check the results
flights_onehot.select('org', 'org_idx', 'org_dummy').distinct().sort('org_idx').show()
from pyspark.ml.feature import StringIndexer
indexer = StringIndexer(inputCol="color", outputCol="color_indexed")
#Next we call the fit() method to initiate the learning process.
indexer_model = indexer.fit(data)
indexed_data = indexer_model.transform(data)
# to view the data
indexed_data.show()
###########################################One Hot Encoding#######################################
from pyspark.ml.feature import OneHotEncoderEstimator
ohe = OneHotEncoderEstimator(inputCols=["color_indexed"],
outputCols=["color_ohe"])
ohe_model = ohe.fit(indexed_data)
encoded_data = ohe_model.transform(indexed_data)
encoded_data.show()
############################ Feature Scaling #####################################
'''
This diversity in scale could cause a lot of problems in some machine learning algorithms e.g. KMeans.
This is because the algorithm may treat some variables as more dominant according to their value range.
For example: consider a dataset about employees. We may have a years of experience column that ranges between 0 → 30
and a salary column with values in thousands. But this does not mean that the salary column is more dominant!
To solve this problem we transform the values to be at the same scale. There are a lot of transformation methods,
we will look at two of them.
Note that scalers are applied on Vector Data Types that is why we need to collect the features using a VectorAssembler first:
'''
from pyspark.ml.feature import VectorAssembler
# $example on$
from pyspark.ml.feature import OneHotEncoderEstimator
# $example off$
from pyspark.sql import SparkSession
if __name__ == "__main__":
spark = SparkSession\
.builder\
.appName("OneHotEncoderEstimatorExample")\
.getOrCreate()
# Note: categorical features are usually first encoded with StringIndexer
# $example on$
df = spark.createDataFrame([
(0.0, 1.0),
(1.0, 0.0),
(2.0, 1.0),
(0.0, 2.0),
(0.0, 1.0),
(2.0, 0.0)
], ["categoryIndex1", "categoryIndex2"])
encoder = OneHotEncoderEstimator(inputCols=["categoryIndex1", "categoryIndex2"],
outputCols=["categoryVec1", "categoryVec2"])
model = encoder.fit(df)
encoded = model.transform(df)
encoded.show()
# $example off$
spark.stop()
print("Cars with null cyl", cars.filter('cyl IS NULL').count(), '\n')
indexer = StringIndexer(inputCol='type',
outputCol='type_idx')
# Assign index values to strings
indexer = indexer.fit(cars)
# Create column with index values
cars = indexer.transform(cars)
cars = cars.withColumn('density', round(cars.weight_kg / cars.length_meters, 2))
cars = cars.withColumn('density_area', round(cars.weight_kg / cars.length_meters**2, 2))
cars = cars.withColumn('density_volume', round(cars.weight_kg / cars.length_meters**3, 2))
onehot = OneHotEncoderEstimator(inputCols=['type_idx'], outputCols=['type_dummy'])
onehot = onehot.fit(cars)
cars = onehot.transform(cars)
pd.set_option('display.max_columns', None) # all cols
pd.set_option('display.width', 161)
pd.set_option('display.max_colwidth', 199)
#print(cars.toPandas().sample(12))
# Check column data types
print('\n', cars.dtypes, '\n')
assembler = VectorAssembler(inputCols=['weight_kg', 'cyl', 'type_dummy', 'density', 'density_area', 'density_volume'],
outputCol='features')
cars = assembler.transform(cars)
kars = cars.select('consumption', 'features')
]
cat_ohe = []
for col in category_id:
cat_ = col.replace('_id', '_ohe')
cat_ohe.append(cat_)
input_features.append(cat_)
data = raw_df.select('Var1', 'Var2', 'Var3', 'Var4', 'Var5', 'Var6', 'Var7',
'Var8', 'NVVar1', 'NVVar2', 'NVVar3', 'NVVar4', 'Cat1_id',
'Cat2_id', 'Cat3_id', 'Cat4_id', 'Cat5_id', 'Cat6_id',
'Cat7_id', 'Cat8_id', 'Cat9_id', 'Cat10_id', 'Cat11_id',
'Cat12_id', 'Calendar_Year', 'Model_Year', 'Claim_Amount')
encoder = OneHotEncoderEstimator(inputCols=category_id, outputCols=cat_ohe)
encoder_data = encoder.fit(data)
data = encoder_data.transform(data)
#assign weight on data
data = data.withColumn('weight',
when((data['Claim_Amount'] != 0), 0.98).otherwise(0.02))
data = data.withColumn('not_zero',
when((data['Claim_Amount'] != 0), 1).otherwise(0))
data = data.select('Var1', 'Var2', 'Var3', 'Var4', 'Var5', 'Var6', 'Var7',
'Var8', 'NVVar1', 'NVVar2', 'NVVar3', 'NVVar4', 'Cat1_ohe',
'Cat2_ohe', 'Cat3_ohe', 'Cat4_ohe', 'Cat5_ohe', 'Cat6_ohe',
'Cat7_ohe', 'Cat8_ohe', 'Cat9_ohe', 'Cat10_ohe',
'Cat11_ohe', 'Cat12_ohe', 'Calendar_Year', 'Model_Year',
'Claim_Amount', 'weight', 'not_zero')
def dataTranform(self, dataInfo):
featuresColm = dataInfo.get(
PredictiveConstants.FEATURESCOLM) # featureColmList -replaced
labelColm = dataInfo.get(PredictiveConstants.LABELCOLM)
modelSheetName = dataInfo.get(PredictiveConstants.MODELSHEETNAME)
modelId = dataInfo.get(PredictiveConstants.MODELID)
storageLocation = dataInfo.get(PredictiveConstants.LOCATIONADDRESS)
indexerPathMapping = {}
oneHotEncoderPathMapping = {}
self.labelColm = None if labelColm == None else labelColm
self.featuresColm = None if featuresColm == None else featuresColm
dataset = self.dataset
vectorizedFeaturescolmName = modelSheetName + PredictiveConstants.DMXFEATURE
dataset = dataset.drop(vectorizedFeaturescolmName)
schemaData = dataset.schema
if self.labelColm is not None:
for labelName in self.labelColm:
label = labelName
else:
label = self.labelColm
nonNumericData = self.nonNumericToString(schemaData=schemaData,
dataset=dataset)
categoricalFeatures = nonNumericData.get(
PredictiveConstants.CATEGORICALFEATURES)
numericalFeatures = nonNumericData.get(
PredictiveConstants.NUMERICALFEATURES)
dataset = nonNumericData.get(PredictiveConstants.DATASET)
schemaData = dataset.schema
# indexing of label column
isLabelIndexed = "no"
if self.labelColm is not None:
labelIndexedInfo = self.isLabelIndexed(schemaData=schemaData,
label=label,
dataset=dataset)
dataset = labelIndexedInfo.get(PredictiveConstants.DATASET)
isLabelIndexed = labelIndexedInfo.get(
PredictiveConstants.ISLABELINDEXED)
labelIndexer = labelIndexedInfo.get(
PredictiveConstants.LABELINDEXER)
# store the label indexer here.
if labelIndexer is not None:
labelIndexerStoragepath = storageLocation + modelId.upper(
) + label.upper() + PredictiveConstants.INDEXER.upper(
) + PredictiveConstants.PARQUETEXTENSION
labelIndexer.save(labelIndexerStoragepath) #correct this
indexerPathMapping.update({label: labelIndexerStoragepath})
oneHotEncodedFeaturesList = []
indexedFeatures = []
nonOneHotEncoded = []
for colm in categoricalFeatures:
indexedColmName = PredictiveConstants.INDEXED_ + colm
oneHotEncodedColName = PredictiveConstants.ONEHOTENCODED_ + colm
indexer = StringIndexer(inputCol=colm,
outputCol=indexedColmName,
handleInvalid="skip").fit(dataset)
dataset = indexer.transform(dataset)
'''store the indexer here- saving mechanism should be modelid+colmName+indexer.parquet -->
not storing the features indexer for now but keeping this for future use.'''
featuresIndexerPath = storageLocation + modelId.upper(
) + colm.upper() + PredictiveConstants.INDEXER.upper(
) + PredictiveConstants.PARQUETEXTENSION
indexer.write().overwrite().save(featuresIndexerPath)
indexerPathMapping.update({colm: featuresIndexerPath})
rowNo = dataset.select(indexedColmName).distinct().count()
'''Incase of only one category in the colm or more than one in case of training and prediction
-- naming it with the onehotencoded colmName to avoid that uncertainity. '''
if (rowNo == 1):
nonOneHotEncoded.append(
oneHotEncodedColName
) # to avoid the problem when only single value colm is passed at the time of prediction.
else:
indexedFeatures.append(indexedColmName)
oneHotEncodedFeaturesList.append(oneHotEncodedColName)
oneHotEncoder = OneHotEncoderEstimator(
inputCols=indexedFeatures,
outputCols=oneHotEncodedFeaturesList,
handleInvalid="error")
oneHotEncoderPath = storageLocation + modelId.upper(
) + PredictiveConstants.ONEHOTENCODED.upper(
) + PredictiveConstants.PARQUETEXTENSION
oneHotEncoder.write().overwrite().save(oneHotEncoderPath)
oneHotEncoderPathMapping.update(
{PredictiveConstants.ONEHOTENCODED: oneHotEncoderPath})
oneHotEncoderFit = oneHotEncoder.fit(dataset)
dataset = oneHotEncoderFit.transform(dataset)
combinedFeatures = oneHotEncodedFeaturesList + numericalFeatures + nonOneHotEncoded
categoryColmListDict = {}
countOfCategoricalColmList = []
for value in categoricalFeatures:
listValue = []
categoryColm = dataset.groupby(value).count()
countOfCategoricalColmList.append(categoryColm.count())
categoryColmJson = categoryColm.toJSON()
for row in categoryColmJson.collect():
categoryColmSummary = json.loads(row)
listValue.append(categoryColmSummary)
categoryColmListDict[value] = listValue
self.numericalFeatures = numericalFeatures
self.categoricalFeatures = categoricalFeatures
if not categoricalFeatures:
maxCategories = 5
else:
maxCategories = max(countOfCategoricalColmList)
dataset = dataset.drop(vectorizedFeaturescolmName)
featureassembler = VectorAssembler(
inputCols=combinedFeatures,
outputCol=vectorizedFeaturescolmName,
handleInvalid="skip")
dataset = featureassembler.transform(dataset)
# retrieve the features colm name after onehotencoding
indexOfFeatures = dataset.schema.names.index(
vectorizedFeaturescolmName)
oneHotEncodedFeaturesDict = dataset.schema.fields[
indexOfFeatures].metadata['ml_attr']['attrs']
idNameFeatures = {}
if not oneHotEncodedFeaturesDict:
idNameFeaturesOrderedTemp = None
else:
for type, value in oneHotEncodedFeaturesDict.items():
for subKey in value:
idNameFeatures[subKey.get("idx")] = subKey.get("name")
idNameFeaturesOrderedTemp = {}
for key in sorted(idNameFeatures):
idNameFeaturesOrderedTemp[key] = idNameFeatures[
key].replace(PredictiveConstants.ONEHOTENCODED_,
"")
idNameFeaturesOrdered = None if idNameFeaturesOrderedTemp == None else idNameFeaturesOrderedTemp
# retrieve the label colm name only after label encoding
indexedLabelNameDict = {}
if isLabelIndexed == "yes":
indexOfLabel = dataset.schema.names.index(label)
indexedLabel = dataset.schema.fields[indexOfLabel].metadata[
"ml_attr"]["vals"]
for value in indexedLabel:
indexedLabelNameDict[indexedLabel.index(value)] = value
# this code was for vector indexer since it is not stable for now from spark end
# so will use it in future if needed.
'''
vec_indexer = VectorIndexer(inputCol='features', outputCol='vec_indexed_features',
maxCategories=maxCategories,
handleInvalid="skip").fit(dataset)
categorical_features = vec_indexer.categoryMaps
print("Choose %d categorical features: %s" %
(len(categorical_features), ", ".join(str(k) for k in categorical_features.keys())))
dataset= vec_indexer.transform(dataset)
'''
result = {
PredictiveConstants.DATASET: dataset,
PredictiveConstants.CATEGORICALFEATURES: categoricalFeatures,
PredictiveConstants.NUMERICALFEATURES: numericalFeatures,
PredictiveConstants.MAXCATEGORIES: maxCategories,
PredictiveConstants.CATEGORYCOLMSTATS: categoryColmListDict,
PredictiveConstants.INDEXEDFEATURES: indexedFeatures,
PredictiveConstants.LABEL: label,
PredictiveConstants.ONEHOTENCODEDFEATURESLIST:
oneHotEncodedFeaturesList,
PredictiveConstants.INDEXEDLABELNAMEDICT: indexedLabelNameDict,
PredictiveConstants.ISLABELINDEXED: isLabelIndexed,
PredictiveConstants.VECTORFEATURES: vectorizedFeaturescolmName,
PredictiveConstants.IDNAMEFEATURESORDERED: idNameFeaturesOrdered,
PredictiveConstants.INDEXERPATHMAPPING: indexerPathMapping,
PredictiveConstants.ONEHOTENCODERPATHMAPPING:
oneHotEncoderPathMapping
}
return result
# Exercise_1
# Import the one hot encoder class
from pyspark.ml.feature import OneHotEncoderEstimator
# Create an instance of the one hot encoder
onehot = OneHotEncoderEstimator(inputCols=['org_idx'], outputCols=['org_dummy'])
# Apply the one hot encoder to the flights data
onehot = onehot.fit(flights)
flights_onehot = onehot.transform(flights)
# Check the results
flights_onehot.select('org', 'org_idx', 'org_dummy').distinct().sort('org_idx').show()
--------------------------------------------------
# Exercise_2
from pyspark.ml.regression import LinearRegression
from pyspark.ml.evaluation import RegressionEvaluator
# Create a regression object and train on training data
regression = LinearRegression(labelCol='duration').fit(flights_train)
# Create predictions for the testing data and take a look at the predictions
predictions = regression.transform(flights_test)
predictions.select('duration', 'prediction').show(5, False)
# Calculate the RMSE
RegressionEvaluator(labelCol='duration').evaluate(predictions)
--------------------------------------------------
# Exercise_3
def train_model(args):
# do not run this test for pytorch lightning below min supported verson
import pytorch_lightning as pl
if LooseVersion(pl.__version__) < LooseVersion(MIN_PL_VERSION):
print("Skip test for pytorch_ligthning=={}, min support version is {}".
format(pl.__version__, MIN_PL_VERSION))
return
# Initialize SparkSession
conf = SparkConf().setAppName('keras_spark_mnist').set(
'spark.sql.shuffle.partitions', '16')
if args.master:
conf.setMaster(args.master)
elif args.num_proc:
conf.setMaster('local[{}]'.format(args.num_proc))
spark = SparkSession.builder.config(conf=conf).getOrCreate()
# Setup our store for intermediate data
store = Store.create(args.work_dir)
# Download MNIST dataset
data_url = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/mnist.bz2'
libsvm_path = os.path.join(args.data_dir, 'mnist.bz2')
if not os.path.exists(libsvm_path):
subprocess.check_output(['wget', data_url, '-O', libsvm_path])
# Load dataset into a Spark DataFrame
df = spark.read.format('libsvm') \
.option('numFeatures', '784') \
.load(libsvm_path)
# One-hot encode labels into SparseVectors
encoder = OneHotEncoderEstimator(inputCols=['label'],
outputCols=['label_vec'],
dropLast=False)
model = encoder.fit(df)
train_df = model.transform(df)
# Train/test split
train_df, test_df = train_df.randomSplit([0.9, 0.1])
# Define the PyTorch model without any Horovod-specific parameters
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.conv2_drop = nn.Dropout2d()
self.fc1 = nn.Linear(320, 50)
self.fc2 = nn.Linear(50, 10)
def forward(self, x):
x = x.float()
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.log_softmax(x)
model = Net()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)
loss = nn.NLLLoss()
# Train a Horovod Spark Estimator on the DataFrame
torch_estimator = hvd.TorchEstimator(
num_proc=args.num_proc,
store=store,
model=model,
optimizer=optimizer,
loss=lambda input, target: loss(input, target.long()),
input_shapes=[[-1, 1, 28, 28]],
feature_cols=['features'],
label_cols=['label'],
batch_size=args.batch_size,
epochs=args.epochs,
verbose=1)
torch_model = torch_estimator.fit(train_df).setOutputCols(['label_prob'])
# Evaluate the model on the held-out test DataFrame
pred_df = torch_model.transform(test_df)
argmax = udf(lambda v: float(np.argmax(v)), returnType=T.DoubleType())
pred_df = pred_df.withColumn('label_pred', argmax(pred_df.label_prob))
evaluator = MulticlassClassificationEvaluator(predictionCol='label_pred',
labelCol='label',
metricName='accuracy')
print('Test accuracy:', evaluator.evaluate(pred_df))
spark.stop()
flights = flights.dropna()
print("\nThe data contains %d records after dropping records with na values." % flights.count())
# Create an indexer for carrier categorical feature
indexer = StringIndexer(inputCol="carrier", outputCol='carrier_idx')
# Indexer identifies categories in the data
indexer_model = indexer.fit(flights)
# Indexer creates a new column with numeric index values
flights_indexed = indexer_model.transform(flights)
# Repeat the process for the org categorical feature
flights_indexed = StringIndexer(inputCol="org", outputCol='org_idx').fit(flights_indexed).transform(flights_indexed)
# Check first five records
flights_indexed.show(5)
flites = flights_indexed.select('carrier', 'org', 'org_idx')
# Create an instance of the one hot encoder
onehot = OneHotEncoderEstimator(inputCols=["org_idx"], outputCols=["org_dummy"])
# Apply the one hot encoder to the flights data
onehot = onehot.fit(flites)
flights_onehot = onehot.transform(flites)
# Check the results
flights_onehot.select('org', 'org_idx', 'org_dummy').distinct().sort('org_idx').show()
spark.stop()
# COMMAND ----------
# MAGIC %md-sandbox
# MAGIC Now each room has a unique numerical value assigned. While we could pass the new `room_type_index` into a machine learning model, it would assume that `Shared room` is twice as much as `Entire home/apt`, which is not the case. Instead, we need to change these values to a binary yes/no value if a listing is for a shared room, entire home, or private room.
# MAGIC
# MAGIC Do this by training and fitting the `OneHotEncoderEstimator`, which only operates on numerical values (this is why we needed to use `StringIndexer` first).
# MAGIC
# MAGIC <img alt="Side Note" title="Side Note" style="vertical-align: text-bottom; position: relative; height:1.75em; top:0.05em; transform:rotate(15deg)" src="https://files.training.databricks.com/static/images/icon-note.webp"/> Certain models, such as random forest, do not need one-hot encoding (and can actually be negatively affected by the process). The models we'll explore in this course, however, do need this process.
# COMMAND ----------
from pyspark.ml.feature import OneHotEncoderEstimator
encoder = OneHotEncoderEstimator(inputCols=["room_type_index"],
outputCols=["encoded_room_type"])
encoderModel = encoder.fit(indexedDF)
encodedDF = encoderModel.transform(indexedDF)
display(encodedDF)
# COMMAND ----------
# MAGIC %md
# MAGIC The new column `encoded_room_type` is a vector. The difference between a sparse and dense vector is whether Spark records all of the empty values. In a sparse vector, like we see here, Spark saves space by only recording the places where the vector has a non-zero value. The value of 0 in the first position indicates that it's a sparse vector. The second value indicates the length of the vector.
# MAGIC
# MAGIC Here's how to read the mapping above:<br><br>
# MAGIC
# MAGIC * `Shared room` maps to the vector `[0, 0]`
# MAGIC * `Entire home/apt` maps to the vector `[0, 1]`
# MAGIC * `Private room` maps to the vector `[1, 0]`
indexer = StringIndexer(inputCol='type', outputCol='type_idx')
# Assign index values to strings
indexer = indexer.fit(cars)
# Create column with index values
cars = indexer.transform(cars)
pd.set_option('display.max_columns', None) # all cols
pd.set_option('display.width', 161)
#print(cars.toPandas().sample(12))
# Check column data types
print('\n', cars.dtypes, '\n')
kars = cars.select('name', 'type', 'type_idx')
print(kars.toPandas().sample(12))
onehot = OneHotEncoderEstimator(inputCols=['type_idx'],
outputCols=['type_dummy'])
onehot = onehot.fit(kars)
kars = onehot.transform(kars)
kars.select('type', 'type_idx',
'type_dummy').distinct().sort('type_idx').show()
print("DenseVector:", DenseVector([1, 0, 0, 0, 0, 7, 0, 0]))
print("SparseVector:", SparseVector(8, {0: 1.0, 5: 7.0}))
spark.stop()
# Download MNIST dataset
data_url = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/mnist.bz2'
libsvm_path = os.path.join(args.data_dir, 'mnist.bz2')
if not os.path.exists(libsvm_path):
subprocess.check_output(['wget', data_url, '-O', libsvm_path])
# Load dataset into a Spark DataFrame
df = spark.read.format('libsvm') \
.option('numFeatures', '784') \
.load(libsvm_path)
# One-hot encode labels into SparseVectors
encoder = OneHotEncoderEstimator(inputCols=['label'],
outputCols=['label_vec'],
dropLast=False)
model = encoder.fit(df)
train_df = model.transform(df)
# Train/test split
train_df, test_df = train_df.randomSplit([0.9, 0.1])
# Define the PyTorch model without any Horovod-specific parameters
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.conv2_drop = nn.Dropout2d()
self.fc1 = nn.Linear(320, 50)
self.fc2 = nn.Linear(50, 10)
"Cat7", "Cat8", "Cat9", "Cat10", "Cat11", "Cat12", "OrdCat", "NVCat"
]
cat_output = [column + "_index" for column in cat_input]
cat_hot = [column + "_hot" for column in cat_output]
indexers = [
StringIndexer(inputCol=column, outputCol=column + "_index").fit(clean)
for column in cat_input
]
pipeline = Pipeline(stages=indexers)
df_r = pipeline.fit(clean).transform(clean)
encoder = OneHotEncoderEstimator(inputCols=cat_output, outputCols=cat_hot)
hot_data = encoder.fit(df_r).transform(df_r)
df = hot_data.select('Blind_Make_index_hot', 'Model_Year_index_hot', 'Var1',
'Var2', 'Var3', 'Var4', 'Var5', 'Var6', 'Var7', 'Var8',
'NVVar1', 'NVVar2', 'NVVar3', 'NVVar4', 'Cat11_index_hot',
'Cat9_index_hot', 'Cat7_index_hot', 'NVCat_index_hot',
'Cat4_index_hot', 'Cat12_index_hot', 'Cat1_index_hot',
'Cat5_index_hot', 'Cat3_index_hot', 'Cat8_index_hot',
'Cat2_index_hot', 'Cat6_index_hot', 'Cat10_index_hot',
'OrdCat_index_hot', 'label', 'Calendar_Year')
assembler = VectorAssembler(inputCols=[
'Blind_Make_index_hot', 'Model_Year_index_hot', 'Var1', 'Var2', 'Var3',
'Var4', 'Var5', 'Var6', 'Var7', 'Var8', 'NVVar1', 'NVVar2', 'NVVar3',
'NVVar4', 'Cat11_index_hot', 'Cat9_index_hot', 'Cat7_index_hot',
'NVCat_index_hot', 'Cat4_index_hot', 'Cat12_index_hot', 'Cat1_index_hot',
'Cat5_index_hot', 'Cat3_index_hot', 'Cat8_index_hot', 'Cat2_index_hot',
mc = BinaryClassificationEvaluator(rawPredictionCol="prediction",
labelCol="label")
cv = CrossValidator(estimator=estimator,
estimatorParamMaps=paramGrid,
evaluator=mc,
numFolds=2)
# for row in train_df.rdd.collect():
# print("row: ", row.uri)
# load_image_from_uri(row.uri)
# cvModel = cv.fit(train_df)
# mc.evaluate(cvModel.transform(test_df))
#
stringIndexer = StringIndexer(inputCol="label_name",
outputCol="categoryIndex")
indexed_dateset = stringIndexer.fit(train_df).transform(train_df)
# encoder = OneHotEncoder(inputCol="categoryIndex", outputCol="categoryVec")
encoder = OneHotEncoderEstimator(inputCols=["categoryIndex"],
outputCols=["categoryVec"])
encoder_model = encoder.fit(indexed_dateset)
image_dataset = encoder_model.transform(indexed_dateset)
image_dataset.show()
transformers = estimator.fit(image_dataset)
stringIndNames = []
encodedNames = []
for v in categFilt:
encodedNames.append(v+"_indE")
stringIndNames.append(v+"_ind")
print(categFilt)
print(stringIndNames)
print("\n", encodedNames)
encoder = OneHotEncoderEstimator(inputCols= stringIndNames, outputCols=encodedNames)
insuranceRaw = encoder.fit(insuranceRaw).transform(insuranceRaw)
insuranceRaw.select('Cat1_indE', 'Cat3_indE').show(30)
#vector assemble the categorical features
from pyspark.ml.feature import VectorAssembler
assembler = VectorAssembler(inputCols=encodedNames, outputCol="catfeatures")
insuranceRaw = assembler.transform(insuranceRaw)
for x in stringIndNames:
insuranceRaw = insuranceRaw.drop(x)
indexer = StringIndexer(inputCol="carrier", outputCol='carrier_idx')
# Indexer identifies categories in the data
indexer_model = indexer.fit(flights)
# Indexer creates a new column with numeric index values
flights_indexed = indexer_model.transform(flights)
# Repeat the process for the org categorical feature
flites = StringIndexer(inputCol="org", outputCol='org_idx').fit(flights_indexed).transform(flights_indexed)
# Create an instance of the one hot encoder
onehot = OneHotEncoderEstimator(inputCols=["org_idx"], outputCols=["org_dummy"])
# Apply the one hot encoder to the flights data
onehot = onehot.fit(flites)
flites = onehot.transform(flites)
pd.set_option('display.max_columns', None) # all cols
pd.set_option('display.width', 199)
pd.set_option('display.max_colwidth', 199)
# Create buckets at 3 hour intervals through the day
buckets = Bucketizer(splits=[0, 3, 6, 9, 12, 15, 18, 21, 24],
inputCol="depart", outputCol="depart_bucket")
# Bucket the departure times
bucketed = buckets.transform(flites)
#bucketed.select("depart", "depart_bucket").show(5)
# Create a one-hot encoder for departure
# COMMAND ----------
# MAGIC %md ### Features added
# MAGIC - day - day of month of click
# MAGIC - hour - hour of day of click
# COMMAND ----------
#Encode categorical features (app, device, os, channel)
from pyspark.ml.feature import OneHotEncoderEstimator
encoder = OneHotEncoderEstimator(
inputCols=["ip", "app", "device", "os", "channel"],
outputCols=["ipVec", "appVec", "deviceVec", "osVec", "channelVec"])
model = encoder.fit(t0_df)
t0_df = model.transform(t0_df)
t0_df.printSchema()
# COMMAND ----------
from pyspark.ml.feature import VectorAssembler
#Select features to actually use in training
inputCols = [
"ipVec", "appVec", "deviceVec", "osVec", "channelVec", "day", "hour"
]
vectorAssembler = VectorAssembler(inputCols=inputCols, outputCol="features")
v_t0_df = vectorAssembler.transform(t0_df)
v_t0_df.printSchema()
# COMMAND ----------
indexer = StringIndexer(inputCol="ItemFamily", outputCol="ItemFamilyNum").fit(store_num_data)
store_num_data_indexed = indexer.transform(store_num_data)
#store_num_data_indexed.show()
indexer = StringIndexer(inputCol="type", outputCol="typeNum").fit(store_num_data_indexed)
store_num_data_indexed = indexer.transform(store_num_data_indexed)
#store_num_data_indexed.show()
# COMMAND ----------
encoder = OneHotEncoderEstimator(inputCols=["typeNum","Cluster","StoreNum","ItemFamilyNum","Month","Day","dow_number"],
outputCols=["typeNumVec","ClusterVec","StoreNumVec","ItemFamilyNumVec", "MonthVec","DayVec","DOWNum"])
model = encoder.fit(store_num_data_indexed)
store_num_data_ind_enc = model.transform(store_num_data_indexed)
#store_num_data_ind_enc.show(5)
# COMMAND ----------
store_num_data_ind_enc = store_num_data_ind_enc.withColumnRenamed("store_num_data_ind_enc.Date", "Date_Date")
# COMMAND ----------
# MAGIC %md ***Using Linear Regression to create model***
# COMMAND ----------
trainingdf = store_num_data_ind_enc.filter(store_num_data_ind_enc.Date_Date <'2014-04-28')