def ExecuteSweep(sweepSpec, optSpec, plotString, plot=False):
evaluationRunSpecifications = []
for sweepValue in sweepSpec[optSpec]:
runSpecification = {}
runSpecification['optimizing'] = optSpec
runSpecification['maxDepth'] = sweepSpec['maxDepth']
runSpecification[optSpec] = sweepValue
evaluationRunSpecifications.append(runSpecification)
## if you want to run in parallel you need to install joblib as described in the lecture notes and adjust the comments on the next three lines...
from joblib import Parallel, delayed
evaluations = Parallel(n_jobs=4)(
delayed(ExecuteEvaluationRun)(runSpec, xTrain, yTrain, folds)
for runSpec in evaluationRunSpecifications)
if plot:
xValues = sweepSpec[optSpec]
series1 = [
runSpec['crossValidationMean'] for runSpec in evaluations
]
errorBarsForSeries1 = [
runSpec['crossValidationErrorBound'] for runSpec in evaluations
]
if folds > 1:
chartName = "Cross Validation Accuracy"
fileName = "CrossValidationAccuracy"
else:
chartName = "Validation Accuracy"
fileName = "ValidationAccuracy"
if useNumeric:
Charting.PlotSeriesWithErrorBars(
[series1], [errorBarsForSeries1], [chartName],
xValues,
chartTitle=plotString + " " + chartName +
" (w/ Numeric Features)",
xAxisTitle=plotString + " Values",
yAxisTitle=chartName,
yBotLimit=0.7,
outputDirectory=kOutputDirectory,
fileName="2-" + plotString + fileName + "Numeric")
else:
Charting.PlotSeriesWithErrorBars(
[series1], [errorBarsForSeries1], [chartName],
xValues,
chartTitle=plotString + " " + chartName,
xAxisTitle=plotString + " Values",
yAxisTitle=chartName,
yBotLimit=0.7,
outputDirectory=kOutputDirectory,
fileName="2-" + plotString + fileName)
return evaluations
def ExecuteSweep(sweepSpec, optSpec, plotString, plot=False):
evaluationRunSpecifications = []
for sweepValue in sweepSpec[optSpec]:
runSpecification = {}
runSpecification['optimizing'] = optSpec
runSpecification['stepSize'] = sweepSpec['stepSize']
runSpecification['convergence'] = sweepSpec['convergence']
runSpecification[optSpec] = sweepValue
evaluationRunSpecifications.append(runSpecification)
## if you want to run in parallel you need to install joblib as described in the lecture notes and adjust the comments on the next three lines...
from joblib import Parallel, delayed
evaluations = Parallel(n_jobs=4)(
delayed(ExecuteEvaluationRun)(runSpec, xTrain, yTrain)
for runSpec in evaluationRunSpecifications)
if plot:
xValues = sweepSpec[optSpec]
series1 = [runSpec['crossValidationMean'] for runSpec in evaluations]
errorBarsForSeries1 = [
runSpec['crossValidationErrorBound'] for runSpec in evaluations
]
# series2 = [ runSpec['runtime'] for runSpec in evaluations ]
Charting.PlotSeriesWithErrorBars(
[series1], [errorBarsForSeries1], ["Cross Validation Accuracy"],
xValues,
chartTitle=plotString + " Cross Validation Accuracy",
xAxisTitle=plotString + " Values",
yAxisTitle="Cross Valid. Accuracy",
yBotLimit=0.7,
outputDirectory=kOutputDirectory,
fileName="1-" + plotString + "CrossValidationAccuracy")
#Charting.PlotSeries([series2], ['runtime'], xValues, chartTitle="Cross Validation Runtime", xAxisTitle=plotString + " Values", yAxisTitle="Run Time", outputDirectory=kOutputDirectory, fileName="1-"+plotString+"CrossValidationRunTime")
return evaluations
seriesFPRs = []
seriesFNRs = []
seriesLabels = []
model = DecisionTree.DecisionTree()
model.fit(xTrainNumeric, yTrain, maxDepth=BestDepthNumeric)
(modelFPRs, modelFNRs,
thresholds) = TabulateModelPerformanceForROC(model, xTestNumeric, yTest)
seriesFPRs.append(modelFPRs)
seriesFNRs.append(modelFNRs)
seriesLabels.append('best numeric model')
model = DecisionTree.DecisionTree()
model.fit(xTrain, yTrain, maxDepth=BestDepth)
(modelFPRs, modelFNRs,
thresholds) = TabulateModelPerformanceForROC(model, xTest, yTest)
seriesFPRs.append(modelFPRs)
seriesFNRs.append(modelFNRs)
seriesLabels.append('best non-numeric model')
Charting.PlotROCs(seriesFPRs,
seriesFNRs,
seriesLabels,
useLines=True,
chartTitle="ROC Comparison Features",
xAxisTitle="False Negative Rate",
yAxisTitle="False Positive Rate",
outputDirectory=kOutputDirectory,
fileName="2-DecistionTreeROC")
##
# Visualize Training run
##
kOutputDirectory = "C:\\temp\\visualize"
import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting
xValues = [i + 1 for i in range(len(trainLosses))]
Charting.PlotSeries([trainLosses, validationLosses],
["Train Loss", "Validate Loss"],
xValues,
useMarkers=False,
chartTitle="Pytorch First Modeling Run",
xAxisTitle="Epoch",
yAxisTitle="Loss",
yBotLimit=0.0,
outputDirectory=kOutputDirectory,
fileName="PyTorch-Initial-TrainValidate")
##
# Evaluate the Model
##
import MachineLearningCourse.MLUtilities.Evaluations.EvaluateBinaryClassification as EvaluateBinaryClassification
import MachineLearningCourse.MLUtilities.Evaluations.ErrorBounds as ErrorBounds
model.train(mode=False)
yTestPredicted = model(xTest)
converg_series = []
# log_convert = {0.1: 0, 0.01: 1, 0.001: 2, 0.0001: 3, 0.00001: 4}
for evaluation in evaluations:
if evaluation["optimizing"] == "convergence":
converg_error_series.append(evaluation['50PercentBound'])
converg_valid_series.append(evaluation['accuracy'])
converg_series.append(evaluation['convergence'])
elif evaluation["optimizing"] == "learning_rate":
learning_error_series.append(evaluation['50PercentBound'])
learning_valid_series.append(evaluation['accuracy'])
learning_series.append(evaluation['learning_rate'])
Charting.PlotSeriesWithErrorBars([converg_valid_series], [converg_error_series], ["Accuracy"], [converg_series], chartTitle="<NN Accuracy on Validation Data>", xAxisTitle="<converg>", yAxisTitle="<Accuracy>", yBotLimit=0.65, outputDirectory=kOutputDirectory, fileName="converg_sweep")
Charting.PlotSeriesWithErrorBars([learning_valid_series], [learning_error_series], ["Accuracy"], [learning_series], chartTitle="<NN Accuracy on Validation Data>", xAxisTitle="<learning>", yAxisTitle="<Accuracy>", yBotLimit=0.65, outputDirectory=kOutputDirectory, fileName="learning_sweep")
roccompare = False
if roccompare:
# A helper function for calculating FN rate and FP rate across a range of thresholds
def TabulateModelPerformanceForROC(model, xValidate, yValidate):
pointsToEvaluate = 100
thresholds = [ x / float(pointsToEvaluate) for x in range(pointsToEvaluate + 1)]
FPRs = []
FNRs = []
try:
for threshold in thresholds:
'convergence': newBestParameters['convergence'],
'numFrequentWords': newBestParameters['numFrequentWords'],
'numMutualInformationWords':
newBestParameters['numMutualInformationWords']
}
result = ExecuteEvaluationRun(bestParameters, xTrainRaw, yTrain, 1)
validationSetAccuracy.append(result['accuracy'])
validationSetAccuracyError.append(result['accuracyErrorBound'])
# Plot Validation Accuracy
lastSweep = sweep[len(sweep) - 1]
Charting.PlotSeriesWithErrorBars([validationSetAccuracy],
[validationSetAccuracyError], ["Accuracy"],
sweep + [lastSweep + 1],
chartTitle="Validation Set Accuracy",
xAxisTitle="Sweep #",
yAxisTitle="Validation Set Accuracy",
yBotLimit=0.8,
outputDirectory=kOutputDirectory,
fileName="7-ValidationSetAccuracy")
print("BestParameters: ", bestParameters)
# ROC of initial vs. best
seriesFPRs = []
seriesFNRs = []
seriesLabels = []
init = {}
init['stepSize'] = 1.0
init['convergence'] = 0.005
init['numFrequentWords'] = 0
init['numMutualInformationWords'] = 20
accuracies = []
errorBarsAccuracy = []
for kv in kValues:
model = BoostedTree.BoostedTree()
model.fit(xTrain, yTrain, maxDepth=maxDepth, k=kv)
accuracy = EvaluateBinaryClassification.Accuracy(yTest, model.predict(xTest))
lower, upper = ErrorBounds.GetAccuracyBounds(accuracy, len(yTest), .5)
print(kv, ": ", accuracy)
accuracies.append(accuracy)
errorBarsAccuracy.append(accuracy-lower)
if bestModel is None:
bestModel = (model, upper)
elif lower > bestModel[1]:
bestModel = (model, upper)
Charting.PlotSeriesWithErrorBars([accuracies], [errorBarsAccuracy], ["k-round tuning accuracy"], kValues, chartTitle="Line/Circle Concept Accuracy", xAxisTitle="Boosting Rounds", yAxisTitle="Test Accuracy", yBotLimit=0.5, outputDirectory=kOutputDirectory, fileName="4-BoostingTreeRoundTuning")
## you can use this to visualize what your model is learning.
accuracy = EvaluateBinaryClassification.Accuracy(yTest, bestModel[0].predict(xTest))
lower, upper = ErrorBounds.GetAccuracyBounds(accuracy, len(yTest), .95)
print("accuracy: ", lower, "-", upper)
visualize = Visualize2D.Visualize2D(kOutputDirectory, "4-My Boosted Tree")
visualize.PlotBinaryConcept(model)
# Or you can use it to visualize individual models that you learened, e.g.:
# visualize.PlotBinaryConcept(model->modelLearnedInRound[2])
## you might like to see the training or test data too, so you might prefer this to simply calling 'PlotBinaryConcept'
#visualize.Plot2DDataAndBinaryConcept(xTrain,yTrain,model)
# And remember to save
xTrain = featurizer.Featurize(xTrainRaw)
xValidate = featurizer.Featurize(xValidateRaw)
xTest = featurizer.Featurize(xTestRaw)
model = LogisticRegression.LogisticRegression()
model.fit(xTrain,yTrain,convergence=convergence, stepSize=stepSize)
(modelFPRs, modelFNRs, thresholds) = TabulateModelPerformanceForROC(model, xValidate, yValidate)
seriesFPRs.append(modelFPRs)
seriesFNRs.append(modelFNRs)
seriesLabels.append('25 Frequent')
#### Learn a model with 25 features by mutual information
featurizer = SMSSpamFeaturize.SMSSpamFeaturize(useHandCraftedFeatures=False)
featurizer.CreateVocabulary(xTrainRaw, yTrain, numMutualInformationWords = 25)
xTrain = featurizer.Featurize(xTrainRaw)
xValidate = featurizer.Featurize(xValidateRaw)
xTest = featurizer.Featurize(xTestRaw)
model = LogisticRegression.LogisticRegression()
model.fit(xTrain,yTrain,convergence=convergence, stepSize=stepSize)
(modelFPRs, modelFNRs, thresholds) = TabulateModelPerformanceForROC(model, xValidate, yValidate)
seriesFPRs.append(modelFPRs)
seriesFNRs.append(modelFNRs)
seriesLabels.append('25 Mutual Information')
Charting.PlotROCs(seriesFPRs, seriesFNRs, seriesLabels, useLines=True, chartTitle="ROC Comparison", xAxisTitle="False Negative Rate", yAxisTitle="False Positive Rate", outputDirectory=kOutputDirectory, fileName="Plot-SMSSpamROCs")
logisticRegressionModel.fit(xTrain,
yTrain,
stepSize=1.0,
convergence=0.001)
trainLosses.append(logisticRegressionModel.loss(xTrain, yTrain))
validationLosses.append(
logisticRegressionModel.loss(xValidate, yValidate))
import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting
# trainLosses, validationLosses, and lossXLabels are parallel arrays with the losses you want to plot at the specified x coordinates
Charting.PlotSeries(
[trainLosses, validationLosses], ['Train', 'Validate'],
lossXLabels,
chartTitle="Num Frequent Words Logistic Regression",
xAxisTitle="Num Frequent Words",
yAxisTitle="Avg. Loss",
outputDirectory=kOutputDirectory,
fileName=
"4-Logistic Regression Num Frequent Words Train vs Validate loss")
trainLosses = []
validationLosses = []
lossXLabels = [1, 10, 20, 30, 40, 50]
for freq in lossXLabels:
# Now get into model training
import MachineLearningCourse.MLUtilities.Learners.LogisticRegression as LogisticRegression
# Remember to create a new featurizer object/vocabulary for each part of the assignment
featurizer = SMSSpamFeaturize.SMSSpamFeaturize(
useHandCraftedFeatures=False)
for i in discountRateRange:
results = trainModel(discountRate=i,
actionProbabilityBase=1.25,
randomActionRate=0.1,
learningRateScale=0.01,
binsPerDimension=8)
acc = np.mean(results)
err = np.std(results)
accResults.append(acc)
accError.append(err)
Charting.PlotSeriesWithErrorBars(
[accResults], [accError], ["CartPole Score (Mean) 20 trials"],
discountRateRange,
useMarkers=False,
chartTitle="CartPole Discount Rate Effect on Learning",
xAxisTitle="Discount Rate",
yAxisTitle="Score",
yBotLimit=20,
outputDirectory=kOutputDirectory,
fileName="1-DiscountRate")
runActionProbability = False
if runActionProbability:
accResults = []
accError = []
for i in actionProbabilityBaseRange:
results = trainModel(discountRate=0.75,
actionProbabilityBase=i,
randomActionRate=0.1,
learningRateScale=0.01,
binsPerDimension=8)
trainingAccuracy = EvaluateBinaryClassification.Accuracy(yTrain, model.predict(xTrain))
lowerTrain, upperTrain = ErrorBounds.GetAccuracyBounds(trainingAccuracy, len(yTrain), .5)
validationAccuracies.append(validationAccuracy)
validationAccuracyErrorBounds.append(validationAccuracy-lower)
trainingAccuracies.append(trainingAccuracy)
trainingAccuracyErrorBounds.append(trainingAccuracy-lowerTrain)
print("k: ", kv, " accuracy: ", lower, "-", upper)
if bestModelBT is None:
bestModelBT = (model, lower, upper, kv)
elif lower > bestModelBT[2]:
bestModelBT = (model, lower, upper, kv)
print("boosted tree - k-rounds: ", bestModelBT[3], " accuracy: ", bestModelBT[1], "-", bestModelBT[2])
Charting.PlotSeriesWithErrorBars([validationAccuracies, trainingAccuracies], [validationAccuracyErrorBounds, trainingAccuracyErrorBounds], ["BT-validation", "BT-training"], kValues, chartTitle="Boosted Decision Tree k-Round Search", xAxisTitle="Boosting Rounds", yAxisTitle="Accuracy", yBotLimit=0.5, outputDirectory=kOutputDirectory, fileName=filename)
tuneDepth = False
if tuneDepth:
featurizer = BlinkFeaturize.BlinkFeaturize()
#featurizer.CreateFeatureSet(xTrainRaw, yTrain, includeEdgeFeatures=True)
#filename = "1-BlinkBoostedTreeEdgeFilterFeaturesOnly-MaxDepth"
featurizer.CreateFeatureSet(xTrainRaw, yTrain, includeEdgeFeatures=True, includeEdgeFeaturesMax=True)
filename = "1-BlinkBoostedTreeEdgeFilterMaxAndAvg-MaxDepth"
xTrain = featurizer.Featurize(xTrainRaw)
xValidate = featurizer.Featurize(xValidateRaw)
xTest = featurizer.Featurize(xTestRaw)
validationAccuracyErrorBounds.append(validationAccuracy - lower)
trainingAccuracies.append(trainingAccuracy)
trainingAccuracyErrorBounds.append(trainingAccuracy - lowerTrain)
print("stepSize: ", stepSize, " accuracy: ", lower, "-", upper)
if bestModelLG is None:
bestModelLG = (model, lower, upper, stepSize)
elif lower > bestModelLG[2]:
bestModelLG = (model, lower, upper, stepSize)
Charting.PlotSeriesWithErrorBars(
[validationAccuracies, trainingAccuracies],
[validationAccuracyErrorBounds, trainingAccuracyErrorBounds],
["LR-validation", "LR-training"],
stepSizes,
chartTitle="Logistic Regression Step Size Search",
xAxisTitle="Step Size",
yAxisTitle="Accuracy",
yBotLimit=0.5,
outputDirectory=kOutputDirectory,
fileName="5-LogisticRegressionStepSize")
bestModelDT = None
runDecisionTree = True
if runDecisionTree:
import MachineLearningCourse.MLUtilities.Learners.DecisionTree as DecisionTree
maxDepths = [1, 10, 50, 100, 500]
validationAccuracies = []
validationAccuracyErrorBounds = []
trainingAccuracies = []
trainingAccuracyErrorBounds = []
def trainModel(m, op, maxE, patience=10, saveChartName=""):
startTime = time.time()
lossFunction = torch.nn.BCELoss(reduction='mean')
trainLosses = []
validationLosses = []
converged = False
epoch = 1
lastValidationLoss = None
currPatience = 0
while not converged and epoch < maxE:
# Reset the gradients in the network to zero
op.zero_grad()
#for batchXTensor, batchYTensor in trainDataSetGenerator:
# x = batchXTensor.to(device)
# y = batchYTensor.to(device)
#
# Do the forward pass
# yPredicted = m(x)
# Compute the total loss summed across training samples in the epoch
# note this is different from our implementation, which took one step
# of gradient descent per sample.
# trainLoss = lossFunction(yPredicted, y)
# Backprop the errors from the loss on this iteration
# trainLoss.backward()
yTrainPredicted = m(xTrain)
trainLoss = lossFunction(yTrainPredicted, yTrain)
trainLoss.backward()
# Do a weight update step
op.step()
# now check the validation loss
m.train(mode=False)
#validationLossTotal = 0
#for batchXTensor, batchYTensor in validationDataSetGenerator:
# x = batchXTensor.to(device)
# y = batchYTensor.to(device)
# yPredicted = m(x)
# validationLoss = lossFunction(yPredicted, y)
# validationLossTotal += validationLoss.item()
#validationLoss = validationLossTotal / len(validationDataSet)
#validationLosses.append(validationLoss)
yValidationPredicted = m(xValidate)
validationLoss = lossFunction(yValidationPredicted, yValidate)
#trainingLossTotal = 0
#for batchXTensor, batchYTensor in trainDataSetGenerator:
# x = batchXTensor.to(device)
# y = batchYTensor.to(device)
# yPredicted = m(x)
# trainLoss = lossFunction(yPredicted, y)
# trainingLossTotal += trainLoss.item()
#trainLosses.append(trainingLossTotal / len(trainingDataSet))
#print("epoch %d: training loss {}, validation loss {}".format(epoch, trainLosses[-1], validationLoss))
#if lastValidationLoss is not None and validationLoss > lastValidationLoss and saveChartName == "":
# converged = True
#else:
# lastValidationLoss = validationLoss
yTrainingPredicted = m(xTrain)
trainLoss = lossFunction(yTrainingPredicted, yTrain)
trainLosses.append(trainLoss.item() / len(yTrain))
validationLosses.append(validationLoss.item() / len(yValidate))
print("epoch {}: training loss {}, validation loss {}".format(
epoch, trainLosses[-1], validationLosses[-1]))
if lastValidationLoss is not None and validationLoss > lastValidationLoss:
if currPatience < patience:
currPatience += 1
else:
converged = True
else:
lastValidationLoss = validationLoss
currPatience = 0
epoch = epoch + 1
m.train(mode=True)
endTime = time.time()
print("Runtime: %s" % (endTime - startTime))
##
# Visualize Training run
##
if saveChartName != "":
xValues = [i + 1 for i in range(len(trainLosses))]
Charting.PlotSeries([trainLosses, validationLosses],
["Train Loss", "Validate Loss"],
xValues,
useMarkers=False,
chartTitle="Blink LeNet Model Loss/Epoch",
xAxisTitle="Epoch",
yAxisTitle="Loss",
yBotLimit=0.0,
outputDirectory=kOutputDirectory,
fileName="4-" + saveChartName)
##
# Get the model accuracy on validation set
##
model.train(mode=False)
#yValidatePredicted = []
#for batchXTensor, batchYTensor in validationDataSetGenerator:
# x = batchXTensor.to(device)
# y = batchYTensor.to(device)
# yPredicted = m(x)
# yValidatePredicted += yPredicted.tolist()
yValidatePredicted = m(xValidate)
return EvaluateBinaryClassification.Accuracy(
yValidate, [1 if pred > 0.5 else 0 for pred in yValidatePredicted])
yTrain,
maxSteps=0,
stepSize=1.0,
convergence=0.0001)
import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting
lossXLabels = [0]
trainLosses = [logisticRegressionModel.loss(xTrain, yTrain)]
validationLosses = [logisticRegressionModel.loss(xValidate, yValidate)]
while not logisticRegressionModel.converged:
# do 100 iterations of training
logisticRegressionModel.incrementalFit(xTrain,
yTrain,
maxSteps=100,
stepSize=1.0,
convergence=0.0001)
lossXLabels.append(logisticRegressionModel.totalGradientDescentSteps)
trainLosses.append(logisticRegressionModel.loss(xTrain, yTrain))
validationLosses.append(
logisticRegressionModel.loss(xValidate, yValidate))
# trainLosses, validationLosses, and lossXLabels are parallel arrays with the losses you want to plot at the specified x coordinates
Charting.PlotSeries(
[trainLosses, validationLosses], ['Train', 'Validate'],
lossXLabels,
chartTitle="Logistic Regression",
xAxisTitle="Gradient Descent Steps",
yAxisTitle="Avg. Loss",
outputDirectory=kOutputDirectory,
fileName="3-Logistic Regression Train vs Validate loss")
convergence=convergence,
momentum=momentum)
if (i + 1) % 100 == 0:
for filterNumber in range(hiddenStructure[0]):
## update the first parameter based on your representation
#VisualizeWeights([model.weight0[0][filterNumber]] + list(model.layers[0][filterNumber][:]), "%s/filters/epoch%d_neuron%d.jpg" % (kOutputDirectory, i+1, filterNumber), sampleStride=sampleStride)
VisualizeWeights([model.weight0[0][filterNumber]] +
list(model.layers[0][filterNumber][:]),
"%s/filters2/epoch%d_neuron%d.jpg" %
(kOutputDirectory, i + 1, filterNumber),
sampleStride=sampleStride)
tLoss = model.loss(xTrain, yTrain)
vLoss = model.loss(xValidate, yValidate)
trainingLosses.append(tLoss)
validationLosses.append(vLoss)
import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting
#Charting.PlotSeries([trainingLosses, validationLosses], ["training loss", "validation loss"], list(range(maxEpochs)), chartTitle="Single Layer Loss", xAxisTitle="epochs", yAxisTitle="loss", outputDirectory=kOutputDirectory+"/visualize\\", fileName="2-SingleLayerModelLoss")
Charting.PlotSeries([trainingLosses, validationLosses],
["training loss", "validation loss"],
list(range(maxEpochs)),
chartTitle="Two Layer Loss",
xAxisTitle="epochs",
yAxisTitle="loss",
outputDirectory=kOutputDirectory + "/visualize\\",
fileName="2-TwoLayerModelLoss")
# Evaluate things...
accuracy = EvaluateBinaryClassification.Accuracy(yValidate,
model.predict(xValidate))
print("Model Accuracy is:", accuracy)