def ExecuteSweep(sweepSpec, optSpec, plotString, plot=False):
    evaluationRunSpecifications = []
    for sweepValue in sweepSpec[optSpec]:
        runSpecification = {}
        runSpecification['optimizing'] = optSpec
        runSpecification['numMutualInformationWords'] = sweepSpec[
            'numMutualInformationWords']
        runSpecification['stepSize'] = sweepSpec['stepSize']
        runSpecification['convergence'] = sweepSpec['convergence']
        runSpecification['numFrequentWords'] = sweepSpec['numFrequentWords']
        runSpecification[optSpec] = sweepValue
        evaluationRunSpecifications.append(runSpecification)

    ## if you want to run in parallel you need to install joblib as described in the lecture notes and adjust the comments on the next three lines...
    from joblib import Parallel, delayed
    evaluations = Parallel(n_jobs=4)(
        delayed(ExecuteEvaluationRun)(runSpec, xTrainRaw, yTrain)
        for runSpec in evaluationRunSpecifications)

    #evaluations = [ ExecuteEvaluationRun(runSpec, xTrainRaw, yTrain) for runSpec in evaluationRunSpecifications ]
    if plot:
        xValues = sweepSpec[optSpec]
        series1 = [runSpec['crossValidationMean'] for runSpec in evaluations]
        errorBarsForSeries1 = [
            runSpec['crossValidationErrorBound'] for runSpec in evaluations
        ]
        series2 = [runSpec['runtime'] for runSpec in evaluations]
        Charting.PlotSeriesWithErrorBars(
            [series1], [errorBarsForSeries1], ["Cross Validation Accuracy"],
            xValues,
            chartTitle=plotString + " Cross Validation Accuracy",
            xAxisTitle=plotString + " Values",
            yAxisTitle="Cross Valid. Accuracy",
            yBotLimit=0.8,
            outputDirectory=kOutputDirectory,
            fileName="7-" + plotString + "CrossValidationAccuracy")
        Charting.PlotSeries([series2], ['runtime'],
                            xValues,
                            chartTitle="Cross Validation Runtime",
                            xAxisTitle=plotString + " Values",
                            yAxisTitle="Run Time",
                            outputDirectory=kOutputDirectory,
                            fileName="7-" + plotString +
                            "CrossValidationRunTime")
    return evaluations
##
# Visualize Training run
##

kOutputDirectory = "C:\\temp\\visualize"

import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting

xValues = [i + 1 for i in range(len(trainLosses))]

Charting.PlotSeries([trainLosses, validationLosses],
                    ["Train Loss", "Validate Loss"],
                    xValues,
                    useMarkers=False,
                    chartTitle="Pytorch First Modeling Run",
                    xAxisTitle="Epoch",
                    yAxisTitle="Loss",
                    yBotLimit=0.0,
                    outputDirectory=kOutputDirectory,
                    fileName="PyTorch-Initial-TrainValidate")

##
# Evaluate the Model
##

import MachineLearningCourse.MLUtilities.Evaluations.EvaluateBinaryClassification as EvaluateBinaryClassification
import MachineLearningCourse.MLUtilities.Evaluations.ErrorBounds as ErrorBounds

model.train(mode=False)
yTestPredicted = model(xTest)
        logisticRegressionModel.fit(xTrain,
                                    yTrain,
                                    stepSize=1.0,
                                    convergence=0.001)
        trainLosses.append(logisticRegressionModel.loss(xTrain, yTrain))
        validationLosses.append(
            logisticRegressionModel.loss(xValidate, yValidate))

    import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting
    # trainLosses, validationLosses, and lossXLabels are parallel arrays with the losses you want to plot at the specified x coordinates
    Charting.PlotSeries(
        [trainLosses, validationLosses], ['Train', 'Validate'],
        lossXLabels,
        chartTitle="Num Frequent Words Logistic Regression",
        xAxisTitle="Num Frequent Words",
        yAxisTitle="Avg. Loss",
        outputDirectory=kOutputDirectory,
        fileName=
        "4-Logistic Regression Num Frequent Words Train vs Validate loss")

    trainLosses = []
    validationLosses = []
    lossXLabels = [1, 10, 20, 30, 40, 50]
    for freq in lossXLabels:
        # Now get into model training
        import MachineLearningCourse.MLUtilities.Learners.LogisticRegression as LogisticRegression

        # Remember to create a new featurizer object/vocabulary for each part of the assignment
        featurizer = SMSSpamFeaturize.SMSSpamFeaturize(
            useHandCraftedFeatures=False)
Exemplo n.º 4
0
def trainModel(m, op, maxE, patience=10, saveChartName=""):
    startTime = time.time()

    lossFunction = torch.nn.BCELoss(reduction='mean')

    trainLosses = []
    validationLosses = []

    converged = False
    epoch = 1
    lastValidationLoss = None

    currPatience = 0
    while not converged and epoch < maxE:
        # Reset the gradients in the network to zero
        op.zero_grad()
        #for batchXTensor, batchYTensor in trainDataSetGenerator:
        #    x = batchXTensor.to(device)
        #    y = batchYTensor.to(device)
        #
        # Do the forward pass
        #    yPredicted = m(x)

        # Compute the total loss summed across training samples in the epoch
        #  note this is different from our implementation, which took one step
        #  of gradient descent per sample.
        #    trainLoss = lossFunction(yPredicted, y)

        # Backprop the errors from the loss on this iteration
        #    trainLoss.backward()
        yTrainPredicted = m(xTrain)
        trainLoss = lossFunction(yTrainPredicted, yTrain)
        trainLoss.backward()
        # Do a weight update step
        op.step()

        # now check the validation loss
        m.train(mode=False)
        #validationLossTotal = 0
        #for batchXTensor, batchYTensor in validationDataSetGenerator:
        #    x = batchXTensor.to(device)
        #    y = batchYTensor.to(device)
        #    yPredicted = m(x)
        #    validationLoss = lossFunction(yPredicted, y)
        #    validationLossTotal += validationLoss.item()
        #validationLoss = validationLossTotal / len(validationDataSet)
        #validationLosses.append(validationLoss)
        yValidationPredicted = m(xValidate)
        validationLoss = lossFunction(yValidationPredicted, yValidate)

        #trainingLossTotal = 0
        #for batchXTensor, batchYTensor in trainDataSetGenerator:
        #    x = batchXTensor.to(device)
        #    y = batchYTensor.to(device)
        #    yPredicted = m(x)
        #    trainLoss = lossFunction(yPredicted, y)
        #    trainingLossTotal += trainLoss.item()
        #trainLosses.append(trainingLossTotal / len(trainingDataSet))

        #print("epoch %d: training loss {}, validation loss {}".format(epoch, trainLosses[-1], validationLoss))
        #if lastValidationLoss is not None and validationLoss > lastValidationLoss and saveChartName == "":
        #    converged = True
        #else:
        #    lastValidationLoss = validationLoss
        yTrainingPredicted = m(xTrain)
        trainLoss = lossFunction(yTrainingPredicted, yTrain)
        trainLosses.append(trainLoss.item() / len(yTrain))
        validationLosses.append(validationLoss.item() / len(yValidate))
        print("epoch {}: training loss {}, validation loss {}".format(
            epoch, trainLosses[-1], validationLosses[-1]))
        if lastValidationLoss is not None and validationLoss > lastValidationLoss:
            if currPatience < patience:
                currPatience += 1
            else:
                converged = True
        else:
            lastValidationLoss = validationLoss
            currPatience = 0
        epoch = epoch + 1
        m.train(mode=True)

    endTime = time.time()
    print("Runtime: %s" % (endTime - startTime))

    ##
    # Visualize Training run
    ##
    if saveChartName != "":
        xValues = [i + 1 for i in range(len(trainLosses))]
        Charting.PlotSeries([trainLosses, validationLosses],
                            ["Train Loss", "Validate Loss"],
                            xValues,
                            useMarkers=False,
                            chartTitle="Blink LeNet Model Loss/Epoch",
                            xAxisTitle="Epoch",
                            yAxisTitle="Loss",
                            yBotLimit=0.0,
                            outputDirectory=kOutputDirectory,
                            fileName="4-" + saveChartName)

    ##
    # Get the model accuracy on validation set
    ##
    model.train(mode=False)
    #yValidatePredicted = []
    #for batchXTensor, batchYTensor in validationDataSetGenerator:
    #        x = batchXTensor.to(device)
    #        y = batchYTensor.to(device)
    #        yPredicted = m(x)
    #        yValidatePredicted += yPredicted.tolist()
    yValidatePredicted = m(xValidate)
    return EvaluateBinaryClassification.Accuracy(
        yValidate, [1 if pred > 0.5 else 0 for pred in yValidatePredicted])
                                           yTrain,
                                           maxSteps=0,
                                           stepSize=1.0,
                                           convergence=0.0001)
    import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting
    lossXLabels = [0]
    trainLosses = [logisticRegressionModel.loss(xTrain, yTrain)]
    validationLosses = [logisticRegressionModel.loss(xValidate, yValidate)]

    while not logisticRegressionModel.converged:
        # do 100 iterations of training
        logisticRegressionModel.incrementalFit(xTrain,
                                               yTrain,
                                               maxSteps=100,
                                               stepSize=1.0,
                                               convergence=0.0001)

        lossXLabels.append(logisticRegressionModel.totalGradientDescentSteps)
        trainLosses.append(logisticRegressionModel.loss(xTrain, yTrain))
        validationLosses.append(
            logisticRegressionModel.loss(xValidate, yValidate))

    # trainLosses, validationLosses, and lossXLabels are parallel arrays with the losses you want to plot at the specified x coordinates
    Charting.PlotSeries(
        [trainLosses, validationLosses], ['Train', 'Validate'],
        lossXLabels,
        chartTitle="Logistic Regression",
        xAxisTitle="Gradient Descent Steps",
        yAxisTitle="Avg. Loss",
        outputDirectory=kOutputDirectory,
        fileName="3-Logistic Regression Train vs Validate loss")
                             convergence=convergence,
                             momentum=momentum)
        if (i + 1) % 100 == 0:
            for filterNumber in range(hiddenStructure[0]):
                ## update the first parameter based on your representation
                #VisualizeWeights([model.weight0[0][filterNumber]] + list(model.layers[0][filterNumber][:]), "%s/filters/epoch%d_neuron%d.jpg" % (kOutputDirectory, i+1, filterNumber), sampleStride=sampleStride)
                VisualizeWeights([model.weight0[0][filterNumber]] +
                                 list(model.layers[0][filterNumber][:]),
                                 "%s/filters2/epoch%d_neuron%d.jpg" %
                                 (kOutputDirectory, i + 1, filterNumber),
                                 sampleStride=sampleStride)
    tLoss = model.loss(xTrain, yTrain)
    vLoss = model.loss(xValidate, yValidate)
    trainingLosses.append(tLoss)
    validationLosses.append(vLoss)

import MachineLearningCourse.MLUtilities.Visualizations.Charting as Charting
#Charting.PlotSeries([trainingLosses, validationLosses], ["training loss", "validation loss"], list(range(maxEpochs)), chartTitle="Single Layer Loss", xAxisTitle="epochs", yAxisTitle="loss", outputDirectory=kOutputDirectory+"/visualize\\", fileName="2-SingleLayerModelLoss")
Charting.PlotSeries([trainingLosses, validationLosses],
                    ["training loss", "validation loss"],
                    list(range(maxEpochs)),
                    chartTitle="Two Layer Loss",
                    xAxisTitle="epochs",
                    yAxisTitle="loss",
                    outputDirectory=kOutputDirectory + "/visualize\\",
                    fileName="2-TwoLayerModelLoss")

# Evaluate things...
accuracy = EvaluateBinaryClassification.Accuracy(yValidate,
                                                 model.predict(xValidate))
print("Model Accuracy is:", accuracy)