def trainModel():
global trainingResult
masterAlgorithm = training.Distributed_Step2MasterFloat64NormEqDense()
for filenameIndex in range(rankId, len(trainDatasetFileNames), comm_size):
trainDataSource = FileDataSource(
trainDatasetFileNames[filenameIndex],
DataSourceIface.notAllocateNumericTable,
DataSourceIface.doDictionaryFromContext)
trainData = HomogenNumericTable(nFeatures, 0,
NumericTableIface.notAllocate)
trainDependentVariables = HomogenNumericTable(
nDependentVariables, 0, NumericTableIface.notAllocate)
mergedData = MergedNumericTable(trainData, trainDependentVariables)
trainDataSource.loadDataBlock(mergedData)
localAlgorithm = training.Distributed_Step1LocalFloat64NormEqDense()
localAlgorithm.input.set(training.data, trainData)
localAlgorithm.input.set(training.dependentVariables,
trainDependentVariables)
pres = localAlgorithm.compute()
masterAlgorithm.input.add(training.partialModels, pres)
mergedData.freeDataMemory()
trainData.freeDataMemory()
trainDependentVariables.freeDataMemory()
pres = masterAlgorithm.compute()
dataArch = InputDataArchive()
pres.serialize(dataArch)
nodeResults = dataArch.getArchiveAsArray()
serializedData = comm.gather(nodeResults)
if rankId == MPI_ROOT:
print("Number of processes is %d." % (len(serializedData)))
masterAlgorithm = training.Distributed_Step2MasterFloat64NormEqDense()
for i in range(comm_size):
dataArch = OutputDataArchive(serializedData[i])
dataForStep2FromStep1 = training.PartialResult()
dataForStep2FromStep1.deserialize(dataArch)
masterAlgorithm.input.add(training.partialModels,
dataForStep2FromStep1)
masterAlgorithm.compute()
trainingResult = masterAlgorithm.finalizeCompute()
def trainModel():
global trainingResult
nodeResults = []
# Create an algorithm object to build the final Naive Bayes model on the master node
masterAlgorithm = training.Distributed_Step2MasterFloat64DefaultDense(nClasses)
for filenameIndex in range(rankId, len(trainDatasetFileNames), comm_size):
# Initialize FileDataSource to retrieve the input data from a .csv file
#print("The worker with rank %d will read %s." % (rankId, trainDatasetFileNames[filenameIndex]))
trainDataSource = FileDataSource(trainDatasetFileNames[filenameIndex],
DataSourceIface.notAllocateNumericTable,
DataSourceIface.doDictionaryFromContext)
# Create Numeric Tables for training data and labels
trainData = HomogenNumericTable(nFeatures, 0, NumericTableIface.notAllocate)
trainDependentVariables = HomogenNumericTable(1, 0, NumericTableIface.notAllocate)
mergedData = MergedNumericTable(trainData, trainDependentVariables)
# Retrieve the data from the input file
trainDataSource.loadDataBlock(mergedData)
# Create an algorithm object to train the Naive Bayes model based on the local-node data
localAlgorithm = training.Distributed_Step1LocalFloat64DefaultDense(nClasses)
# Pass a training data set and dependent values to the algorithm
localAlgorithm.input.set(classifier.training.data, trainData)
localAlgorithm.input.set(classifier.training.labels, trainDependentVariables)
# Train the Naive Bayes model on local nodes
pres = localAlgorithm.compute()
# Serialize partial results required by step 2
dataArch = InputDataArchive()
pres.serialize(dataArch)
masterAlgorithm.input.add(classifier.training.partialModels, pres)
"""
nodeResults.append(dataArch.getArchiveAsArray().copy())
localAlgorithm.clean()
"""
mergedData.freeDataMemory()
trainData.freeDataMemory()
trainDependentVariables.freeDataMemory()
# Transfer partial results to step 2 on the root node
pres = masterAlgorithm.compute()
dataArch = InputDataArchive()
pres.serialize(dataArch)
nodeResults.append(dataArch.getArchiveAsArray().copy())
serializedData = comm.gather(nodeResults)
if rankId == MPI_ROOT:
# Create an algorithm object to build the final Naive Bayes model on the master node
masterAlgorithm = training.Distributed_Step2MasterFloat64DefaultDense(nClasses)
for currentRank in range(len(serializedData)):
for currentBlock in range(0, len(serializedData[currentRank])):
# Deserialize partial results from step 1
dataArch = OutputDataArchive(serializedData[currentRank][currentBlock])
dataForStep2FromStep1 = classifier.training.PartialResult()
dataForStep2FromStep1.deserialize(dataArch)
# Set the local Naive Bayes model as input for the master-node algorithm
masterAlgorithm.input.add(classifier.training.partialModels, dataForStep2FromStep1)
# Merge and finalizeCompute the Naive Bayes model on the master node
masterAlgorithm.compute()
trainingResult = masterAlgorithm.finalizeCompute()
def testModel():
thresholdValues = np.linspace(-25.0, 25.0, num=101)
numberOfCorrectlyClassifiedObjects = np.zeros(len(thresholdValues))
numberOfObjectsInTestFiles = 0
numberOfNonzeroObjectsInTestFiles = 0
for filenameIndex in range(0, len(testDatasetFileNames)):
testDataSource = FileDataSource(
testDatasetFileNames[filenameIndex],
DataSourceIface.doAllocateNumericTable,
DataSourceIface.doDictionaryFromContext)
testData = HomogenNumericTable(nFeatures, 0,
NumericTableIface.notAllocate)
testGroundTruth = HomogenNumericTable(nDependentVariables, 0,
NumericTableIface.notAllocate)
mergedData = MergedNumericTable(testData, testGroundTruth)
testDataSource.loadDataBlock(mergedData)
algorithm = prediction.Batch_Float64DefaultDense()
algorithm.input.setNumericTableInput(prediction.data, testData)
algorithm.input.setModelInput(prediction.model,
trainingResult.get(training.model))
predictionResult = algorithm.compute()
block1 = BlockDescriptor()
block2 = BlockDescriptor()
testGroundTruth.getBlockOfRows(0, testGroundTruth.getNumberOfRows(),
readOnly, block1)
predictionResult.get(prediction.prediction).getBlockOfRows(
0, testGroundTruth.getNumberOfRows(), readOnly, block2)
y_true = getClassVector(block1.getArray(), 0.000000000000)
predictionRegression = block2.getArray()
for thresholdIndex in range(0, len(thresholdValues)):
y_pred = getClassVector(predictionRegression,
thresholdValues[thresholdIndex])
numberOfCorrectlyClassifiedObjects[
thresholdIndex] += accuracy_score(y_true,
y_pred,
normalize=False)
numberOfObjectsInTestFiles += len(y_true)
numberOfNonzeroObjectsInTestFiles += np.count_nonzero(y_true)
mergedData.freeDataMemory()
testData.freeDataMemory()
testGroundTruth.freeDataMemory()
classificationAccuracyResult = np.zeros(len(thresholdValues))
best_threshold = None
best_accuracy = -1
for thresholdIndex in range(0, len(thresholdValues)):
classificationAccuracyResult[
thresholdIndex] = numberOfCorrectlyClassifiedObjects[
thresholdIndex] / numberOfObjectsInTestFiles
if (classificationAccuracyResult[thresholdIndex] > best_accuracy):
best_threshold = thresholdValues[thresholdIndex]
best_accuracy = classificationAccuracyResult[thresholdIndex]
print('Best threshold:{:.4f}. Best accuracy:{:.4f}'.format(
best_threshold, best_accuracy))
print(
'Test set. Number of objects of 0 class:{:.4f}.Number of objects of 1 class:{:.4f}. '
'Frequency of 1 class:{:.4f}'.format(
numberOfObjectsInTestFiles - numberOfNonzeroObjectsInTestFiles,
numberOfNonzeroObjectsInTestFiles,
numberOfNonzeroObjectsInTestFiles / numberOfObjectsInTestFiles))
indexOfZeroThreshold = np.where(thresholdValues == 0.0)[0][0]
print('Threshold=0. Classification accuracy:{:.4f}'.format(
classificationAccuracyResult[indexOfZeroThreshold]))