def gatherPartialResultsFromNodes(partialResult, partialResults, partialResultArchLength, partialResultLocalBuffer, partialResultMasterBuffer): dataArch = InputDataArchive() partialResult.serialize(dataArch) if partialResultArchLength == 0: partialResultArchLength = dataArch.getSizeOfArchive() # Serialized data is of equal size on each node if rankId == MPI_ROOT and len(partialResultMasterBuffer) == 0: partialResultMasterBuffer = np.zeros(partialResultArchLength * nNodes, dtype=np.uint8) if len(partialResultLocalBuffer) == 0: partialResultLocalBuffer = np.zeros(partialResultArchLength, dtype=np.uint8) dataArch.copyArchiveToArray(partialResultLocalBuffer) # Transfer partial results to step 2 on the root node partialResultMasterBuffer = comm.gather(partialResultLocalBuffer) if rankId == MPI_ROOT: for node in range(nNodes): # Deserialize partial results from step 1 dataArch = OutputDataArchive(partialResultMasterBuffer[node]) partialResults[node] = training.PartialResult() partialResults[node].deserialize(dataArch)
def finalizeMergeOnMasterNode(partsRDD): # Create an algorithm to compute PCA decomposition using the correlation method on the master node pcaMaster = pca.Distributed(step2Master, method=pca.correlationDense) covarianceSparse = covariance.Distributed(step2Master, method=covariance.fastCSR) pcaMaster.parameter.covariance = covarianceSparse parts_list = partsRDD.collect() # Add partial results computed on local nodes to the algorithm on the master node for key, pres in parts_list: dataArch = OutputDataArchive(pres) deserialized_pres = pca.PartialResult(pca.correlationDense) deserialized_pres.deserialize(dataArch) pcaMaster.input.add(pca.partialResults, deserialized_pres) # Compute PCA decomposition on the master node pcaMaster.compute() # Finalize computations and retrieve the results res = pcaMaster.finalizeCompute() return { 'eigenvectors': res.get(pca.eigenvectors), 'eigenvalues': res.get(pca.eigenvalues) }
def deserialize(self, serialObjectDict=None, fileName=None, useCompression=False): import daal if fileName != None and serialObjectDict == None: bufferArray = np.load(fileName) buffArrObjName = open(fileName.rsplit(".", 1)[0] + ".txt", "r").read() elif fileName == None and any(serialObjectDict): bufferArray = serialObjectDict["Array Object"] buffArrObjName = serialObjectDict["Object Information"] else: warnings.warn( 'Expecting "bufferArray" or "fileName" argument, NOT both') raise SystemExit if useCompression == True: bufferArray = MultiSVM.decompress(self, bufferArray) dataArch = OutputDataArchive(bufferArray) try: deSerialObj = eval(buffArrObjName) except AttributeError: deSerialObj = HomogenNumericTable() deSerialObj.deserialize(dataArch) return deSerialObj
def finalizeMergeOnMasterNode(partsRDD): # Create an algorithm to compute a dense variance-covariance matrix on the master node covarianceMaster = covariance.Distributed(step=step2Master, method=covariance.defaultDense) parts_list = partsRDD.collect() # Add partial results computed on local nodes to the algorithm on the master node for _, val in parts_list: dataArch = OutputDataArchive(val) deserialized_val = covariance.PartialResult() deserialized_val.deserialize(dataArch) covarianceMaster.input.add(covariance.partialResults, deserialized_val) # Compute a dense variance-covariance matrix on the master node covarianceMaster.compute() # Finalize computations and retrieve the results res = covarianceMaster.finalizeCompute() result = {} result['covariance'] = res.get(covariance.covariance) result['mean'] = res.get(covariance.mean) return result
def deserializeDAALObject(buffer, object): # Create a data archive to deserialize the numeric table dataArch = OutputDataArchive(buffer) # Deserialize the numeric table from the data archive object.deserialize(dataArch) return object
def deserializePartialResult(buffer, module, partial=True): dataArch = OutputDataArchive(buffer) if partial: deserialized_pres = module.PartialResult() else: deserialized_pres = module.Result() deserialized_pres.deserialize(dataArch) return deserialized_pres
def deserializeTrainingResult(self, buffer): # Create a data archive to deserialize the numeric table dataArch = OutputDataArchive(buffer) # Create a numeric table object self.trainingResult = training.Result() # Deserialize the numeric table from the data archive self.trainingResult.deserialize(dataArch) return self.trainingResult
def trainModel(): global trainingResult # Retrieve the input data from a .csv file trainDataTable = createSparseTable(trainDatasetFileNames[rankId]) # Initialize FileDataSource to retrieve the input data from a .csv file trainLabelsSource = FileDataSource(trainGroundTruthFileNames[rankId], DataSourceIface.doAllocateNumericTable, DataSourceIface.doDictionaryFromContext) # Retrieve the data from input files trainLabelsSource.loadDataBlock() # Create an algorithm object to train the Naive Bayes model based on the local-node data localAlgorithm = training.Distributed(step1Local, nClasses, method=training.fastCSR) # Pass a training data set and dependent values to the algorithm localAlgorithm.input.set(classifier.training.data, trainDataTable) localAlgorithm.input.set(classifier.training.labels, trainLabelsSource.getNumericTable()) # Train the Naive Bayes model on local nodes pres = localAlgorithm.compute() # Serialize partial results required by step 2 dataArch = InputDataArchive() pres.serialize(dataArch) nodeResults = dataArch.getArchiveAsArray() # Transfer partial results to step 2 on the root node serializedData = comm.gather(nodeResults) if rankId == MPI_ROOT: # Create an algorithm object to build the final Naive Bayes model on the master node masterAlgorithm = training.Distributed(step2Master, nClasses, method=training.fastCSR) for i in range(nBlocks): # Deserialize partial results from step 1 dataArch = OutputDataArchive(serializedData[i]) dataForStep2FromStep1 = training.PartialResult() dataForStep2FromStep1.deserialize(dataArch) # Set the local Naive Bayes model as input for the master-node algorithm masterAlgorithm.input.add(training.partialModels, dataForStep2FromStep1) # Merge and finalizeCompute the Naive Bayes model on the master node masterAlgorithm.compute() trainingResult = masterAlgorithm.finalizeCompute()
def deserializeCSRNumericTable(buffer): dataArch = OutputDataArchive(buffer) dataTable = CSRNumericTable(np.zeros(1, dtype=np.float64), np.zeros(1, dtype=np.uint64), np.zeros(1, dtype=np.uint64), 0, 0) dataTable.deserialize(dataArch) return dataTable
def deserializeDataCollection(buffer): # Create a data archive to deserialize the numeric table dataArch = OutputDataArchive(buffer) # Create a data collection object collection = DataCollection() # Deserialize the numeric table from the data archive collection.deserialize(dataArch) return collection
def deserializeNumericTable(buffer): # Create a data archive to deserialize the numeric table dataArch = OutputDataArchive(buffer) # Create a numeric table object dataTable = HomogenNumericTable() # Deserialize the numeric table from the data archive dataTable.deserialize(dataArch) return dataTable
def computeOnMasterNode(): global R, serializedData # Create an algorithm to compute QR decomposition on the master node algorithm = qr.Distributed(step2Master) for i in range(nBlocks): # Deserialize partial results from step 1 dataArch = OutputDataArchive(serializedData[i]) dataForStep2FromStep1 = DataCollection() dataForStep2FromStep1.deserialize(dataArch) algorithm.input.add(qr.inputOfStep2FromStep1, i, dataForStep2FromStep1) # Compute QR decomposition pres = algorithm.compute() inputForStep3FromStep2 = pres.getCollection(qr.outputOfStep2ForStep3) for i in range(nBlocks): # Serialize partial results to transfer to local nodes for step 3 dataArch = InputDataArchive() inputForStep3FromStep2[i].serialize(dataArch) length = dataArch.getSizeOfArchive() serializedData[i] = np.empty(length, dtype=np.uint8) dataArch.copyArchiveToArray(serializedData[i]) # Result class from qr res = algorithm.getResult() R = res.get(qr.matrixR)
def trainModel(): global trainingResult masterAlgorithm = training.Distributed_Step2MasterFloat64NormEqDense() for filenameIndex in range(rankId, len(trainDatasetFileNames), comm_size): trainDataSource = FileDataSource( trainDatasetFileNames[filenameIndex], DataSourceIface.notAllocateNumericTable, DataSourceIface.doDictionaryFromContext) trainData = HomogenNumericTable(nFeatures, 0, NumericTableIface.notAllocate) trainDependentVariables = HomogenNumericTable( nDependentVariables, 0, NumericTableIface.notAllocate) mergedData = MergedNumericTable(trainData, trainDependentVariables) trainDataSource.loadDataBlock(mergedData) localAlgorithm = training.Distributed_Step1LocalFloat64NormEqDense() localAlgorithm.input.set(training.data, trainData) localAlgorithm.input.set(training.dependentVariables, trainDependentVariables) pres = localAlgorithm.compute() masterAlgorithm.input.add(training.partialModels, pres) mergedData.freeDataMemory() trainData.freeDataMemory() trainDependentVariables.freeDataMemory() pres = masterAlgorithm.compute() dataArch = InputDataArchive() pres.serialize(dataArch) nodeResults = dataArch.getArchiveAsArray() serializedData = comm.gather(nodeResults) if rankId == MPI_ROOT: print("Number of processes is %d." % (len(serializedData))) masterAlgorithm = training.Distributed_Step2MasterFloat64NormEqDense() for i in range(comm_size): dataArch = OutputDataArchive(serializedData[i]) dataForStep2FromStep1 = training.PartialResult() dataForStep2FromStep1.deserialize(dataArch) masterAlgorithm.input.add(training.partialModels, dataForStep2FromStep1) masterAlgorithm.compute() trainingResult = masterAlgorithm.finalizeCompute()
def finalizeComputestep1Local(): global Ui, serializedData # Transfer partial results from the root node nodeResults = comm.scatter(serializedData) # Deserialize partial results from step 2 dataArch = OutputDataArchive(nodeResults) dataFromStep2ForStep3 = DataCollection() dataFromStep2ForStep3.deserialize(dataArch) # Create an algorithm to compute SVD on the master node algorithm = svd.Distributed(step3Local) algorithm.input.set(svd.inputOfStep3FromStep1, dataFromStep1ForStep3) algorithm.input.set(svd.inputOfStep3FromStep2, dataFromStep2ForStep3) # Compute SVD algorithm.compute() res = algorithm.finalizeCompute() Ui = res.get(svd.leftSingularMatrix)
def computeStep2Master(dataFromStep1ForStep2_RDD): nBlocks = int(dataFromStep1ForStep2_RDD.count()) dataFromStep1ForStep2_list = dataFromStep1ForStep2_RDD.collect() # Create an algorithm to compute SVD on the master node svdStep2Master = svd.Distributed(step2Master, method=svd.defaultDense) for key, data_collection in dataFromStep1ForStep2_list: dataArch = OutputDataArchive(data_collection) deserialized_data_collection = DataCollection() deserialized_data_collection.deserialize(dataArch) svdStep2Master.input.add(svd.inputOfStep2FromStep1, key, deserialized_data_collection) # Compute SVD in step 2 pres = svdStep2Master.compute() inputForStep3FromStep2 = pres.getCollection(svd.outputOfStep2ForStep3) data_list = [] for key, _ in dataFromStep1ForStep2_list: dc = inputForStep3FromStep2[key] serialized_dc = serializeNumericTable(dc) data_list.append((key, serialized_dc)) # Make PairRDD from the list dataFromStep2ForStep3_RDD = sc.parallelize(data_list, nBlocks) res = svdStep2Master.finalizeCompute() result = { 'Sigma': res.get(svd.singularValues), 'V': res.get(svd.rightSingularMatrix), 'from2_for3': dataFromStep2ForStep3_RDD } return result
def finalizeComputestep1Local(): global Qi, serializedData # Transfer partial results from the root node nodeResults = comm.scatter(serializedData) # Deserialize partial results from step 2 dataArch = OutputDataArchive(nodeResults) dataFromStep2ForStep3 = DataCollection() dataFromStep2ForStep3.deserialize(dataArch) # Create an algorithm to compute QR decomposition on the master node algorithm = qr.Distributed(step3Local) algorithm.input.set(qr.inputOfStep3FromStep1, dataFromStep1ForStep3) algorithm.input.set(qr.inputOfStep3FromStep2, dataFromStep2ForStep3) # Compute QR decomposition algorithm.compute() res = algorithm.finalizeCompute() Qi = res.get(qr.matrixQ)
def broadcastWeightsAndBiasesToNodes(wb): wbBuffer = None # Serialize weights and biases on the root node if rankId == MPI_ROOT: if not wb: # Weights and biases table should be valid and not NULL on master return HomogenNumericTable() wbDataArch = InputDataArchive() wb.serialize(wbDataArch) wbBuffer = np.zeros(wbDataArch.getSizeOfArchive(), dtype=np.uint8) wbDataArch.copyArchiveToArray(wbBuffer) # Broadcast the serialized weights and biases wbBuffer = comm.bcast(wbBuffer) # Deserialize weights and biases wbDataArchLocal = OutputDataArchive(wbBuffer) wbLocal = HomogenNumericTable(ntype=np.float32) wbLocal.deserialize(wbDataArchLocal) return wbLocal
def computeOnMasterNode(): global serializedData, Sigma, V # Create an algorithm to compute SVD on the master node algorithm = svd.Distributed(step2Master) for i in range(nBlocks): # Deserialize partial results from step 1 dataArch = OutputDataArchive(serializedData[i]) dataForStep2FromStep1 = DataCollection() dataForStep2FromStep1.deserialize(dataArch) algorithm.input.add(svd.inputOfStep2FromStep1, i, dataForStep2FromStep1) # Compute SVD # DistributedPartialResult class from svd pres = algorithm.compute() inputForStep3FromStep2 = pres.getCollection(svd.outputOfStep2ForStep3) for i in range(nBlocks): # Serialize partial results to transfer to local nodes for step 3 dataArch = InputDataArchive() inputForStep3FromStep2[i].serialize(dataArch) length = dataArch.getSizeOfArchive() serializedData[i] = np.empty(length, dtype=np.uint8) dataArch.copyArchiveToArray(serializedData[i]) # DistributedPartialResult class from svd res = algorithm.getResult() Sigma = res.get(svd.singularValues) V = res.get(svd.rightSingularMatrix)
dataArch = InputDataArchive() pres.serialize(dataArch) nodeResults = dataArch.getArchiveAsArray() # Transfer partial results to step 2 on the root node serializedData = comm.gather(nodeResults) if rankId == MPI_ROOT: # Create an algorithm to compute low order moments on the master node masterAlgorithm = low_order_moments.Distributed( step2Master, method=low_order_moments.fastCSR) for i in range(nBlocks): # Deserialize partial results from step 1 dataArch = OutputDataArchive(serializedData[i]) dataForStep2FromStep1 = low_order_moments.PartialResult() dataForStep2FromStep1.deserialize(dataArch) # Set local partial results as input for the master-node algorithm masterAlgorithm.input.add(low_order_moments.partialResults, dataForStep2FromStep1) # Merge and finalizeCompute low order moments on the master node masterAlgorithm.compute() res = masterAlgorithm.finalizeCompute() # Print the results printNumericTable(res.get(low_order_moments.minimum), "Minimum:")
perNodeArchLength = dataArch.getSizeOfArchive() nodeResults = dataArch.getArchiveAsArray() # Transfer partial results to step 2 on the root node data = comm_size.gather(nodeResults, MPI_ROOT) if rankId == MPI_ROOT: # Create an algorithm to compute a variance-covariance matrix on the master node masterAlgorithm = covariance.Distributed(step2Master) for i in range(nBlocks): # Deserialize partial results from step 1 dataArch = OutputDataArchive(data[i]) dataForStep2FromStep1 = covariance.PartialResult() dataForStep2FromStep1.deserialize(dataArch) # Set local partial results as input for the master-node algorithm masterAlgorithm.input.add(covariance.partialResults, dataForStep2FromStep1) # Merge and finalizeCompute a dense variance-covariance matrix on the master node */ masterAlgorithm.compute() result = masterAlgorithm.finalizeCompute() # Print the results printNumericTable(result.get(covariance.covariance), "Covariance matrix:") printNumericTable(result.get(covariance.mean), "Mean vector:")
def trainModel(): global trainingResult nodeResults = [] # Create an algorithm object to build the final Naive Bayes model on the master node masterAlgorithm = training.Distributed_Step2MasterFloat64DefaultDense(nClasses) for filenameIndex in range(rankId, len(trainDatasetFileNames), comm_size): # Initialize FileDataSource to retrieve the input data from a .csv file #print("The worker with rank %d will read %s." % (rankId, trainDatasetFileNames[filenameIndex])) trainDataSource = FileDataSource(trainDatasetFileNames[filenameIndex], DataSourceIface.notAllocateNumericTable, DataSourceIface.doDictionaryFromContext) # Create Numeric Tables for training data and labels trainData = HomogenNumericTable(nFeatures, 0, NumericTableIface.notAllocate) trainDependentVariables = HomogenNumericTable(1, 0, NumericTableIface.notAllocate) mergedData = MergedNumericTable(trainData, trainDependentVariables) # Retrieve the data from the input file trainDataSource.loadDataBlock(mergedData) # Create an algorithm object to train the Naive Bayes model based on the local-node data localAlgorithm = training.Distributed_Step1LocalFloat64DefaultDense(nClasses) # Pass a training data set and dependent values to the algorithm localAlgorithm.input.set(classifier.training.data, trainData) localAlgorithm.input.set(classifier.training.labels, trainDependentVariables) # Train the Naive Bayes model on local nodes pres = localAlgorithm.compute() # Serialize partial results required by step 2 dataArch = InputDataArchive() pres.serialize(dataArch) masterAlgorithm.input.add(classifier.training.partialModels, pres) """ nodeResults.append(dataArch.getArchiveAsArray().copy()) localAlgorithm.clean() """ mergedData.freeDataMemory() trainData.freeDataMemory() trainDependentVariables.freeDataMemory() # Transfer partial results to step 2 on the root node pres = masterAlgorithm.compute() dataArch = InputDataArchive() pres.serialize(dataArch) nodeResults.append(dataArch.getArchiveAsArray().copy()) serializedData = comm.gather(nodeResults) if rankId == MPI_ROOT: # Create an algorithm object to build the final Naive Bayes model on the master node masterAlgorithm = training.Distributed_Step2MasterFloat64DefaultDense(nClasses) for currentRank in range(len(serializedData)): for currentBlock in range(0, len(serializedData[currentRank])): # Deserialize partial results from step 1 dataArch = OutputDataArchive(serializedData[currentRank][currentBlock]) dataForStep2FromStep1 = classifier.training.PartialResult() dataForStep2FromStep1.deserialize(dataArch) # Set the local Naive Bayes model as input for the master-node algorithm masterAlgorithm.input.add(classifier.training.partialModels, dataForStep2FromStep1) # Merge and finalizeCompute the Naive Bayes model on the master node masterAlgorithm.compute() trainingResult = masterAlgorithm.finalizeCompute()
def trainModel(comm, rankId): trainingResult = None # Initialize FileDataSource to retrieve the input data from a .csv file trainDataSource = FileDataSource(trainDatasetFileNames[rankId], DataSourceIface.notAllocateNumericTable, DataSourceIface.doDictionaryFromContext) # Create Numeric Tables for training data and labels trainData = HomogenNumericTable(NUM_FEATURES, 0, NumericTableIface.doNotAllocate) trainDependentVariables = HomogenNumericTable( NUM_DEPENDENT_VARS, 0, NumericTableIface.doNotAllocate) mergedData = MergedNumericTable(trainData, trainDependentVariables) # Retrieve the data from the input file trainDataSource.loadDataBlock(mergedData) # Create an algorithm object to train the ridge regression model based on the local-node data localAlgorithm = training.Distributed(step1Local) # Pass a training data set and dependent values to the algorithm localAlgorithm.input.set(training.data, trainData) localAlgorithm.input.set(training.dependentVariables, trainDependentVariables) # Train the ridge regression model on local nodes pres = localAlgorithm.compute() # Serialize partial results required by step 2 dataArch = InputDataArchive() pres.serialize(dataArch) # Transfer partial results to step 2 on the root node nodeResults = dataArch.getArchiveAsArray() serializedData = comm.gather(nodeResults) if rankId == MPI_ROOT: # Create an algorithm object to build the final ridge regression model on the master node masterAlgorithm = training.Distributed(step2Master) for i in range(NUM_BLOCKS): # Deserialize partial results from step 1 dataArch = OutputDataArchive(serializedData[i]) dataForStep2FromStep1 = training.PartialResult() dataForStep2FromStep1.deserialize(dataArch) # Set the local ridge regression model as input for the master-node algorithm masterAlgorithm.input.add(training.partialModels, dataForStep2FromStep1) # Merge and finalizeCompute the ridge regression model on the master node masterAlgorithm.compute() trainingResult = masterAlgorithm.finalizeCompute() # Retrieve the algorithm results printNumericTable( trainingResult.get(training.model).getBeta(), "Ridge Regression coefficients:") return trainingResult