def gatherPartialResultsFromNodes(partialResult, partialResults,
partialResultArchLength,
partialResultLocalBuffer,
partialResultMasterBuffer):
dataArch = InputDataArchive()
partialResult.serialize(dataArch)
if partialResultArchLength == 0:
partialResultArchLength = dataArch.getSizeOfArchive()
# Serialized data is of equal size on each node
if rankId == MPI_ROOT and len(partialResultMasterBuffer) == 0:
partialResultMasterBuffer = np.zeros(partialResultArchLength * nNodes,
dtype=np.uint8)
if len(partialResultLocalBuffer) == 0:
partialResultLocalBuffer = np.zeros(partialResultArchLength,
dtype=np.uint8)
dataArch.copyArchiveToArray(partialResultLocalBuffer)
# Transfer partial results to step 2 on the root node
partialResultMasterBuffer = comm.gather(partialResultLocalBuffer)
if rankId == MPI_ROOT:
for node in range(nNodes):
# Deserialize partial results from step 1
dataArch = OutputDataArchive(partialResultMasterBuffer[node])
partialResults[node] = training.PartialResult()
partialResults[node].deserialize(dataArch)
def serialize(self, data, fileName=None, useCompression=False):
buffArrObjName = (str (type (data)).split ()[1].split ('>')[0] + "()").replace ("'", '')
dataArch = InputDataArchive ()
data.serialize (dataArch)
length = dataArch.getSizeOfArchive ()
bufferArray = np.zeros (length, dtype=np.ubyte)
dataArch.copyArchiveToArray (bufferArray)
if useCompression == True:
if fileName != None:
if len(fileName.rsplit (".", 1))==2:
fileName = fileName.rsplit (".", 1)[0]
compressedData = LinearRegression.compress (self,bufferArray)
np.save (fileName, compressedData)
else:
comBufferArray = LinearRegression.compress (self,bufferArray)
serialObjectDict = {"Array Object": comBufferArray,
"Object Information": buffArrObjName}
return serialObjectDict
else:
if fileName != None:
if len (fileName.rsplit (".", 1)) == 2:
fileName = fileName.rsplit (".", 1)[0]
np.save (fileName, bufferArray)
else:
serialObjectDict = {"Array Object": bufferArray,
"Object Information": buffArrObjName}
return serialObjectDict
infoFile = open (fileName + ".txt", "w")
infoFile.write (buffArrObjName)
infoFile.close ()
def serializeDAALObject(data):
# Create a data archive to serialize the numeric table
dataArch = InputDataArchive()
# Serialize the numeric table into the data archive
data.serialize(dataArch)
length = dataArch.getSizeOfArchive()
buffer = np.zeros(length, dtype=np.uint8)
dataArch.copyArchiveToArray(buffer)
return buffer
def serializeTrainingResult(self):
# Create a data archive to serialize the numeric table
dataArch = InputDataArchive()
# Serialize the numeric table into the data archive
self.trainingResult.serialize(dataArch)
# Get the length of the serialized data in bytes
length = dataArch.getSizeOfArchive()
# Store the serialized data in an array
buffer = np.zeros(length, dtype=np.ubyte)
dataArch.copyArchiveToArray(buffer)
return buffer
def serialize(
self,
data,
fileName=None,
useCompression=False,
):
buffArrObjName = (str(type(data)).split()[1].split('>')[0] +
'()').replace("'", '')
dataArch = InputDataArchive()
data.serialize(dataArch)
length = dataArch.getSizeOfArchive()
bufferArray = np.zeros(length, dtype=np.ubyte)
dataArch.copyArchiveToArray(bufferArray)
if useCompression == True:
if fileName != None:
if len(fileName.rsplit('.', 1)) == 2:
fileName = fileName.rsplit('.', 1)[0]
compressedData = Kmeans.compress(self, bufferArray)
np.save(fileName, compressedData)
else:
comBufferArray = Kmeans.compress(self, bufferArray)
serialObjectDict = {
'Array Object': comBufferArray,
'Object Information': buffArrObjName
}
return serialObjectDict
else:
if fileName != None:
if len(fileName.rsplit('.', 1)) == 2:
fileName = fileName.rsplit('.', 1)[0]
np.save(fileName, bufferArray)
else:
serialObjectDict = {
'Array Object': bufferArray,
'Object Information': buffArrObjName
}
return serialObjectDict
infoFile = open(fileName + '.txt', 'w')
infoFile.write(buffArrObjName)
infoFile.close()
def broadcastWeightsAndBiasesToNodes(wb):
wbBuffer = None
# Serialize weights and biases on the root node
if rankId == MPI_ROOT:
if not wb:
# Weights and biases table should be valid and not NULL on master
return HomogenNumericTable()
wbDataArch = InputDataArchive()
wb.serialize(wbDataArch)
wbBuffer = np.zeros(wbDataArch.getSizeOfArchive(), dtype=np.uint8)
wbDataArch.copyArchiveToArray(wbBuffer)
# Broadcast the serialized weights and biases
wbBuffer = comm.bcast(wbBuffer)
# Deserialize weights and biases
wbDataArchLocal = OutputDataArchive(wbBuffer)
wbLocal = HomogenNumericTable(ntype=np.float32)
wbLocal.deserialize(wbDataArchLocal)
return wbLocal
dataSource.loadDataBlock()
# Create an algorithm to compute a variance-covariance matrix on local nodes
localAlgorithm = covariance.Distributed(step1Local)
# Set the input data set to the algorithm
localAlgorithm.input.set(covariance.data, dataSource.getNumericTable())
# Compute a variance-covariance matrix
pres = localAlgorithm.compute()
# Serialize partial results required by step 2
dataArch = InputDataArchive()
pres.serialize(dataArch)
perNodeArchLength = dataArch.getSizeOfArchive()
nodeResults = dataArch.getArchiveAsArray()
# Transfer partial results to step 2 on the root node
data = comm_size.gather(nodeResults, MPI_ROOT)
if rankId == MPI_ROOT:
# Create an algorithm to compute a variance-covariance matrix on the master node
masterAlgorithm = covariance.Distributed(step2Master)
for i in range(nBlocks):
# Deserialize partial results from step 1
dataArch = OutputDataArchive(data[i])