# ###############################################
# --------- Information of parallel pool --------
printMessage('\nParallel information', rank)
printMessage('=' * 75, rank)
[
Istart_elemsE, Iend_elemsE, Istart_boundaries, Iend_boundaries,
Istart_receivers, Iend_receivers
] = getRanges(elemsE, boundaries, receivers, size, rank)
# ###############################################
# ----- Create and setup parallel structures ----
# Left-hand side
A = createParallelMatrix(ndofs, ndofs, nnz, communicator=None)
# Right-hand side
b = createParallelVector(ndofs, communicator=None)
# X vector
x = createParallelVector(ndofs, communicator=None)
# ###############################################
# -------------- Parallel assembly --------------
printMessage('\nParallel assembly', rank)
printMessage('=' * 75, rank)
# Create and start log event for assembly task
assemblerLog = startLogEvent("Assembler")
assemblerLog.begin()
# System assembly
[A, b, elapsedTimeAssembly
] = parallelAssembler(modelling, A, b, nodes, elemsE, elemsN, elemsF,
elemsSigma, Istart_elemsE, Iend_elemsE, nEdges,
nFaces, rank)
# ###############################################
# --------- Information of parallel pool --------
printMessage('\nParallel information', rank)
printMessage('=' * 75, rank)
[
Istart_elemsE, Iend_elemsE, Istart_bEdges, Iend_bEdges,
Istart_receivers, Iend_receivers
] = getRanges(elemsE, bEdges, receivers, size, rank)
# ###############################################
# ----- Create and setup parallel structures ----
# Left-hand side
A = createParallelMatrix(nEdges, nEdges, nnz, communicator=None)
# Right-hand side
b = createParallelVector(nEdges, communicator=None)
# X vector
x = createParallelVector(nEdges, communicator=None)
# ###############################################
# -------------- Parallel assembly --------------
printMessage('\nParallel assembly', rank)
printMessage('=' * 75, rank)
# Create and start log event for assembly task
assemblerLog = startLogEvent("Assembler")
assemblerLog.begin()
# System assembly
[A, b,
elapsedTimeAssembly] = parallelAssembler(modelling, A, b, nodes, elemsE,
elemsN, elemsSigma,
Istart_elemsE, Iend_elemsE, rank)
def postProcessingFields(receivers, modelling, x, Iend_receivers,
Istart_receivers, edgeOrder, nodalOrder,
numDimensions, rank):
''' Compute the CSEM modelling output: primary electric field, secondary
electric field and total electric field on receivers position.
:param petsc matrix receivers: data receivers to compute electric fields
:param object_modelling model: CSEM modelling with physical parameters.
:param petsc vector x: solution vector
:param int Iend_receivers: last range for receivers
:param int Istart_receivers: init range for receivers
:param int edgeOrder: order of tetrahedral edge element
:param int nodalOrder: order of tetrahedral nodal element
:param int numDimensions: number of dimensions
:param int rank: MPI rank
:return: elapsedTimepostprocessing
:rtype: float
'''
# Start timer
Init_postprocessing = getTime()
# Number of receivers
nReceivers = receivers.getSize()[0]
nReceiversLocal = Iend_receivers-Istart_receivers
# Print number of receivers per MPI task
PETSc.Sys.Print(' Number of receivers:', nReceivers)
PETSc.Sys.syncPrint(' Rank: ', rank, ' is post-processing ',
nReceiversLocal, ' receivers')
PETSc.Sys.syncFlush()
# Read edges-connectivity for receivers
# Auxiliar arrays
dataRecv = np.zeros(edgeOrder, dtype=np.float)
edgesIdxRecv = np.zeros((nReceiversLocal, edgeOrder), dtype=PETSc.IntType)
idx = 0
for iRecv in np.arange(Istart_receivers, Iend_receivers):
# Get data of iRecv
temp = np.asarray(receivers.getRow(iRecv))
dataRecv[:] = np.real(temp[1, 19:25])
# Edge-indexes for iRecv
edgesIdxRecv[idx, :] = (dataRecv).astype(PETSc.IntType)
idx += 1
# Gather global solution of x to local vector
# Sequential vector for gather tasks
x_local = createSequentialVector(edgeOrder*nReceiversLocal,
communicator=None)
# Build Index set in PETSc format
IS_edges = PETSc.IS().createGeneral(edgesIdxRecv.flatten(),
comm=PETSc.COMM_WORLD)
# Build gather vector
gatherVector = PETSc.Scatter().create(x, IS_edges, x_local, None)
# Ghater values
gatherVector.scatter(x, x_local, PETSc.InsertMode.INSERT_VALUES,
PETSc.ScatterMode.FORWARD)
# Post-processing electric fields
# Create parallel structures
EpX = createParallelVector(nReceivers, communicator=None)
EpY = createParallelVector(nReceivers, communicator=None)
EpZ = createParallelVector(nReceivers, communicator=None)
EsX = createParallelVector(nReceivers, communicator=None)
EsY = createParallelVector(nReceivers, communicator=None)
EsZ = createParallelVector(nReceivers, communicator=None)
EtX = createParallelVector(nReceivers, communicator=None)
EtY = createParallelVector(nReceivers, communicator=None)
EtZ = createParallelVector(nReceivers, communicator=None)
EpDense = createParallelDenseMatrix(nReceivers, numDimensions,
communicator=None)
EsDense = createParallelDenseMatrix(nReceivers, numDimensions,
communicator=None)
EtDense = createParallelDenseMatrix(nReceivers, numDimensions,
communicator=None)
# Reshape auxiliar array
dataRecv = np.zeros(numDimensions+nodalOrder*numDimensions+nodalOrder,
dtype=np.float)
# Compute fields for all local receivers
idx = 0
for iRecv in np.arange(Istart_receivers, Iend_receivers):
# Get data of iRecv
temp = np.asarray(receivers.getRow(iRecv))
dataRecv[:] = np.real(temp[1, 0:19])
# Receivers coordinates
coordReceiver = dataRecv[0:3]
# Element coordinates
coordElement = dataRecv[3:15]
# Nodal-indexes
nodesElement = (dataRecv[15:19]).astype(PETSc.IntType)
# Compute fields
[EpRecv, EsRecv, EtRecv] = computeFieldsReceiver(modelling,
coordReceiver,
coordElement,
nodesElement,
x_local[idx *
edgeOrder:
(idx *
edgeOrder) +
edgeOrder],
edgeOrder,
numDimensions)
idx += 1
# Set primary field components
EpX.setValue(iRecv, EpRecv[0], addv=PETSc.InsertMode.INSERT_VALUES)
EpY.setValue(iRecv, EpRecv[1], addv=PETSc.InsertMode.INSERT_VALUES)
EpZ.setValue(iRecv, EpRecv[2], addv=PETSc.InsertMode.INSERT_VALUES)
EpDense.setValue(iRecv, 0, EpRecv[0],
addv=PETSc.InsertMode.INSERT_VALUES)
EpDense.setValue(iRecv, 1, EpRecv[1],
addv=PETSc.InsertMode.INSERT_VALUES)
EpDense.setValue(iRecv, 2, EpRecv[2],
addv=PETSc.InsertMode.INSERT_VALUES)
# Set secondary field components
EsX.setValue(iRecv, EsRecv[0],
addv=PETSc.InsertMode.INSERT_VALUES)
EsY.setValue(iRecv, EsRecv[1],
addv=PETSc.InsertMode.INSERT_VALUES)
EsZ.setValue(iRecv, EsRecv[2],
addv=PETSc.InsertMode.INSERT_VALUES)
EsDense.setValue(iRecv, 0, EsRecv[0],
addv=PETSc.InsertMode.INSERT_VALUES)
EsDense.setValue(iRecv, 1, EsRecv[1],
addv=PETSc.InsertMode.INSERT_VALUES)
EsDense.setValue(iRecv, 2, EsRecv[2],
addv=PETSc.InsertMode.INSERT_VALUES)
# Set total field components
EtX.setValue(iRecv, EtRecv[0], addv=PETSc.InsertMode.INSERT_VALUES)
EtY.setValue(iRecv, EtRecv[1], addv=PETSc.InsertMode.INSERT_VALUES)
EtZ.setValue(iRecv, EtRecv[2], addv=PETSc.InsertMode.INSERT_VALUES)
EtDense.setValue(iRecv, 0, EtRecv[0],
addv=PETSc.InsertMode.INSERT_VALUES)
EtDense.setValue(iRecv, 1, EtRecv[1],
addv=PETSc.InsertMode.INSERT_VALUES)
EtDense.setValue(iRecv, 2, EtRecv[2],
addv=PETSc.InsertMode.INSERT_VALUES)
# Start global vector assembly
EpX.assemblyBegin(), EpY.assemblyBegin(), EpZ.assemblyBegin()
EsX.assemblyBegin(), EsY.assemblyBegin(), EsZ.assemblyBegin()
EtX.assemblyBegin(), EtY.assemblyBegin(), EtZ.assemblyBegin()
EpDense.assemblyBegin(), EsDense.assemblyBegin(), EtDense.assemblyBegin()
# End global vector assembly
EpX.assemblyEnd(), EpY.assemblyEnd(), EpZ.assemblyEnd()
EsX.assemblyEnd(), EsY.assemblyEnd(), EsZ.assemblyEnd()
EtX.assemblyEnd(), EtY.assemblyEnd(), EtZ.assemblyEnd()
EpDense.assemblyEnd(), EsDense.assemblyEnd(), EtDense.assemblyEnd()
# Verify if directory exists
MASTER = 0
if rank == MASTER:
checkIfDirectoryExist(modelling['DIR_NAME'] + '/Output/Petsc')
checkIfDirectoryExist(modelling['DIR_NAME'] + '/Output/Ascii')
checkIfDirectoryExist(modelling['DIR_NAME'] + '/Output/Matlab')
# Print
PETSc.Sys.Print(' Saving output:')
# Export electric fields (petsc format)
printMessage(' Petsc format', rank)
# Save primary electric field
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EpX.dat',
EpX, communicator=None)
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EpY.dat',
EpY, communicator=None)
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EpZ.dat',
EpZ, communicator=None)
writeDenseMatrix(modelling['DIR_NAME'] + '/Output/Petsc/Ep.dat',
EpDense, communicator=None)
# Save secondary electric field
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EsX.dat',
EsX, communicator=None)
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EsY.dat',
EsY, communicator=None)
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EsZ.dat',
EsZ, communicator=None)
writeDenseMatrix(modelling['DIR_NAME'] + '/Output/Petsc/Es.dat',
EsDense, communicator=None)
# Save total electric field
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EtX.dat',
EtX, communicator=None)
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EtY.dat',
EtY, communicator=None)
writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EtZ.dat',
EtZ, communicator=None)
writeDenseMatrix(modelling['DIR_NAME'] + '/Output/Petsc/Et.dat',
EtDense, communicator=None)
# Export electric fields (Ascii and Matlab format)
if rank == MASTER:
# Export electric fields (Ascii format)
# Save primary electric field
printMessage(' Ascii format', rank)
dataEp = exportPetscToAscii(nReceivers,
modelling['DIR_NAME'] +
'/Output/Petsc/EpX.dat',
modelling['DIR_NAME'] +
'/Output/Petsc/EpY.dat',
modelling['DIR_NAME'] +
'/Output/Petsc/EpZ.dat',
modelling['DIR_NAME'] +
'/Output/Ascii/Ep.dat')
# Save secondary electric field
dataEs = exportPetscToAscii(nReceivers,
modelling['DIR_NAME'] +
'/Output/Petsc/EsX.dat',
modelling['DIR_NAME'] +
'/Output/Petsc/EsY.dat',
modelling['DIR_NAME'] +
'/Output/Petsc/EsZ.dat',
modelling['DIR_NAME'] +
'/Output/Ascii/Es.dat')
# Save total electric field
dataEt = exportPetscToAscii(nReceivers,
modelling['DIR_NAME'] +
'/Output/Petsc/EtX.dat',
modelling['DIR_NAME'] +
'/Output/Petsc/EtY.dat',
modelling['DIR_NAME'] +
'/Output/Petsc/EtZ.dat',
modelling['DIR_NAME'] +
'/Output/Ascii/Et.dat')
# Export electric fields (Matlab format)
printMessage(' Matlab format', rank)
# Save primary electric field
exportNumpytoMatlab(dataEp, modelling['DIR_NAME'] +
'/Output/Matlab/Ep.mat', electricField='Primary')
# Save secondary electric field
exportNumpytoMatlab(dataEs, modelling['DIR_NAME'] +
'/Output/Matlab/Es.mat', electricField='Secondary')
# Save total electric field
exportNumpytoMatlab(dataEt, modelling['DIR_NAME'] +
'/Output/Matlab/Et.mat', electricField='Total')
# Remove temporal files (petsc)
filesToDelete = [modelling['DIR_NAME'] + '/Output/Petsc/EpX.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EpY.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EpZ.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EsX.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EsY.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EsZ.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EtX.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EtY.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EtZ.dat',
modelling['DIR_NAME'] + '/Output/Petsc/EpX.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EpY.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EpZ.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EsX.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EsY.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EsZ.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EtX.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EtY.dat.info',
modelling['DIR_NAME'] + '/Output/Petsc/EtZ.dat.info']
for iFile in np.arange(len(filesToDelete)):
removeFile(filesToDelete[iFile])
# End timer
End_postprocessing = getTime()
# Elapsed time in assembly
elapsedTimepostprocessing = End_postprocessing-Init_postprocessing
return elapsedTimepostprocessing