from petgem.preprocessing.preprocessing import preprocessingDataReceivers from petgem.preprocessing.preprocessing import preprocessingNNZ from petgem.preprocessing.preprocessing import printPreprocessingSummary # ############################################### # COMMENT: PETGEM preprocessing is a sequential task, therefore # rank == 0 in order to use parallel PETGEM functions # ############################################### # ----------- Print header (Master) ------------- rank = 0 printPetgemHeader(rank) # ############################################### # ----------------- User input ------------------ printMessage('\nInit', rank) printMessage('=' * 75, rank) input_params = sys.argv preprocessing = readPreprocessingParams(input_params, rank) # ############################################### # ----------- Check and read user input --------- printMessage('\nData preprocessing', rank) printMessage('=' * 75, rank) # ############################################### # ----------- Data preprocessing --------- # Nodal coordinates preprocessing (nodes) nNodes = preprocessNodes(preprocessing['MESH_FILE'], preprocessing['OUT_DIR'], rank)
# ############################################### # -- Get rank, size and MASTER of the MPI pool -- MASTER, rank, size = getRankSize() # ############################################### # ----- Define geometry and EFEM constants ----- edgeOrder, nodalOrder, numDimensions = defineEFEMConstants() # ############################################### # ----------- Print header (Master) ------------- printPetgemHeader(rank) # ############################################### # ----------------- User input ------------------ printMessage('\nInit', rank) printMessage('=' * 75, rank) input_params = sys.argv # ############################################### # ------- Check and read parameters file -------- modelling = readUserParams(input_params, rank) # ############################################### # ---------------- Read mesh -------------------- printMessage('\nImport files', rank) printMessage('=' * 75, rank) # Create and start log event for importing task importLog = startLogEvent("Import_files") importLog.begin() # Read nodes coordinates
def postProcessingFields(receivers, modelling, x, Iend_receivers, Istart_receivers, edgeOrder, nodalOrder, numDimensions, rank): ''' Compute the CSEM modelling output: primary electric field, secondary electric field and total electric field on receivers position. :param petsc matrix receivers: data receivers to compute electric fields :param object_modelling model: CSEM modelling with physical parameters. :param petsc vector x: solution vector :param int Iend_receivers: last range for receivers :param int Istart_receivers: init range for receivers :param int edgeOrder: order of tetrahedral edge element :param int nodalOrder: order of tetrahedral nodal element :param int numDimensions: number of dimensions :param int rank: MPI rank :return: elapsedTimepostprocessing :rtype: float ''' # Start timer Init_postprocessing = getTime() # Number of receivers nReceivers = receivers.getSize()[0] nReceiversLocal = Iend_receivers-Istart_receivers # Print number of receivers per MPI task PETSc.Sys.Print(' Number of receivers:', nReceivers) PETSc.Sys.syncPrint(' Rank: ', rank, ' is post-processing ', nReceiversLocal, ' receivers') PETSc.Sys.syncFlush() # Read edges-connectivity for receivers # Auxiliar arrays dataRecv = np.zeros(edgeOrder, dtype=np.float) edgesIdxRecv = np.zeros((nReceiversLocal, edgeOrder), dtype=PETSc.IntType) idx = 0 for iRecv in np.arange(Istart_receivers, Iend_receivers): # Get data of iRecv temp = np.asarray(receivers.getRow(iRecv)) dataRecv[:] = np.real(temp[1, 19:25]) # Edge-indexes for iRecv edgesIdxRecv[idx, :] = (dataRecv).astype(PETSc.IntType) idx += 1 # Gather global solution of x to local vector # Sequential vector for gather tasks x_local = createSequentialVector(edgeOrder*nReceiversLocal, communicator=None) # Build Index set in PETSc format IS_edges = PETSc.IS().createGeneral(edgesIdxRecv.flatten(), comm=PETSc.COMM_WORLD) # Build gather vector gatherVector = PETSc.Scatter().create(x, IS_edges, x_local, None) # Ghater values gatherVector.scatter(x, x_local, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.FORWARD) # Post-processing electric fields # Create parallel structures EpX = createParallelVector(nReceivers, communicator=None) EpY = createParallelVector(nReceivers, communicator=None) EpZ = createParallelVector(nReceivers, communicator=None) EsX = createParallelVector(nReceivers, communicator=None) EsY = createParallelVector(nReceivers, communicator=None) EsZ = createParallelVector(nReceivers, communicator=None) EtX = createParallelVector(nReceivers, communicator=None) EtY = createParallelVector(nReceivers, communicator=None) EtZ = createParallelVector(nReceivers, communicator=None) EpDense = createParallelDenseMatrix(nReceivers, numDimensions, communicator=None) EsDense = createParallelDenseMatrix(nReceivers, numDimensions, communicator=None) EtDense = createParallelDenseMatrix(nReceivers, numDimensions, communicator=None) # Reshape auxiliar array dataRecv = np.zeros(numDimensions+nodalOrder*numDimensions+nodalOrder, dtype=np.float) # Compute fields for all local receivers idx = 0 for iRecv in np.arange(Istart_receivers, Iend_receivers): # Get data of iRecv temp = np.asarray(receivers.getRow(iRecv)) dataRecv[:] = np.real(temp[1, 0:19]) # Receivers coordinates coordReceiver = dataRecv[0:3] # Element coordinates coordElement = dataRecv[3:15] # Nodal-indexes nodesElement = (dataRecv[15:19]).astype(PETSc.IntType) # Compute fields [EpRecv, EsRecv, EtRecv] = computeFieldsReceiver(modelling, coordReceiver, coordElement, nodesElement, x_local[idx * edgeOrder: (idx * edgeOrder) + edgeOrder], edgeOrder, numDimensions) idx += 1 # Set primary field components EpX.setValue(iRecv, EpRecv[0], addv=PETSc.InsertMode.INSERT_VALUES) EpY.setValue(iRecv, EpRecv[1], addv=PETSc.InsertMode.INSERT_VALUES) EpZ.setValue(iRecv, EpRecv[2], addv=PETSc.InsertMode.INSERT_VALUES) EpDense.setValue(iRecv, 0, EpRecv[0], addv=PETSc.InsertMode.INSERT_VALUES) EpDense.setValue(iRecv, 1, EpRecv[1], addv=PETSc.InsertMode.INSERT_VALUES) EpDense.setValue(iRecv, 2, EpRecv[2], addv=PETSc.InsertMode.INSERT_VALUES) # Set secondary field components EsX.setValue(iRecv, EsRecv[0], addv=PETSc.InsertMode.INSERT_VALUES) EsY.setValue(iRecv, EsRecv[1], addv=PETSc.InsertMode.INSERT_VALUES) EsZ.setValue(iRecv, EsRecv[2], addv=PETSc.InsertMode.INSERT_VALUES) EsDense.setValue(iRecv, 0, EsRecv[0], addv=PETSc.InsertMode.INSERT_VALUES) EsDense.setValue(iRecv, 1, EsRecv[1], addv=PETSc.InsertMode.INSERT_VALUES) EsDense.setValue(iRecv, 2, EsRecv[2], addv=PETSc.InsertMode.INSERT_VALUES) # Set total field components EtX.setValue(iRecv, EtRecv[0], addv=PETSc.InsertMode.INSERT_VALUES) EtY.setValue(iRecv, EtRecv[1], addv=PETSc.InsertMode.INSERT_VALUES) EtZ.setValue(iRecv, EtRecv[2], addv=PETSc.InsertMode.INSERT_VALUES) EtDense.setValue(iRecv, 0, EtRecv[0], addv=PETSc.InsertMode.INSERT_VALUES) EtDense.setValue(iRecv, 1, EtRecv[1], addv=PETSc.InsertMode.INSERT_VALUES) EtDense.setValue(iRecv, 2, EtRecv[2], addv=PETSc.InsertMode.INSERT_VALUES) # Start global vector assembly EpX.assemblyBegin(), EpY.assemblyBegin(), EpZ.assemblyBegin() EsX.assemblyBegin(), EsY.assemblyBegin(), EsZ.assemblyBegin() EtX.assemblyBegin(), EtY.assemblyBegin(), EtZ.assemblyBegin() EpDense.assemblyBegin(), EsDense.assemblyBegin(), EtDense.assemblyBegin() # End global vector assembly EpX.assemblyEnd(), EpY.assemblyEnd(), EpZ.assemblyEnd() EsX.assemblyEnd(), EsY.assemblyEnd(), EsZ.assemblyEnd() EtX.assemblyEnd(), EtY.assemblyEnd(), EtZ.assemblyEnd() EpDense.assemblyEnd(), EsDense.assemblyEnd(), EtDense.assemblyEnd() # Verify if directory exists MASTER = 0 if rank == MASTER: checkIfDirectoryExist(modelling['DIR_NAME'] + '/Output/Petsc') checkIfDirectoryExist(modelling['DIR_NAME'] + '/Output/Ascii') checkIfDirectoryExist(modelling['DIR_NAME'] + '/Output/Matlab') # Print PETSc.Sys.Print(' Saving output:') # Export electric fields (petsc format) printMessage(' Petsc format', rank) # Save primary electric field writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EpX.dat', EpX, communicator=None) writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EpY.dat', EpY, communicator=None) writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EpZ.dat', EpZ, communicator=None) writeDenseMatrix(modelling['DIR_NAME'] + '/Output/Petsc/Ep.dat', EpDense, communicator=None) # Save secondary electric field writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EsX.dat', EsX, communicator=None) writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EsY.dat', EsY, communicator=None) writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EsZ.dat', EsZ, communicator=None) writeDenseMatrix(modelling['DIR_NAME'] + '/Output/Petsc/Es.dat', EsDense, communicator=None) # Save total electric field writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EtX.dat', EtX, communicator=None) writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EtY.dat', EtY, communicator=None) writePetscVector(modelling['DIR_NAME'] + '/Output/Petsc/EtZ.dat', EtZ, communicator=None) writeDenseMatrix(modelling['DIR_NAME'] + '/Output/Petsc/Et.dat', EtDense, communicator=None) # Export electric fields (Ascii and Matlab format) if rank == MASTER: # Export electric fields (Ascii format) # Save primary electric field printMessage(' Ascii format', rank) dataEp = exportPetscToAscii(nReceivers, modelling['DIR_NAME'] + '/Output/Petsc/EpX.dat', modelling['DIR_NAME'] + '/Output/Petsc/EpY.dat', modelling['DIR_NAME'] + '/Output/Petsc/EpZ.dat', modelling['DIR_NAME'] + '/Output/Ascii/Ep.dat') # Save secondary electric field dataEs = exportPetscToAscii(nReceivers, modelling['DIR_NAME'] + '/Output/Petsc/EsX.dat', modelling['DIR_NAME'] + '/Output/Petsc/EsY.dat', modelling['DIR_NAME'] + '/Output/Petsc/EsZ.dat', modelling['DIR_NAME'] + '/Output/Ascii/Es.dat') # Save total electric field dataEt = exportPetscToAscii(nReceivers, modelling['DIR_NAME'] + '/Output/Petsc/EtX.dat', modelling['DIR_NAME'] + '/Output/Petsc/EtY.dat', modelling['DIR_NAME'] + '/Output/Petsc/EtZ.dat', modelling['DIR_NAME'] + '/Output/Ascii/Et.dat') # Export electric fields (Matlab format) printMessage(' Matlab format', rank) # Save primary electric field exportNumpytoMatlab(dataEp, modelling['DIR_NAME'] + '/Output/Matlab/Ep.mat', electricField='Primary') # Save secondary electric field exportNumpytoMatlab(dataEs, modelling['DIR_NAME'] + '/Output/Matlab/Es.mat', electricField='Secondary') # Save total electric field exportNumpytoMatlab(dataEt, modelling['DIR_NAME'] + '/Output/Matlab/Et.mat', electricField='Total') # Remove temporal files (petsc) filesToDelete = [modelling['DIR_NAME'] + '/Output/Petsc/EpX.dat', modelling['DIR_NAME'] + '/Output/Petsc/EpY.dat', modelling['DIR_NAME'] + '/Output/Petsc/EpZ.dat', modelling['DIR_NAME'] + '/Output/Petsc/EsX.dat', modelling['DIR_NAME'] + '/Output/Petsc/EsY.dat', modelling['DIR_NAME'] + '/Output/Petsc/EsZ.dat', modelling['DIR_NAME'] + '/Output/Petsc/EtX.dat', modelling['DIR_NAME'] + '/Output/Petsc/EtY.dat', modelling['DIR_NAME'] + '/Output/Petsc/EtZ.dat', modelling['DIR_NAME'] + '/Output/Petsc/EpX.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EpY.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EpZ.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EsX.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EsY.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EsZ.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EtX.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EtY.dat.info', modelling['DIR_NAME'] + '/Output/Petsc/EtZ.dat.info'] for iFile in np.arange(len(filesToDelete)): removeFile(filesToDelete[iFile]) # End timer End_postprocessing = getTime() # Elapsed time in assembly elapsedTimepostprocessing = End_postprocessing-Init_postprocessing return elapsedTimepostprocessing