def load(self, vec, filename="control.dat"): """ Load a vector from a file. DumbCheckpoint requires that the mesh, FunctionSpace and parallel decomposition are identical between store and load. """ viewer = PETSc.Viewer().createBinary(filename, mode="r") vec.vec_wo().load(viewer)
def store(self, vec, filename="control.dat"): """ Store the vector to a file to be reused in a later computation. DumbCheckpoint requires that the mesh, FunctionSpace and parallel decomposition are identical between store and load. """ viewer = PETSc.Viewer().createBinary(filename, mode="w") viewer.view(vec.vec_ro())
def save_mesh(mesh, fname, fpath='.'): """ :arg mesh: mesh to be saved in DMPlex format. :kwarg fname: file name (without '.h5' extension). :arg fpath: directory to store the file. """ if COMM_WORLD.size > 1: raise IOError("Saving a mesh to HDF5 only works in serial.") try: plex = mesh.topology_dm except AttributeError: plex = mesh._topology_dm # Backwards compatability viewer = PETSc.Viewer().createHDF5(os.path.join(fpath, fname + '.h5'), 'w') viewer(plex)
def export_final_state(inputdir, bathymetry_2d): """ Export fields to be used in a subsequent simulation """ if not os.path.exists(inputdir): os.makedirs(inputdir) print_output("Exporting fields for subsequent simulation") chk = DumbCheckpoint(inputdir + "/bathymetry", mode=FILE_CREATE) chk.store(bathymetry_2d, name="bathymetry") File(inputdir + '/bathout.pvd').write(bathymetry_2d) chk.close() plex = bathymetry_2d.function_space().mesh()._topology_dm viewer = PETSc.Viewer().createHDF5(inputdir + '/myplex.h5', 'w') viewer(plex)
def _from_gmsh(filename): """Read a Gmsh .msh file from `filename`""" # Create a read-only PETSc.Viewer gmsh_viewer = PETSc.Viewer().create() gmsh_viewer.setType("ascii") gmsh_viewer.setFileMode("r") gmsh_viewer.setFileName(filename) gmsh_plex = PETSc.DMPlex().createGmsh(gmsh_viewer) if gmsh_plex.hasLabel("Face Sets"): boundary_ids = gmsh_plex.getLabelIdIS("Face Sets").getIndices() gmsh_plex.createLabel("boundary_ids") for bid in boundary_ids: faces = gmsh_plex.getStratumIS("Face Sets", bid).getIndices() for f in faces: gmsh_plex.setLabelValue("boundary_ids", f, bid) return gmsh_plex
def export_final_state( inputdir, uv, elev, ): """ Export fields to be used in a subsequent simulation """ if not os.path.exists(inputdir): os.makedirs(inputdir) print_output("Exporting fields for subsequent simulation") chk = DumbCheckpoint(inputdir + "/velocity", mode=FILE_CREATE) chk.store(uv, name="velocity") File(inputdir + '/velocityout.pvd').write(uv) chk.close() chk = DumbCheckpoint(inputdir + "/elevation", mode=FILE_CREATE) chk.store(elev, name="elevation") File(inputdir + '/elevationout.pvd').write(elev) chk.close() plex = elev.function_space().mesh()._topology_dm viewer = PETSc.Viewer().createHDF5(inputdir + '/myplex.h5', 'w') viewer(plex)
def fixed_point_iteration(self, **parsed_args): """ Apply a goal-oriented metric-based mesh adaptation fixed point iteration loop for a tidal farm modelling problem. """ parsed_args = AttrDict(parsed_args) options = self.options expected = { "miniter", "maxiter", "load_index", "qoi_rtol", "element_rtol", "error_indicator", "approach", "h_min", "h_max", "turbine_h_min", "turbine_h_max", "a_max", "target_complexity", "base_complexity", "flux_form", "norm_order", "no_final_run", } if not expected.issubset(set(parsed_args.keys())): missing = expected.difference(set(parsed_args.keys())) raise AttributeError(f"Missing required arguments {missing}") output_dir = options.output_directory end_time = options.simulation_end_time dt = options.timestep approach = parsed_args.approach h_min = parsed_args.h_min h_max = parsed_args.h_max turbine_h_min = parsed_args.turbine_h_min turbine_h_max = parsed_args.turbine_h_max a_max = parsed_args.a_max num_timesteps = int(np.round(end_time / dt)) target = num_timesteps * parsed_args.target_complexity base = num_timesteps * parsed_args.base_complexity num_subintervals = self.num_subintervals timesteps = [dt] * num_subintervals p = parsed_args.norm_order no_final_run = parsed_args.no_final_run if COMM_WORLD.size > 1: raise NotImplementedError( "Mmg2d only supports serial mesh adaptation") # Enter fixed point iteration miniter = parsed_args.miniter maxiter = parsed_args.maxiter if miniter > maxiter: print_output( f"miniter {miniter} and maxiter {maxiter} are incompatible") miniter = maxiter print_output(f"Setting miniter={miniter}, maxiter={maxiter}") qoi_rtol = parsed_args.qoi_rtol element_rtol = parsed_args.element_rtol converged_reason = None load_index = parsed_args.load_index if load_index > 0: self.keep_log = True fp_iteration = load_index msg = "Termination due to {:s} after {:d} iterations" cells = [] if load_index == 0 else list( np.load(f"{output_dir}/num_cells_progress.npy")) qois = [] if load_index == 0 else list( np.load(f"{output_dir}/J_progress.npy")) self.J = 0 if load_index == 0 else qois[-1] def check_cell_count_convergence(): if len(cells) >= max(2, miniter): elements_converged = True for nc, nc_old in zip(cells[-1], cells[-2]): if abs(nc - nc_old) > element_rtol * nc_old: elements_converged = False break if elements_converged: return "converged element counts" def check_qoi_convergence(): if len(qois) >= max(2, miniter): if abs(qois[-1] - qois[-2]) < qoi_rtol * qois[-2]: return "converged quantity of interest" # Check for convergence (of loaded data) converged_reason = check_qoi_convergence( ) or check_cell_count_convergence() # Load meshes, if requested if load_index > 0: for i in range(num_subintervals): fname = f"{output_dir}/mesh_fp{fp_iteration}_{i}" if not os.path.exists(fname + ".h5"): raise IOError(f"Cannot load mesh file {fname}.") print_output(f"\n--- Loading plex {i+1}\n{fname}") plex = PETSc.DMPlex().create() plex.createFromFile(fname + ".h5") self.meshes[i] = Mesh(plex) # Do final run, if loaded data has already converged if converged_reason is not None: if not no_final_run: print(f"converged_reason: {converged_reason}") self._final_run() print_output(msg.format(converged_reason, fp_iteration + 1)) print_output(f"Energy output: {self.J/3.6e+09} MWh") return # Enter the fixed point iteration loop while fp_iteration <= maxiter: print_output( f"Start time for fp_iteration {fp_iteration}: {datetime.datetime.now()}" ) outfiles = AttrDict({}) if fp_iteration < miniter: converged_reason = None elif fp_iteration == maxiter: converged_reason = converged_reason or "maximum number of iterations reached" # Ramp up the target complexity target_ramp = ramp_complexity(base, target, fp_iteration) # Create metrics kw = dict(metric_parameters=dict(dm_plex_metric_verbosity=10)) metrics = [RiemannianMetric(mesh, **kw) for mesh in self.meshes] metric_fns = [metric.function for metric in metrics] # Load metric data, if available loaded = False if fp_iteration == load_index: for i, metric in enumerate(metric_fns): fpath = self.root_dir if load_index == 0 else output_dir ext = "" if load_index == 0 else "_fp{fp_iteration}" fname = f"{fpath}/metric{i}{ext}" if os.path.exists(fname + ".h5"): print_output( f"\n--- Loading metric on mesh {i+1}\n{fname}") try: with DumbCheckpoint(fname, mode=FILE_READ) as chk: chk.load(metric, name="Metric") loaded = True except Exception: raise IOError( f"Cannot load metric data on mesh {i+1}") elif loaded: raise IOError("Remove partial metric data") if not loaded: # Solve forward and adjoint on each subinterval if converged_reason is None: print_output( f"\n--- Forward-adjoint sweep {fp_iteration}\n") solutions = self.solve_adjoint() else: if not no_final_run: print(f"converged_reason: {converged_reason}") self._final_run() # Check for QoI convergence converged_reason = converged_reason or check_qoi_convergence() if converged_reason is not None: if not no_final_run: print(f"converged_reason: {converged_reason}") self._final_run() qois.append(self.J) np.save(f"{output_dir}/J_progress.npy", qois) # Escape if converged if converged_reason is not None: print_output(msg.format(converged_reason, fp_iteration + 1)) print_output(f"Energy output: {self.J/3.6e+09} MWh") break # Create vtu output files outfiles.forward = File(f"{output_dir}/Forward2d.pvd") outfiles.forward_old = File(f"{output_dir}/ForwardOld2d.pvd") outfiles.adjoint_next = File(f"{output_dir}/AdjointNext2d.pvd") outfiles.adjoint = File(f"{output_dir}/Adjoint2d.pvd") # Construct metric with pyadjoint.stop_annotating(): print_output(f"\n--- Error estimation {fp_iteration}\n") for i, mesh in enumerate(self.meshes): options.rebuild_mesh_dependent_components(mesh) options.get_bnd_conditions( self.function_spaces.swe2d[i]) update_forcings = options.update_forcings # Create error estimator ee = ErrorEstimator( options, mesh=mesh, metric=approach, error_estimator=parsed_args.error_indicator, ) # Loop over all exported timesteps N = len(solutions.swe2d.adjoint[i]) for j in range(N): if i < num_subintervals - 1 and j == N - 1: continue # Plot fields args = [] for f in outfiles: args.extend(solutions.swe2d[f][i][j].split()) name = "adjoint " if "adjoint" in f else "" args[-2].rename( (name + "velocity").capitalize()) args[-1].rename( (name + "elevation").capitalize()) outfiles[f].write(*args[-2:]) # Construct metric at current timestep t = i * end_time / num_subintervals + dt * (j + 1) update_forcings(t) metric_step = ee.metric(*args, **parsed_args) # Apply trapezium rule metric_step *= 0.5 * dt if j in (0, N - 1) else dt metric_fns[i] += metric_step # Stash metric data print_output( f"\n--- Storing metric data on mesh {i+1}\n") fname = f"{output_dir}/metric{i}_fp{fp_iteration}" with DumbCheckpoint(fname, mode=FILE_CREATE) as chk: chk.store(metric_fns[i], name="Metric") if fp_iteration == 0: fname = f"{self.root_dir}/metric{i}" with DumbCheckpoint(fname, mode=FILE_CREATE) as chk: chk.store(metric_fns[i], name="Metric") # Apply space-time normalisation print_output(f"\n--- Metric processing {fp_iteration}\n") space_time_normalise(metric_fns, end_time, timesteps, target_ramp, p) # Enforce element constraints, accounting for turbines hmins = [] hmaxs = [] for mesh in self.meshes: P0 = get_functionspace(mesh, "DG", 0) hmin = Function(P0).assign(h_min) hmax = Function(P0).assign(h_max) for tag in self.qoi_farm_ids.flatten(): hmin.assign(turbine_h_min, subset=mesh.cell_subset(tag)) hmax.assign(turbine_h_max, subset=mesh.cell_subset(tag)) hmins.append(hmin) hmaxs.append(hmax) enforce_element_constraints(metric_fns, hmins, hmaxs, a_max) # Plot metrics outfiles.metric = File(f"{output_dir}/Metric2d.pvd") for metric in metric_fns: outfiles.metric.write(metric) # Adapt meshes print_output(f"\n--- Mesh adaptation {fp_iteration}\n") outfiles.mesh = File(f"{output_dir}/Mesh2d.pvd") for i, metric in enumerate(metrics): self.meshes[i] = Mesh(adapt(self.meshes[i], metric)) outfiles.mesh.write(self.meshes[i].coordinates) cells.append([mesh.num_cells() for mesh in self.meshes]) np.save(f"{output_dir}/num_cells_progress.npy", cells) # Check for convergence of element count check_cell_count_convergence() # Save mesh data to disk for i, mesh in enumerate(self.meshes): fname = f"{output_dir}/mesh_fp{fp_iteration+1}_{i}.h5" viewer = PETSc.Viewer().createHDF5(fname, "w") viewer(mesh.topology_dm) # Increment print_output( f"End time for fp_iteration {fp_iteration}: {datetime.datetime.now()}" ) fp_iteration += 1 print_output(msg.format(converged_reason, fp_iteration + 1)) print_output(f"Energy output: {self.J/3.6e+09} MWh")
def output_time(start, end, **kwargs): """ Used by ``explosive_source.py`` at the end of a run to record to file useful information. """ verbose = kwargs.get('verbose', False) tofile = kwargs.get('tofile', False) meshid = kwargs.get('meshid', 'default_mesh') ntimesteps = kwargs.get('ntimesteps', 0) nloops = kwargs.get('nloops', 0) tile_size = kwargs.get('tile_size', 0) partitioning = kwargs.get('partitioning', 'chunk') extra_halo = 'yes' if kwargs.get('extra_halo', False) else 'no' explicit_mode = kwargs.get('explicit_mode', None) glb_maps = 'yes' if kwargs.get('glb_maps', False) else 'no' poly_order = kwargs.get('poly_order', -1) domain = kwargs.get('domain', 'default_domain') coloring = kwargs.get('coloring', 'default') prefetch = 'yes' if kwargs.get('prefetch', False) else 'no' function_spaces = kwargs.get('function_spaces', []) backend = os.environ.get("SLOPE_BACKEND", "SEQUENTIAL") name = os.path.splitext(os.path.basename(sys.argv[0]))[0] # Cut away the extension avg = lambda v: (sum(v) / len(v)) if v else 0.0 # Where do I store the output ? output_dir = os.getcwd() # Find number of processes, and number of threads per process rank = MPI.COMM_WORLD.rank num_procs = MPI.COMM_WORLD.size num_threads = int(os.environ.get("OMP_NUM_THREADS", 1)) if backend == 'OMP' else 1 # What execution mode is this? if num_procs == 1 and num_threads == 1: versions = ['sequential', 'openmp', 'mpi', 'mpi_openmp'] elif num_procs == 1 and num_threads > 1: versions = ['openmp'] elif num_procs > 1 and num_threads == 1: versions = ['mpi'] else: versions = ['mpi_openmp'] # Determine the total execution time (Python + kernel execution + MPI cost if rank in range(1, num_procs): MPI.COMM_WORLD.isend([start, end], dest=0) elif rank == 0: starts, ends = [0]*num_procs, [0]*num_procs starts[0], ends[0] = start, end for i in range(1, num_procs): starts[i], ends[i] = MPI.COMM_WORLD.recv(source=i) min_start, max_end = min(starts), max(ends) tot = round(max_end - min_start, 3) print "Time stepping: ", tot, "s" # Exit if user doesn't want timings to be recorded if not tofile: return # Determine (on rank 0): # ACT - Average Compute Time, pure kernel execution - # ACCT - Average Compute and Communication Time (ACS + MPI cost) # For this, first dump PETSc performance log info to temporary file as # currently there's no other clean way of accessing the times in petsc4py logfile = os.path.join(output_dir, 'seigenlog.py') vwr = PETSc.Viewer().createASCII(logfile) vwr.pushFormat(PETSc.Viewer().Format().ASCII_INFO_DETAIL) PETSc.Log().view(vwr) PETSc.Options().delValue('log_view') if rank == 0: with open(logfile, 'r') as f: content = f.read() exec(content) in globals(), locals() compute_times = [Stages['Main Stage']['ParLoopCKernel'][i]['time'] for i in range(num_procs)] mpi_times = [Stages['Main Stage']['ParLoopHaloEnd'][i]['time'] for i in range(num_procs)] ACT = round(avg(compute_times), 3) AMT = round(avg(mpi_times), 3) ACCT = ACT + AMT print "Average Compute Time: ", ACT, "s" print "Average Compute and Communication Time: ", ACCT, "s" # Determine if a multi-node execution platform = os.environ.get('NODENAME', 'unknown') # Adjust /tile_size/ and /version/ based on the problem that was actually run assert nloops >= 0 if nloops == 0: tile_size = 0 mode = "untiled" elif explicit_mode: mode = "fs%d" % explicit_mode else: mode = "loops%d" % nloops ### Print timings to file ### def fix(values): new_values = [] for v in values: try: new_v = int(v) except ValueError: try: new_v = float(v) except ValueError: new_v = v.strip() if new_v != '': new_values.append(new_v) return tuple(new_values) if rank == 0: for version in versions: timefile = os.path.join(output_dir, "times", name, "poly_%d" % poly_order, domain, meshid, version, platform, "np%d_nt%d.txt" % (num_procs, num_threads)) # Create directory and file (if not exist) if not os.path.exists(os.path.dirname(timefile)): os.makedirs(os.path.dirname(timefile)) if not os.path.exists(timefile): open(timefile, 'a').close() # Read the old content, add the new time value, order # everything based on <execution time, #loops tiled>, write # back to the file (overwriting existing content) with open(timefile, "r+") as f: lines = [line.split('|') for line in f if line.strip()][2:] lines = [fix(i) for i in lines] lines += [(tot, ACT, ACCT, ntimesteps, mode, tile_size, partitioning, extra_halo, glb_maps, coloring, prefetch)] lines.sort(key=lambda x: x[0]) template = "| " + "%9s | " * 11 prepend = template % ('time', 'ACT', 'ACCT', 'timesteps', 'mode', 'tilesize', 'partmode', 'extrahalo', 'glbmaps', 'coloring', 'prefetch') lines = "\n".join([prepend, '-'*133] + [template % i for i in lines]) + "\n" f.seek(0) f.write(lines) f.truncate() ### Print DoFs summary to file ### dofsfile = os.path.join(output_dir, "times", name, "dofs_summary.txt") if rank == 0 and not os.path.exists(dofsfile): with open(dofsfile, 'a') as f: f.write("poly:numprocs:[fs1_dofs;fs2_dofs;...]\n") tot_dofs = [MPI.COMM_WORLD.allreduce(fs.dof_count, op=mpi4py.MPI.SUM) for fs in function_spaces] if rank == 0: with open(dofsfile, "a") as f: f.write("%d:%d:%s\n" % (poly_order, num_procs, ';'.join([str(i) for i in tot_dofs]))) ### Print summary output to screen ### if rank == 0 and verbose: for i in range(num_procs): fs_info = ", ".join(["%s=%d" % (fs.name, fs.dof_count) for fs in function_spaces]) tot_time = compute_times[i] + mpi_times[i] offC = (ends[i] - starts[i]) - tot_time offCperc = (offC / (ends[i] - starts[i]))*100 mpiPerc = (mpi_times[i] / (ends[i] - starts[i]))*100 print "Rank %d: comp=%.2fs, mpi=%.2fs -- tot=%.2fs (py=%.2fs, %.2f%%; mpi_oh=%.2f%%; fs=[%s])" % \ (i, compute_times[i], mpi_times[i], tot_time, offC, offCperc, mpiPerc, fs_info) sys.stdout.flush() MPI.COMM_WORLD.barrier() # Clean up if rank == 0: os.remove(logfile)
solver_params = { 'pyamg_tol': pyamg_tol, 'pyamg_maxiter': pyamg_maxiter, 'ksp_monitor': None, } solver = vs.LinearVariationalSolver(problem, solver_parameters=solver_params) # Build matrix and store text file A = assemble(a).M.handle store_mat = True if store_mat: dim = mesh.geometric_dimension() file_name = f"matrix_txt_files/helmholtz-{dim}D-h{h}.txt" from firedrake.petsc import PETSc myviewer = PETSc.Viewer().createASCII( file_name, mode=PETSc.Viewer.Format.ASCII_COMMON, comm=PETSc.COMM_WORLD) A.view(myviewer) # set up pyamg preconditioner and solve pc = solver.snes.getKSP().pc pc.setType(pc.Type.PYTHON) pc.setPythonContext( AMGTransmissionPreconditioner(wave_number, fspace, A, tol=pyamg_tol, maxiter=pyamg_maxiter, use_plane_waves=True)) solver.solve()