def semi_implicit_step(self): state = self.state dt = state.timestepping.dt alpha = state.timestepping.alpha with timed_stage("Apply forcing terms"): self.forcing.apply((1-alpha)*dt, state.xn, state.xn, state.xstar, implicit=False) for k in range(state.timestepping.maxk): with timed_stage("Advection"): for name, advection in self.active_advection: # first computes ubar from state.xn and state.xnp1 advection.update_ubar(state.xn, state.xnp1, alpha) # advects a field from xstar and puts result in xp advection.apply(self.xstar_fields[name], self.xp_fields[name]) state.xrhs.assign(0.) # xrhs is the residual which goes in the linear solve for i in range(state.timestepping.maxi): with timed_stage("Apply forcing terms"): self.forcing.apply(alpha*dt, state.xp, state.xnp1, state.xrhs, implicit=True, incompressible=self.incompressible) state.xrhs -= state.xnp1 with timed_stage("Implicit solve"): self.linear_solver.solve() # solves linear system and places result in state.dy state.xnp1 += state.dy self._apply_bcs()
def solve(self, r_u, r_p, r_b): self._ru.assign(r_u) self._rp.assign(r_p) self._rb.assign(r_b) # Fields for solution self._u.assign(0.0) self._p.assign(0.0) self._b.assign(0.0) # initialize solver fields self._up.assign(0.0) self._btmp.assign(0.0) with timed_stage('UP Solver'): self._up_solver.solve() self._u.assign(self._up.sub(0)) self._p.assign(self._up.sub(1)) with timed_stage('B Solver'): self._b_solver.solve() self._b.assign(assemble(self._rb - self._dt_half_N2 * self._btmp)) return self._u, self._p, self._b
def picard_step(self): self.solution_lag.assign(self.solution) # solve for self.u_star with timed_stage("momentum_solve"): self.predictor_solver.solve() # pressure correction solve, solves for final solution (corrected velocity and pressure) with timed_stage("correction_solve"): self.solver.solve()
def run(self, t, tmax, pickup=False): """ This is the timeloop. After completing the semi implicit step any passively advected fields are updated, implicit diffusion and physics updates are applied (if required). """ state = self.state t = self.setup_timeloop(state, t, tmax, pickup) dt = state.timestepping.dt while t < tmax - 0.5*dt: logger.info("at start of timestep, t=%s, dt=%s" % (t, dt)) t += dt state.t.assign(t) state.xnp1.assign(state.xn) for name, evaluation in self.prescribed_fields: state.fields(name).project(evaluation(t)) self.semi_implicit_step() for name, advection in self.passive_advection: field = getattr(state.fields, name) # first computes ubar from state.xn and state.xnp1 advection.update_ubar(state.xn, state.xnp1, state.timestepping.alpha) # advects a field from xn and puts result in xnp1 advection.apply(field, field) state.xb.assign(state.xn) state.xn.assign(state.xnp1) with timed_stage("Diffusion"): for name, diffusion in self.diffused_fields: field = getattr(state.fields, name) diffusion.apply(field, field) with timed_stage("Physics"): for physics in self.physics_list: physics.apply() with timed_stage("Dump output"): state.dump(t) if state.output.checkpoint: state.chkpt.close() logger.info("TIMELOOP complete. t=%s, tmax=%s" % (t, tmax))
def warmup(self): self.initialize() un, Dn = self.state up, Dp = self.updates up.assign(un) Dp.assign(Dn) with timed_stage("Warm up: DU Residuals"): self.Dsolver.solve() self.Usolver.solve() with timed_stage("Warm up: Linear solve"): self.DUsolver.solve()
def run(self, t, tmax, pickup=False): """ This is the timeloop. After completing the semi implicit step any passively transported fields are updated, implicit diffusion and physics updates are applied (if required). """ state = self.state if pickup: t = state.pickup_from_checkpoint() state.setup_diagnostics() with timed_stage("Dump output"): state.setup_dump(t, tmax, pickup) state.t.assign(t) self.x.initialise(state) while float(state.t) < tmax - 0.5*float(state.dt): logger.info(f'at start of timestep, t={float(state.t)}, dt={float(state.dt)}') self.x.update() self.timestep() for field in self.x.np1: state.fields(field.name()).assign(field) with timed_stage("Physics"): for physics in self.physics_list: physics.apply() # TODO: Hack to ensure that xnp1 fields are updated for field in self.x.np1: field.assign(state.fields(field.name())) state.t.assign(state.t + state.dt) with timed_stage("Dump output"): state.dump(float(state.t)) if state.output.checkpoint: state.chkpt.close() logger.info(f'TIMELOOP complete. t={float(state.t)}, tmax={tmax}')
def timestep(self): xn = self.x.n xnp1 = self.x.np1 xstar = self.x.star xp = self.x.p xrhs = self.xrhs dy = self.dy with timed_stage("Apply forcing terms"): self.forcing.apply(xn, xn, xstar(self.field_name), "explicit") xp(self.field_name).assign(xstar(self.field_name)) for k in range(self.maxk): with timed_stage("Transport"): for name, scheme in self.active_transport: # transports a field from xstar and puts result in xp scheme.apply(xstar(name), xp(name)) xrhs.assign(0.) # xrhs is the residual which goes in the linear solve for i in range(self.maxi): with timed_stage("Apply forcing terms"): self.forcing.apply(xp, xnp1, xrhs, "implicit") xrhs -= xnp1(self.field_name) with timed_stage("Implicit solve"): self.linear_solver.solve(xrhs, dy) # solves linear system and places result in state.dy xnp1X = xnp1(self.field_name) xnp1X += dy # Update xnp1 values for active tracers not included in the linear solve self.copy_active_tracers(xp, xnp1) self._apply_bcs() for name, scheme in self.auxiliary_schemes: # transports a field from xn and puts result in xnp1 scheme.apply(xn(name), xnp1(name)) with timed_stage("Diffusion"): for name, scheme in self.diffusion_schemes: scheme.apply(xnp1(name), xnp1(name))
def warmup(self): """Warm up solver by taking one time-step.""" self._initialize() un, pn, bn = self._state.split() self._up.assign(0.0) with timed_stage("Warmup: Velocity-Pressure-Solve"): self.up_solver.solve() un.assign(self._up.sub(0)) pn.assign(self._up.sub(1)) self._btmp.assign(0.0) with timed_stage("Warmup: Buoyancy-Solve"): self.b_solver.solve() bn.assign(assemble(bn - self._dt_half_N2 * self._btmp))
def setup_timeloop(self, t, tmax, pickup): """ Setup the timeloop by setting up diagnostics, dumping the fields and picking up from a previous run, if required """ self.state.setup_diagnostics() with timed_stage("Dump output"): self.state.setup_dump(tmax, pickup) t = self.state.dump(t, pickup) return t
def solve(self): with timed_stage('copy_3d_to_2d'): # execute par loop op2.par_loop(self.kernel, self.fs_3d.mesh().cell_set, self.output_2d.dat(op2.WRITE, self.fs_2d.cell_node_map()), self.input_3d.dat(op2.READ, self.fs_3d.cell_node_map()), self.idx(op2.READ), iterate=self.iter_domain) if self.do_rt_scaling: self.rt_scale_solver.solve()
def warmup(self): state = self.state un, pn, bn = state.split() un.assign(self.u0) pn.assign(self.p0) bn.assign(self.b0) with timed_stage("Warm up: Solver"): un1, pn1, bn1 = self.gravity_wave_solver.solve(un, pn, bn)
def solve(self): with timed_stage('constant_3d'): # execute par loop op2.par_loop(self.kernel, self.fs_3d.mesh().cell_set, self.output_3d.dat(op2.WRITE, self.fs_3d.cell_node_map()), self.input_3d.dat(op2.READ, self.fs_3d.cell_node_map()), self.idx(op2.READ), iterate=self.iter_domain)
def run_simulation(self, tmax, write=False, dumpfreq=100): PETSc.Sys.Print(""" Running Skamarock and Klemp Gravity wave problem:\n method: %s,\n model degree: %d,\n refinements: %d,\n hybridization: %s,\n tmax: %s """ % (self.method, self.model_degree, self.refinement_level, self.hybridization, tmax)) state = self.state un, pn, bn = state.split() un.assign(self.u0) pn.assign(self.p0) bn.assign(self.b0) dumpcount = dumpfreq if write: dumpcount = self.write(dumpcount, dumpfreq) self.gravity_wave_solver._up_solver.snes.setConvergenceHistory() self.gravity_wave_solver._up_solver.snes.ksp.setConvergenceHistory() self.gravity_wave_solver._b_solver.snes.setConvergenceHistory() self.gravity_wave_solver._b_solver.snes.ksp.setConvergenceHistory() t = 0.0 while t < tmax - self.Dt / 2: t += self.Dt self.sim_time.append(t) un1, pn1, bn1 = self.gravity_wave_solver.solve(un, pn, bn) un.assign(un1) pn.assign(pn1) bn.assign(bn1) outer_ksp = self.gravity_wave_solver._up_solver.snes.ksp if self.hybridization: ctx = outer_ksp.getPC().getPythonContext() inner_ksp = ctx.trace_ksp else: ksps = outer_ksp.getPC().getFieldSplitSubKSP() _, inner_ksp = ksps # Collect ksp iterations self.ksp_outer_its.append(outer_ksp.getIterationNumber()) self.ksp_inner_its.append(inner_ksp.getIterationNumber()) if write: with timed_stage("Dump output"): dumpcount = self.write(dumpcount, dumpfreq)
def setup_timeloop(self, state, t, tmax, pickup): """ Setup the timeloop by setting up diagnostics, dumping the fields and picking up from a previous run, if required """ if pickup: t = state.pickup_from_checkpoint() state.setup_diagnostics() with timed_stage("Dump output"): state.setup_dump(t, tmax, pickup) return t
def solve(self): with timed_stage("copy_3d_to_2d"): # execute par loop op2.par_loop( self.kernel, self.fs_3d.mesh().cell_set, self.output_2d.dat(op2.WRITE, self.fs_2d.cell_node_map()), self.input_3d.dat(op2.READ, self.fs_3d.cell_node_map()), self.idx(op2.READ), iterate=self.iter_domain, ) if self.do_rt_scaling: self.rt_scale_solver.solve()
def apply(self, field): """ Applies the limiter on the given field (in place) :arg field: :class:`Function` to limit """ with timed_stage('limiter'): if self.is_vector: tmp_func = self.P1DG.get_work_function() fs = field.function_space() for i in range(fs.value_size): tmp_func.dat.data_with_halos[:] = field.dat.data_with_halos[:, i] super(VertexBasedP1DGLimiter, self).apply(tmp_func) field.dat.data_with_halos[:, i] = tmp_func.dat.data_with_halos[:] self.P1DG.restore_work_function(tmp_func) else: super(VertexBasedP1DGLimiter, self).apply(field)
def warmupSolve(level=2, mesh_arg=(10, 10)): ##Setup mesh = UnitSquareMesh(mesh_arg[0], mesh_arg[1]) nlevel = level mh = MeshHierarchy(mesh, nlevel) V = FunctionSpace(mh[-1], 'CG', 2) u = function.Function(V) f = function.Function(V) v = TestFunction(V) F = dot(grad(u), grad(v)) * dx - f * v * dx bcs = DirichletBC(V, 0.0, (1, 2, 3, 4)) x = SpatialCoordinate(V.mesh()) with offloading(cuda): with timed_stage("warmup"): f.interpolate(-0.5 * pi * pi * (4 * cos(pi * x[0]) - 5 * cos(pi * x[0] * 0.5) + 2) * sin(pi * x[1])) solve(F == 0, u, bcs=bcs, solver_parameters=parameters)
def apply(self, field): """ Applies the limiter on the given field (in place) :arg field: :class:`Function` to limit """ with timed_stage('limiter'): if self.is_vector: tmp_func = self.P1DG.get_work_function() fs = field.function_space() for i in range(fs.value_size): tmp_func.dat.data_with_halos[:] = field.dat.data_with_halos[:, i] #super(VertexBasedP1DGLimiter, self).apply(tmp_func) self.compute_bounds(tmp_func) self.apply_limiter(tmp_func) field.dat.data_with_halos[:, i] = tmp_func.dat.data_with_halos[:] self.P1DG.restore_work_function(tmp_func) else: #super(VertexBasedP1DGLimiter, self).apply(field) # is the same, but seems less confusing, WPan 2020-03-27 self.compute_bounds(field) self.apply_limiter(field)
def run(self, t, tmax, pickup=False): state = self.state state.xn.assign(state.x_init) xstar_fields = {name: func for (name, func) in zip(state.fieldlist, state.xstar.split())} xp_fields = {name: func for (name, func) in zip(state.fieldlist, state.xp.split())} dt = state.timestepping.dt alpha = state.timestepping.alpha if state.mu is not None: mu_alpha = [0., dt] else: mu_alpha = [None, None] with timed_stage("Dump output"): t = state.dump(t, pickup) while t < tmax + 0.5*dt: if state.output.Verbose: print "STEP", t, dt t += dt with timed_stage("Apply forcing terms"): self.forcing.apply((1-alpha)*dt, state.xn, state.xn, state.xstar, mu_alpha=mu_alpha[0]) state.xnp1.assign(state.xn) for k in range(state.timestepping.maxk): with timed_stage("Compute ubar"): self._set_ubar() # computes state.ubar from state.xn and state.xnp1 with timed_stage("Advection"): for field, advection in self.advection_dict.iteritems(): # advects a field from xstar and puts result in xp advection.apply(xstar_fields[field], xp_fields[field]) state.xrhs.assign(0.) # xrhs is the residual which goes in the linear solve for i in range(state.timestepping.maxi): with timed_stage("Apply forcing terms"): self.forcing.apply(alpha*dt, state.xp, state.xnp1, state.xrhs, mu_alpha=mu_alpha[1], incompressible=self.incompressible) state.xrhs -= state.xnp1 with timed_stage("Implicit solve"): self.linear_solver.solve() # solves linear system and places result in state.dy state.xnp1 += state.dy self._apply_bcs() state.xn.assign(state.xnp1) with timed_stage("Diffusion"): for name, diffusion in self.diffusion_dict.iteritems(): diffusion.apply(state.field_dict[name], state.field_dict[name]) with timed_stage("Dump output"): state.dump(t, pickup=False) state.diagnostic_dump() print "TIMELOOP complete. t= "+str(t-dt)+" tmax="+str(tmax)
def solve(self): with timed_stage("vert_integral"): self.solver.solve()
m, vp_fields, vp_coupling, dt, vp_bcs, solver_parameters=vp_solver_parameters, predictor_solver_parameters=predictor_solver_parameters, picard_iterations=1) # performs pseudo timestep to get good initial pressure # this is to avoid inconsistencies in terms (viscosity and advection) that # are meant to decouple from pressure projection, but won't if pressure is not initialised # do this here, so we can see the initial pressure in pressure_0.pvtu if not DUMP: # should not be done when picking up with timed_stage('initial_pressure'): vp_timestepper.initialize_pressure() #u_timestepper = DIRK33(u_eq, u, u_fields, dt, u_bcs, solver_parameters=u_solver_parameters) temp_timestepper = DIRK33(temp_eq, temp, temp_fields, dt, temp_bcs, solver_parameters=temp_solver_parameters) sal_timestepper = DIRK33(sal_eq, sal, sal_fields, dt, sal_bcs, solver_parameters=sal_solver_parameters)
depth_profile_to_csv(depth_profile1km, velocity_depth_profile1km, "1km", '0.0') depth_profile_to_csv(depth_profile2km, velocity_depth_profile2km, "2km", '0.0') depth_profile_to_csv(depth_profile4km, velocity_depth_profile4km, "4km", '0.0') depth_profile_to_csv(depth_profile6km, velocity_depth_profile6km, "6km", '0.0') ######## # Begin time stepping t = 0.0 step = 0 PROFILING = False if PROFILING: while t < T - 0.5 * dt: with timed_stage('velocity-pressure'): vp_timestepper.advance(t) u_timestepper.advance(t) with timed_stage('temperature'): temp_timestepper.advance(t) with timed_stage('salinity'): sal_timestepper.advance(t) step += 1 t += dt #with timed_region('output'): # Output files # if step % output_step == 0: else: while t < T - 0.5 * dt: vp_timestepper.advance(t)
def spcs(W, mesh, nue, bc, U_inf, t, dt, T, outfile, order, IP_stabilityparam_type=None, u_init=None, p_init=None, output=False): with PETSc.Log.Event("spcs configuration"): #functions and normal U = W.sub(0) P = W.sub(1) u, p = TrialFunctions(W) v, q = TestFunctions(W) f = Function(U) #get solver parameters #[parameters_pres_iter,parameters_corr,parameters_pres,parameters_pres_better,parameters_velo,parameters_velo_initial,nn]=defineSolverParameters() parameters_iter = defineSolverParameters()[1] parameters_velo = parameters_iter[0] parameters_pres = parameters_iter[1] parameters_corr = parameters_iter[2] PETSc.Sys.Print("Solver parameter sets used: \n", parameters_iter) #split up boundary conditions [bc_norm, bc_tang, bc_expr_list] = bc with PETSc.Log.Event("initial values"): PETSc.Sys.Print( "\nCALCULATE INITIAL VALUES" ) ######################################################## #check if analytical initial condition is given if (u_init): u_init_sol = Function(U).project(u_init) else: #calculate inital velocity with potential flow u_init_sol = initial_velocity(W, dt, mesh, bc, nue, order, IP_stabilityparam_type) divtest = Function(P).project(div(u_init_sol)) PETSc.Sys.Print("Div error of initial velocity", errornorm(divtest, Function(P))) #check if analytical solutions is given if p_init: x, y = SpatialCoordinate(W.mesh()) p_init_sol = Function(P).project(p_init) else: #with the initial value calculate initial pressure #with Poission euqation including some non-divergence free velocity p_init_sol = initial_pressure(W, dt, mesh, nue, bc, u_init_sol, order, IP_stabilityparam_type) with PETSc.Log.Event("build forms"): PETSc.Sys.Print( "\nBUILD FORMS" ) ##################################################################### v_k = Function(U) u_n = Function(U, name="Velocity") p_n = Function(P, name="Pressure") div_old = Function(P) v_knew_hat = Function(U) beta = Function(P) eq_pred = build_predictor_form(W, dt, mesh, nue, bc_tang, v_k, u_n, p_n, order, IP_stabilityparam_type) eq_upd = build_update_form(W, dt, mesh, bc_tang, div_old) eq_corr = build_corrector_form(W, dt, mesh, v_knew_hat, beta) with PETSc.Log.Event("build problems and solvers"): PETSc.Sys.Print( "\nBUILD PROBLEM AND SOLVERS" ) ######################################################## nullspace = MixedVectorSpaceBasis( W, [W.sub(0), VectorSpaceBasis(constant=True)]) def nullspace_basis(T): return VectorSpaceBasis(constant=True) appctx = {'trace_nullspace': nullspace_basis} with PETSc.Log.Event("predictor"): w_pred = Function(U) predictor = LinearVariationalProblem(lhs(eq_pred), rhs(eq_pred), w_pred, bc_norm) solver_pred = LinearVariationalSolver( predictor, solver_parameters=parameters_velo) with PETSc.Log.Event("update"): w_upd = Function(W) nullspace = MixedVectorSpaceBasis( W, [W.sub(0), VectorSpaceBasis(constant=True)]) update = LinearVariationalProblem(lhs(eq_upd), rhs(eq_upd), w_upd, bc_norm) solver_upd = LinearVariationalSolver( update, solver_parameters=parameters_pres, appctx=appctx) with PETSc.Log.Event("corrector"): w_corr = Function(U) corrector = LinearVariationalProblem(lhs(eq_corr), rhs(eq_corr), w_corr, bc_norm) solver_corr = LinearVariationalSolver( corrector, solver_parameters=parameters_corr) with PETSc.Log.Event("time progressing"): #initialise time stepping u_n.assign(u_init_sol) p_n.assign(p_init_sol) #save & divtest if output: outfile.write(u_n, p_n, time=0) divtest = Function(P).project(div(u_n)) PETSc.Sys.Print("Div error of initial velocity %d" % errornorm(divtest, Function(P))) PETSc.Sys.Print( "\nTIME PROGRESSING" ) ################################################################ #outerloop for time progress n = 1 while n < (T + 1): #update time-dependent boundary t.assign(n) PETSc.Sys.Print("t is: ", n * dt) PETSc.Sys.Print("n is: ", n) print(bc_expr_list[0]) if bc_tang: bc_tang[0] = [bc_tang[0][0].project(bc_expr_list[0]), 1] bc_tang[1] = [bc_tang[1][0].project(bc_expr_list[1]), 2] bc_tang[2] = [bc_tang[2][0].project(bc_expr_list[2]), 3] bc_tang[3] = [bc_tang[3][0].project(bc_expr_list[3]), 4] #update start value of picard iteration counter = 0 v_k.assign(u_n) with PETSc.Log.Event("picard iteration"): PETSc.Sys.Print( "\n1)PREDICTOR" ) ################################################################## #loop for non-linearity while (True): with timed_stage("predictor solve"): solver_pred.solve() #convergence criterion eps = errornorm(v_k, w_pred) #l2 by default counter += 1 PETSc.Sys.Print("Picard iteration counter: ", counter, "Picard iteration norm: ", eps) if (counter > 6): #eps<10**(-8)): PETSc.Sys.Print("Picard iteration converged") break else: v_k.assign(w_pred) PETSc.Sys.Print( "\n2) PRESSURE UPDATE" ) ######################################################### #first modify update form div_old_temp = Function(P).project(div(w_pred)) div_old.assign(div_old_temp) PETSc.Sys.Print("Div error of predictor velocity", errornorm(div_old, Function(P))) #solve update equation with timed_stage("update solve"): solver_upd.solve() wsol, betasol = w_upd.split() #update pressure p_knew = Function(P).assign(p_n + betasol) PETSc.Sys.Print( "\n3) CORRECTOR" ) ############################################################## #first modify corrector form v_knew_hat.assign(w_pred) beta.assign(betasol) #then solve with timed_stage("corrector solve"): solver_corr.solve() usol = Function(U).assign(w_corr) #divtest divtest = Function(P).project(div(usol)) PETSc.Sys.Print("Div error of corrector velocity", errornorm(divtest, Function(P))) #update for next time step u_n.assign(usol) p_n.assign(p_knew) #write in vtk file if output: outfile.write(u_n, p_n, time=n) n += 1 #final time step solution sol = Function(W) sol.sub(0).assign(u_n) sol.sub(1).assign(p_n) return sol
def func_wrapper(*args, **kwargs): with timed_stage('{}'.format(func.__name__[:15])): return func(*args, **kwargs)
def run_simulation(self, tmax, write=False, dumpfreq=100): PETSc.Sys.Print(""" Running linear Boussinesq simulation with parameters:\n Hybridization: %s,\n Model order: %s,\n Refinements: %s,\n Layers: %s,\n Lid height (m): %s,\n Radius (m): %s,\n Coriolis: %s,\n Horizontal Courant number: %s,\n Approx. Delta x (m): %s,\n Time-step size (s): %s,\n Stop time (s): %s. """ % (self._hybridization, self._order, self._refinements, self._nlayers, self._H, self._R, self._coriolis, self._nu_cfl, self._dx_avg, self._dt, tmax)) t = 0.0 self._initialize() un, pn, bn = self._state.split() dumpcount = dumpfreq if write: dumpcount = self.write(dumpcount, dumpfreq) self.up_solver.snes.setConvergenceHistory() self.up_solver.snes.ksp.setConvergenceHistory() self.b_solver.snes.setConvergenceHistory() self.b_solver.snes.ksp.setConvergenceHistory() while t < tmax: t += self._dt self._sim_time.append(t) # Solve for new u and p field self._up.assign(0.0) with timed_stage("Velocity-Pressure-Solve"): self.up_solver.solve() # Update state with new u and p un.assign(self._up.sub(0)) pn.assign(self._up.sub(1)) # Reconstruct b using new u and p self._btmp.assign(0.0) with timed_stage("Buoyancy-Solve"): self.b_solver.solve() bn.assign(assemble(bn - self._dt_half_N2 * self._btmp)) # Collect residual reductions for u-p system r0 = self.up_solver.snes.ksp.getRhs() # Assemble u-p residual self._assemble_upr() rn = self._up_residual r0norm = r0.norm() rnnorm = rn.dat.norm reduction = rnnorm / r0norm self._up_residual_reductions.append(reduction) # Collect KSP iterations outer_ksp = self.up_solver.snes.ksp if self._hybridization: ctx = outer_ksp.getPC().getPythonContext() inner_ksp = ctx.trace_ksp else: ksps = outer_ksp.getPC().getFieldSplitSubKSP() _, inner_ksp = ksps outer_its = outer_ksp.getIterationNumber() inner_its = inner_ksp.getIterationNumber() self._outer_ksp_iterations.append(outer_its) self._inner_ksp_iterations.append(inner_its) if write: with timed_stage("Dump output"): dumpcount = self.write(dumpcount, dumpfreq)
# Set up time stepping routines vp_timestepper = PressureProjectionTimeIntegrator([mom_eq, cty_eq], m, vp_fields, vp_coupling, dt, vp_bcs, solver_parameters=vp_solver_parameters, predictor_solver_parameters=predictor_solver_parameters, picard_iterations=1, pressure_nullspace=VectorSpaceBasis(constant=True)) # performs pseudo timestep to get good initial pressure # this is to avoid inconsistencies in terms (viscosity and advection) that # are meant to decouple from pressure projection, but won't if pressure is not initialised # do this here, so we can see the initial pressure in pressure_0.pvtu if not DUMP: # should not be done when picking up with timed_stage('initial_pressure'): vp_timestepper.initialize_pressure() ########## # Set up Vectorfolder folder = "/data/ekman3D/extruded_meshes/"+str(args.date)+"_3d_ekman_dt"+str(dt)+\ "_dtOut"+str(output_dt)+"_T"+str(T)+"_ip2_Muh"+str(mu_h.values()[0])+"_Muv"+str(mu_v.values()[0])+"fromdump7.8days/" ########### # Output files for velocity, pressure, temperature and salinity v_file = File(folder+"velocity.pvd") v_file.write(v_)
def stepper(self, t_start, t_end, w, t_visualization): """ Timesteps the shallow water equations from t_start to t_end using a 3rd order SSP Runge-Kutta scheme :param t_start: start time :type t_start: float :param t_end: end time :type t_end: float :param w: Current state vector function :param t_visualization: time interval to write the free surface water depth to a .pvd file :type t_visualization: float """ self.w = w # setup the solver - this is a timed stage for profiling reasons! with timed_stage("Setup of forms and solver"): self.__solver_setup() # used as the original state vector in each RK3 step self.w_old = Function(self.V).assign(self.w) # initial slope modification self.__update_slope_modification() hout = Function(self.v_h) hout.rename("free surface depth") # free surface depth hout_file = File("h.pvd") # bed depth bout = Function(self.v_h).project(self.b_) bout.rename("topography") bout_file = File("b.pvd") self.Project = Projector( conditional(self.h <= (self.plot_tol * self.E), self.b_, self.h + self.b_), hout) self.Project.project() hout_file.write(hout) bout_file.write(bout) self.t = t_start # start counter of how many time dumps self.c = 1 # warning boolean marker self.wmark = 0 while self.t < t_end: # find new timestep with timed_stage("Finding adaptive time-step"): self.dt = self.AT.FindTimestep(self.w) self.Dt.assign(self.dt) # check that prescribed timestep doesn't fall below minimum timestep if self.dt < self.MinTimestep: if self.wmark is False: warning(RED % "Minimum timestep has been reached. " + "Simulation might become unstable.") self.wmark = 1 self.dt = self.MinTimestep self.Dt.assign(self.dt) # check if remaining time to next time dump is less than timestep # correct if neeeded if self.dt + self.t > self.c * t_visualization: self.dt = (self.c * t_visualization) - self.t self.Dt.assign(self.dt) # check if remaining time to end time is less than timestep # correct if needed if (self.t <= (self.c + 1) * t_visualization) and (self.dt + self.t > t_end): self.dt = t_end - self.t self.Dt.assign(self.dt) with timed_stage("Runge-Kutta time-stepping scheme"): self.solver.solve() self.w.assign(self.w_) # slope limiter with timed_stage("Slope limiting"): self.__update_slope_limiter() # slope modification with timed_stage("Slope modification"): self.__update_slope_modification() self.solver.solve() self.w.assign((3.0 / 4.0) * self.w_old + (1.0 / 4.0) * self.w_) # slope limiter with timed_stage("Slope limiting"): self.__update_slope_limiter() # slope modification with timed_stage("Slope modification"): self.__update_slope_modification() self.solver.solve() self.w.assign((1.0 / 3.0) * self.w_old + (2.0 / 3.0) * self.w_) # slope limiter with timed_stage("Slope limiting"): self.__update_slope_limiter() # slope modification with timed_stage("Slope modification"): self.__update_slope_modification() self.w_old.assign(self.w) # timstep complete - dump realisation if needed self.t += self.dt with timed_stage("Visualization"): if self.t == self.c * t_visualization: self.Project.project() hout_file.write(hout) bout_file.write(bout) self.c += 1 self.dt = self.initial_dt self.Dt.assign(self.dt) # return timestep self.dt = self.initial_dt self.Dt.assign(self.dt) return self.w
def run_simulation(self, tmax, write=False, dumpfreq=100): PETSc.Sys.Print(""" Running Williamson 5 test with parameters:\n method: %s,\n model degree: %d,\n refinements: %d,\n hybridization: %s,\n tmax: %s """ % (self.method, self.model_degree, self.refinement_level, self.hybridization, tmax)) t = 0.0 un, Dn = self.state up, Dp = self.updates deltau, deltaD = self.DU.split() dumpcount = dumpfreq if write: dumpcount = self.write(dumpcount, dumpfreq) self.Dsolver.snes.setConvergenceHistory() self.Dsolver.snes.ksp.setConvergenceHistory() self.Usolver.snes.setConvergenceHistory() self.Usolver.snes.ksp.setConvergenceHistory() self.DUsolver.snes.setConvergenceHistory() self.DUsolver.snes.ksp.setConvergenceHistory() while t < tmax - self.Dt / 2: t += self.Dt up.assign(un) Dp.assign(Dn) for i in range(4): self.sim_time.append(t) self.picard_seq.append(i + 1) with timed_stage("DU Residuals"): self.Dsolver.solve() self.Usolver.solve() with timed_stage("Linear solve"): self.DUsolver.solve() # Here we collect the reductions in the linear residual. # Get rhs from ksp r0 = self.DUsolver.snes.ksp.getRhs() # Assemble the problem residual (b - Ax) res = assemble(self.FuD, mat_type="aij") bnorm = r0.norm() rnorm = res.dat.norm r_factor = rnorm / bnorm self.reductions.append(r_factor) up += deltau Dp += deltaD outer_ksp = self.DUsolver.snes.ksp if self.hybridization: ctx = outer_ksp.getPC().getPythonContext() inner_ksp = ctx.trace_ksp else: ksps = outer_ksp.getPC().getFieldSplitSubKSP() _, inner_ksp = ksps # Collect ksp iterations self.ksp_outer_its.append(outer_ksp.getIterationNumber()) self.ksp_inner_its.append(inner_ksp.getIterationNumber()) un.assign(up) Dn.assign(Dp) if write: with timed_stage("Dump output"): dumpcount = self.write(dumpcount, dumpfreq)
def apply(self, field): with timed_stage('limiter'): super(VertexBasedP1DGLimiter, self).apply(field)
Planet radius (m): %s,\n Atmospheric lid (m) %s,\n Mixed method: %s,\n Model degree: %s,\n Hybridization: %s,\n Horizontal CFL: %s,\n Dt (s): %s,\n Dx (km): %s,\n Dz (m): %s,\n Solver rtol: %s,\n KSP monitor: %s. """ % (solver._R, solver.H, solver.method, solver.model_degree, solver.hybridization, solver.courant, solver.Dt, solver.dx_max / 1000, solver.dz, solver.rtol, solver.monitor)) with timed_stage("Warm up"): solver.warmup() PETSc.Sys.Print("""Warm up complete. Profiling linear solver.""") solver.run_profile() PETSc.Sys.Print("""Run complete. Extracting data.""") PETSc.Log.Stage("UP Solver").push() comm = solver.comm snes = PETSc.Log.Event("SNESSolve").getPerfInfo() ksp = PETSc.Log.Event("KSPSolve").getPerfInfo() pcsetup = PETSc.Log.Event("PCSetUp").getPerfInfo()
type=int, default=1, help='Telescope factor for telescoping solvers (set to number of NODES)') args, unknown = parser.parse_known_args() results = Path(args.resultsdir) if (not results.is_dir()) and (COMM_WORLD.rank == 0): try: results.mkdir() except FileExistsError: print('File', cwd, 'already exists, cannot create directory with the same name') csvfile = ResultsCSV(args) with timed_stage('Mesh_hierarchy'): distribution_parameters = { "partition": True, "overlap_type": (DistributedMeshOverlapType.VERTEX, 1) } mesh = BoxMesh(args.baseN, args.baseN, args.baseN, 1, 1, 1, distribution_parameters=distribution_parameters) hierarchy = MeshHierarchy(mesh, args.nref) mesh = hierarchy[-1] with timed_stage('Problem_setup'):
tp.M = amd.p1metric else: tp.indicate_error() print("Error estimator: {:.4e}".format(tp.estimators['dwr'])) if i > 0 and np.abs(tp.estimators['dwr'] - estimator_old) < estimator_rtol * estimator_old: print("Number of elements: ", tp.mesh.num_cells()) print("Number of dofs: ", sum(tp.V.dof_count)) print("Converged error estimator!") break qoi_old = tp.qoi num_cells_old = tp.mesh.num_cells() estimator_old = tp.estimators['dwr'] # Adapt mesh with timed_stage('mesh adapt {:d}'.format(i)): tp.adapt_mesh() mh = MeshHierarchy(tp.mesh, 1) sol = tp.solution print('\n' + 80 * '#') print('SUMMARY') print(80 * '#' + '\n') print('Approach: {:s}'.format(op.approach)) print('Target: {:.1f}'.format(op.target)) print("Number of elements: {:d}".format(tp.mesh.num_cells())) print("Number of dofs: {:d}".format(sum(tp.V.dof_count))) print("Quantity of interest: {:.4e}".format(tp.qoi)) print(80 * '#')
#depth_profile_to_csv(depth_profile500m, velocity_depth_profile500m, "500m", '0.0') #depth_profile_to_csv(depth_profile1km, velocity_depth_profile1km, "1km", '0.0') #depth_profile_to_csv(depth_profile2km, velocity_depth_profile2km, "2km", '0.0') #depth_profile_to_csv(depth_profile4km, velocity_depth_profile4km, "4km", '0.0') #depth_profile_to_csv(depth_profile6km, velocity_depth_profile6km, "6km", '0.0') ######## # Begin time stepping t = 0.0 step = 0 while t < T - 0.5*dt: with timed_stage('velocity-pressure'): vp_timestepper.advance(t) #u_timestepper.advance(t) with timed_stage('temperature'): temp_timestepper.advance(t) with timed_stage('salinity'): sal_timestepper.advance(t) step += 1 t += dt with timed_stage('output'): if step % output_step == 0: # dumb checkpoint for starting from spin up tape.add_block(DiagnosticBlock(adj_s_file, sal)) tape.add_block(DiagnosticBlock(adj_t_file, temp))
def stepper(self, t_start, t_end, w, t_visualization): """ Timesteps the shallow water equations from t_start to t_end using a 3rd order SSP Runge-Kutta scheme :param t_start: start time :type t_start: float :param t_end: end time :type t_end: float :param w: Current state vector function :param t_visualization: time interval to write the free surface water depth to a .pvd file :type t_visualization: float """ self.w = w # setup the solver - this is a timed stage for profiling reasons! with timed_stage("Setup of forms and solver"): self.__solver_setup() # used as the original state vector in each RK3 step self.w_old = Function(self.V).assign(self.w) # initial slope modification self.__update_slope_modification() hout = Function(self.v_h) hout.rename("free surface depth") # free surface depth hout_file = File("h.pvd") # bed depth bout = Function(self.v_h).project(self.b_) bout.rename("topography") bout_file = File("b.pvd") self.Project = Projector(conditional(self.h <= (self.plot_tol * self.E), self.b_, self.h + self.b_), hout) self.Project.project() hout_file.write(hout) bout_file.write(bout) self.t = t_start # start counter of how many time dumps self.c = 1 # warning boolean marker self.wmark = 0 while self.t < t_end: # find new timestep with timed_stage("Finding adaptive time-step"): self.dt = self.AT.FindTimestep(self.w) self.Dt.assign(self.dt) # check that prescribed timestep doesn't fall below minimum timestep if self.dt < self.MinTimestep: if self.wmark is False: warning(RED % "Minimum timestep has been reached. " + "Simulation might become unstable.") self.wmark = 1 self.dt = self.MinTimestep self.Dt.assign(self.dt) # check if remaining time to next time dump is less than timestep # correct if neeeded if self.dt + self.t > self.c * t_visualization: self.dt = (self.c * t_visualization) - self.t self.Dt.assign(self.dt) # check if remaining time to end time is less than timestep # correct if needed if (self.t <= (self.c + 1) * t_visualization) and (self.dt + self.t > t_end): self.dt = t_end - self.t self.Dt.assign(self.dt) with timed_stage("Runge-Kutta time-stepping scheme"): self.solver.solve() self.w.assign(self.w_) # slope limiter with timed_stage("Slope limiting"): self.__update_slope_limiter() # slope modification with timed_stage("Slope modification"): self.__update_slope_modification() self.solver.solve() self.w.assign((3.0 / 4.0) * self.w_old + (1.0 / 4.0) * self.w_) # slope limiter with timed_stage("Slope limiting"): self.__update_slope_limiter() # slope modification with timed_stage("Slope modification"): self.__update_slope_modification() self.solver.solve() self.w.assign((1.0 / 3.0) * self.w_old + (2.0 / 3.0) * self.w_) # slope limiter with timed_stage("Slope limiting"): self.__update_slope_limiter() # slope modification with timed_stage("Slope modification"): self.__update_slope_modification() self.w_old.assign(self.w) # timstep complete - dump realisation if needed self.t += self.dt with timed_stage("Visualization"): if self.t == self.c * t_visualization: self.Project.project() hout_file.write(hout) bout_file.write(bout) self.c += 1 self.dt = self.initial_dt self.Dt.assign(self.dt) # return timestep self.dt = self.initial_dt self.Dt.assign(self.dt) return self.w
def run_williamson5(problem_cls, Dt, refinements, method, model_degree, mesh_degree, nsteps, hybridization, write=False, cold=False): # Radius of the Earth (m) R = 6371220.0 # Max depth height (m) H = 5960.0 if cold: PETSc.Sys.Print(""" Running cold initialization for the problem set:\n method: %s,\n model degree: %s,\n hybridization: %s,\n """ % (method, model_degree, bool(hybridization))) problem = problem_cls(refinement_level=refinements, R=R, H=H, Dt=Dt, method=method, hybridization=hybridization, model_degree=model_degree, mesh_degree=mesh_degree, monitor=args.monitor) problem.warmup() return problem = problem_cls(refinement_level=refinements, R=R, H=H, Dt=Dt, method=method, hybridization=hybridization, model_degree=model_degree, mesh_degree=mesh_degree, monitor=args.monitor) cfl = problem.courant dx_max = problem.dx_max PETSc.Sys.Print(""" Dt = %s,\n Courant number (approximate): %s,\n Dx (max): %s km,\n nsteps: %s. """ % (Dt, cfl, dx_max / 1000, nsteps)) comm = problem.comm day = 24. * 60. * 60. if args.profile: tmax = nsteps * Dt else: tmax = 15 * day PETSc.Sys.Print("Running 15 day simulation\n") # If writing simulation output, write out fields in 5-day intervals dumpfreq = 5 * day / Dt PETSc.Sys.Print("Warm up with one-step.\n") with timed_stage("Warm up"): problem.warmup() PETSc.Log.Stage("Warm up: Linear solve").push() prepcsetup = PETSc.Log.Event("PCSetUp").getPerfInfo() pre_res_eval = PETSc.Log.Event("SNESFunctionEval").getPerfInfo() pre_jac_eval = PETSc.Log.Event("SNESJacobianEval").getPerfInfo() pre_res_eval_time = comm.allreduce(pre_res_eval["time"], op=MPI.SUM) / comm.size pre_jac_eval_time = comm.allreduce(pre_jac_eval["time"], op=MPI.SUM) / comm.size pre_setup_time = comm.allreduce(prepcsetup["time"], op=MPI.SUM) / comm.size if problem.hybridization: prehybridinit = PETSc.Log.Event("HybridInit").getPerfInfo() prehybridinit_time = comm.allreduce(prehybridinit["time"], op=MPI.SUM) / comm.size PETSc.Log.Stage("Warm up: Linear solve").pop() PETSc.Sys.Print("Warm up done. Profiling run for %d steps.\n" % nsteps) problem.initialize() problem.run_simulation(tmax, write=write, dumpfreq=dumpfreq) PETSc.Sys.Print("Simulation complete.\n") PETSc.Log.Stage("Linear solve").push() snes = PETSc.Log.Event("SNESSolve").getPerfInfo() ksp = PETSc.Log.Event("KSPSolve").getPerfInfo() pcsetup = PETSc.Log.Event("PCSetUp").getPerfInfo() pcapply = PETSc.Log.Event("PCApply").getPerfInfo() jac_eval = PETSc.Log.Event("SNESJacobianEval").getPerfInfo() residual = PETSc.Log.Event("SNESFunctionEval").getPerfInfo() snes_time = comm.allreduce(snes["time"], op=MPI.SUM) / comm.size ksp_time = comm.allreduce(ksp["time"], op=MPI.SUM) / comm.size pc_setup_time = comm.allreduce(pcsetup["time"], op=MPI.SUM) / comm.size pc_apply_time = comm.allreduce(pcapply["time"], op=MPI.SUM) / comm.size jac_eval_time = comm.allreduce(jac_eval["time"], op=MPI.SUM) / comm.size res_eval_time = comm.allreduce(residual["time"], op=MPI.SUM) / comm.size ref = problem.refinement_level num_cells = comm.allreduce(problem.num_cells, op=MPI.SUM) if problem.hybridization: results_data = "results/hybrid_%s_data_W5_ref%d_Dt%s_NS%d.csv" % ( problem.method, ref, Dt, nsteps) results_timings = "results/hybrid_%s_profile_W5_ref%d_Dt%s_NS%d.csv" % ( problem.method, ref, Dt, nsteps) RHS = PETSc.Log.Event("HybridRHS").getPerfInfo() trace = PETSc.Log.Event("SCSolve").getPerfInfo() proj = PETSc.Log.Event("HybridProject").getPerfInfo() full_recon = PETSc.Log.Event("SCBackSub").getPerfInfo() hybridbreak = PETSc.Log.Event("HybridBreak").getPerfInfo() hybridupdate = PETSc.Log.Event("HybridUpdate").getPerfInfo() hybridinit = PETSc.Log.Event("HybridInit").getPerfInfo() # Time to reconstruct (backsub) and project full_recon_time = comm.allreduce(full_recon["time"], op=MPI.SUM) / comm.size # Project only projection = comm.allreduce(proj["time"], op=MPI.SUM) / comm.size # Backsub only = Total Recon time - projection time recon_time = full_recon_time - projection transfer = comm.allreduce(hybridbreak["time"], op=MPI.SUM) / comm.size update_time = comm.allreduce(hybridupdate["time"], op=MPI.SUM) / comm.size trace_solve = comm.allreduce(trace["time"], op=MPI.SUM) / comm.size rhstime = comm.allreduce(RHS["time"], op=MPI.SUM) / comm.size inittime = comm.allreduce(hybridinit["time"], op=MPI.SUM) / comm.size other = ksp_time - (trace_solve + transfer + projection + recon_time + rhstime) full_solve = (transfer + trace_solve + rhstime + recon_time + projection + update_time) else: results_data = "results/gmres_%s_data_W5_ref%d_Dt%s_NS%d.csv" % ( problem.method, ref, Dt, nsteps) results_timings = "results/gmres_%s_profile_W5_ref%d_Dt%s_NS%d.csv" % ( problem.method, ref, Dt, nsteps) KSPSchur = PETSc.Log.Event("KSPSolve_FS_Schu").getPerfInfo() KSPF0 = PETSc.Log.Event("KSPSolve_FS_0").getPerfInfo() KSPLow = PETSc.Log.Event("KSPSolve_FS_Low").getPerfInfo() schur_time = comm.allreduce(KSPSchur["time"], op=MPI.SUM) / comm.size f0_time = comm.allreduce(KSPF0["time"], op=MPI.SUM) / comm.size ksplow_time = comm.allreduce(KSPLow["time"], op=MPI.SUM) / comm.size other = ksp_time - (schur_time + f0_time + ksplow_time) PETSc.Log.Stage("Linear solve").pop() if COMM_WORLD.rank == 0: if not os.path.exists(os.path.dirname('results/')): os.makedirs(os.path.dirname('results/')) data = { "OuterIters": problem.ksp_outer_its, "InnerIters": problem.ksp_inner_its, "PicardIters": problem.picard_seq, "SimTime": problem.sim_time, "ResidualReductions": problem.reductions } dofs = problem.DU.dof_dset.layout_vec.getSize() time_data = { "PETSCLogKSPSolve": ksp_time, "PETSCLogPCApply": pc_apply_time, "PETSCLogPCSetup": pc_setup_time, "PETSCLogPreSetup": pre_setup_time, "PETSCLogPreSNESJacobianEval": pre_jac_eval_time, "PETSCLogPreSNESFunctionEval": pre_res_eval_time, "SNESSolve": snes_time, "SNESFunctionEval": res_eval_time, "SNESJacobianEval": jac_eval_time, "num_processes": problem.comm.size, "method": problem.method, "model_degree": problem.model_degree, "refinement_level": problem.refinement_level, "total_dofs": dofs, "num_cells": num_cells, "Dt": Dt, "CFL": cfl, "nsteps": nsteps, "DxMax": dx_max } if problem.hybridization: updates = { "HybridTraceSolve": trace_solve, "HybridRHS": rhstime, "HybridBreak": transfer, "HybridReconstruction": recon_time, "HybridProjection": projection, "HybridFullRecovery": full_recon_time, "HybridUpdate": update_time, "HybridInit": inittime, "PreHybridInit": prehybridinit_time, "HybridFullSolveTime": full_solve, "HybridKSPOther": other } else: updates = { "KSPSchur": schur_time, "KSPF0": f0_time, "KSPFSLow": ksplow_time, "KSPother": other } time_data.update(updates) df_data = pd.DataFrame(data) df_data.to_csv(results_data, index=False, mode="w", header=True) df_time = pd.DataFrame(time_data, index=[0]) df_time.to_csv(results_timings, index=False, mode="w", header=True)