def semi_implicit_step(self):
state = self.state
dt = state.timestepping.dt
alpha = state.timestepping.alpha
with timed_stage("Apply forcing terms"):
self.forcing.apply((1-alpha)*dt, state.xn, state.xn,
state.xstar, implicit=False)
for k in range(state.timestepping.maxk):
with timed_stage("Advection"):
for name, advection in self.active_advection:
# first computes ubar from state.xn and state.xnp1
advection.update_ubar(state.xn, state.xnp1, alpha)
# advects a field from xstar and puts result in xp
advection.apply(self.xstar_fields[name], self.xp_fields[name])
state.xrhs.assign(0.) # xrhs is the residual which goes in the linear solve
for i in range(state.timestepping.maxi):
with timed_stage("Apply forcing terms"):
self.forcing.apply(alpha*dt, state.xp, state.xnp1,
state.xrhs, implicit=True,
incompressible=self.incompressible)
state.xrhs -= state.xnp1
with timed_stage("Implicit solve"):
self.linear_solver.solve() # solves linear system and places result in state.dy
state.xnp1 += state.dy
self._apply_bcs()
def solve(self, r_u, r_p, r_b):
self._ru.assign(r_u)
self._rp.assign(r_p)
self._rb.assign(r_b)
# Fields for solution
self._u.assign(0.0)
self._p.assign(0.0)
self._b.assign(0.0)
# initialize solver fields
self._up.assign(0.0)
self._btmp.assign(0.0)
with timed_stage('UP Solver'):
self._up_solver.solve()
self._u.assign(self._up.sub(0))
self._p.assign(self._up.sub(1))
with timed_stage('B Solver'):
self._b_solver.solve()
self._b.assign(assemble(self._rb - self._dt_half_N2 * self._btmp))
return self._u, self._p, self._b
def picard_step(self):
self.solution_lag.assign(self.solution)
# solve for self.u_star
with timed_stage("momentum_solve"):
self.predictor_solver.solve()
# pressure correction solve, solves for final solution (corrected velocity and pressure)
with timed_stage("correction_solve"):
self.solver.solve()
def run(self, t, tmax, pickup=False):
"""
This is the timeloop. After completing the semi implicit step
any passively advected fields are updated, implicit diffusion and
physics updates are applied (if required).
"""
state = self.state
t = self.setup_timeloop(state, t, tmax, pickup)
dt = state.timestepping.dt
while t < tmax - 0.5*dt:
logger.info("at start of timestep, t=%s, dt=%s" % (t, dt))
t += dt
state.t.assign(t)
state.xnp1.assign(state.xn)
for name, evaluation in self.prescribed_fields:
state.fields(name).project(evaluation(t))
self.semi_implicit_step()
for name, advection in self.passive_advection:
field = getattr(state.fields, name)
# first computes ubar from state.xn and state.xnp1
advection.update_ubar(state.xn, state.xnp1, state.timestepping.alpha)
# advects a field from xn and puts result in xnp1
advection.apply(field, field)
state.xb.assign(state.xn)
state.xn.assign(state.xnp1)
with timed_stage("Diffusion"):
for name, diffusion in self.diffused_fields:
field = getattr(state.fields, name)
diffusion.apply(field, field)
with timed_stage("Physics"):
for physics in self.physics_list:
physics.apply()
with timed_stage("Dump output"):
state.dump(t)
if state.output.checkpoint:
state.chkpt.close()
logger.info("TIMELOOP complete. t=%s, tmax=%s" % (t, tmax))
def warmup(self):
self.initialize()
un, Dn = self.state
up, Dp = self.updates
up.assign(un)
Dp.assign(Dn)
with timed_stage("Warm up: DU Residuals"):
self.Dsolver.solve()
self.Usolver.solve()
with timed_stage("Warm up: Linear solve"):
self.DUsolver.solve()
def run(self, t, tmax, pickup=False):
"""
This is the timeloop. After completing the semi implicit step
any passively transported fields are updated, implicit diffusion and
physics updates are applied (if required).
"""
state = self.state
if pickup:
t = state.pickup_from_checkpoint()
state.setup_diagnostics()
with timed_stage("Dump output"):
state.setup_dump(t, tmax, pickup)
state.t.assign(t)
self.x.initialise(state)
while float(state.t) < tmax - 0.5*float(state.dt):
logger.info(f'at start of timestep, t={float(state.t)}, dt={float(state.dt)}')
self.x.update()
self.timestep()
for field in self.x.np1:
state.fields(field.name()).assign(field)
with timed_stage("Physics"):
for physics in self.physics_list:
physics.apply()
# TODO: Hack to ensure that xnp1 fields are updated
for field in self.x.np1:
field.assign(state.fields(field.name()))
state.t.assign(state.t + state.dt)
with timed_stage("Dump output"):
state.dump(float(state.t))
if state.output.checkpoint:
state.chkpt.close()
logger.info(f'TIMELOOP complete. t={float(state.t)}, tmax={tmax}')
def timestep(self):
xn = self.x.n
xnp1 = self.x.np1
xstar = self.x.star
xp = self.x.p
xrhs = self.xrhs
dy = self.dy
with timed_stage("Apply forcing terms"):
self.forcing.apply(xn, xn, xstar(self.field_name), "explicit")
xp(self.field_name).assign(xstar(self.field_name))
for k in range(self.maxk):
with timed_stage("Transport"):
for name, scheme in self.active_transport:
# transports a field from xstar and puts result in xp
scheme.apply(xstar(name), xp(name))
xrhs.assign(0.) # xrhs is the residual which goes in the linear solve
for i in range(self.maxi):
with timed_stage("Apply forcing terms"):
self.forcing.apply(xp, xnp1, xrhs, "implicit")
xrhs -= xnp1(self.field_name)
with timed_stage("Implicit solve"):
self.linear_solver.solve(xrhs, dy) # solves linear system and places result in state.dy
xnp1X = xnp1(self.field_name)
xnp1X += dy
# Update xnp1 values for active tracers not included in the linear solve
self.copy_active_tracers(xp, xnp1)
self._apply_bcs()
for name, scheme in self.auxiliary_schemes:
# transports a field from xn and puts result in xnp1
scheme.apply(xn(name), xnp1(name))
with timed_stage("Diffusion"):
for name, scheme in self.diffusion_schemes:
scheme.apply(xnp1(name), xnp1(name))
def warmup(self):
"""Warm up solver by taking one time-step."""
self._initialize()
un, pn, bn = self._state.split()
self._up.assign(0.0)
with timed_stage("Warmup: Velocity-Pressure-Solve"):
self.up_solver.solve()
un.assign(self._up.sub(0))
pn.assign(self._up.sub(1))
self._btmp.assign(0.0)
with timed_stage("Warmup: Buoyancy-Solve"):
self.b_solver.solve()
bn.assign(assemble(bn - self._dt_half_N2 * self._btmp))
def setup_timeloop(self, t, tmax, pickup):
"""
Setup the timeloop by setting up diagnostics, dumping the fields and
picking up from a previous run, if required
"""
self.state.setup_diagnostics()
with timed_stage("Dump output"):
self.state.setup_dump(tmax, pickup)
t = self.state.dump(t, pickup)
return t
def solve(self):
with timed_stage('copy_3d_to_2d'):
# execute par loop
op2.par_loop(self.kernel, self.fs_3d.mesh().cell_set,
self.output_2d.dat(op2.WRITE, self.fs_2d.cell_node_map()),
self.input_3d.dat(op2.READ, self.fs_3d.cell_node_map()),
self.idx(op2.READ),
iterate=self.iter_domain)
if self.do_rt_scaling:
self.rt_scale_solver.solve()
def warmup(self):
state = self.state
un, pn, bn = state.split()
un.assign(self.u0)
pn.assign(self.p0)
bn.assign(self.b0)
with timed_stage("Warm up: Solver"):
un1, pn1, bn1 = self.gravity_wave_solver.solve(un, pn, bn)
def solve(self):
with timed_stage('constant_3d'):
# execute par loop
op2.par_loop(self.kernel,
self.fs_3d.mesh().cell_set,
self.output_3d.dat(op2.WRITE,
self.fs_3d.cell_node_map()),
self.input_3d.dat(op2.READ,
self.fs_3d.cell_node_map()),
self.idx(op2.READ),
iterate=self.iter_domain)
def run_simulation(self, tmax, write=False, dumpfreq=100):
PETSc.Sys.Print("""
Running Skamarock and Klemp Gravity wave problem:\n
method: %s,\n
model degree: %d,\n
refinements: %d,\n
hybridization: %s,\n
tmax: %s
""" % (self.method, self.model_degree, self.refinement_level,
self.hybridization, tmax))
state = self.state
un, pn, bn = state.split()
un.assign(self.u0)
pn.assign(self.p0)
bn.assign(self.b0)
dumpcount = dumpfreq
if write:
dumpcount = self.write(dumpcount, dumpfreq)
self.gravity_wave_solver._up_solver.snes.setConvergenceHistory()
self.gravity_wave_solver._up_solver.snes.ksp.setConvergenceHistory()
self.gravity_wave_solver._b_solver.snes.setConvergenceHistory()
self.gravity_wave_solver._b_solver.snes.ksp.setConvergenceHistory()
t = 0.0
while t < tmax - self.Dt / 2:
t += self.Dt
self.sim_time.append(t)
un1, pn1, bn1 = self.gravity_wave_solver.solve(un, pn, bn)
un.assign(un1)
pn.assign(pn1)
bn.assign(bn1)
outer_ksp = self.gravity_wave_solver._up_solver.snes.ksp
if self.hybridization:
ctx = outer_ksp.getPC().getPythonContext()
inner_ksp = ctx.trace_ksp
else:
ksps = outer_ksp.getPC().getFieldSplitSubKSP()
_, inner_ksp = ksps
# Collect ksp iterations
self.ksp_outer_its.append(outer_ksp.getIterationNumber())
self.ksp_inner_its.append(inner_ksp.getIterationNumber())
if write:
with timed_stage("Dump output"):
dumpcount = self.write(dumpcount, dumpfreq)
def setup_timeloop(self, state, t, tmax, pickup):
"""
Setup the timeloop by setting up diagnostics, dumping the fields and
picking up from a previous run, if required
"""
if pickup:
t = state.pickup_from_checkpoint()
state.setup_diagnostics()
with timed_stage("Dump output"):
state.setup_dump(t, tmax, pickup)
return t
def solve(self):
with timed_stage("copy_3d_to_2d"):
# execute par loop
op2.par_loop(
self.kernel,
self.fs_3d.mesh().cell_set,
self.output_2d.dat(op2.WRITE, self.fs_2d.cell_node_map()),
self.input_3d.dat(op2.READ, self.fs_3d.cell_node_map()),
self.idx(op2.READ),
iterate=self.iter_domain,
)
if self.do_rt_scaling:
self.rt_scale_solver.solve()
def apply(self, field):
"""
Applies the limiter on the given field (in place)
:arg field: :class:`Function` to limit
"""
with timed_stage('limiter'):
if self.is_vector:
tmp_func = self.P1DG.get_work_function()
fs = field.function_space()
for i in range(fs.value_size):
tmp_func.dat.data_with_halos[:] = field.dat.data_with_halos[:, i]
super(VertexBasedP1DGLimiter, self).apply(tmp_func)
field.dat.data_with_halos[:, i] = tmp_func.dat.data_with_halos[:]
self.P1DG.restore_work_function(tmp_func)
else:
super(VertexBasedP1DGLimiter, self).apply(field)
def warmupSolve(level=2, mesh_arg=(10, 10)):
##Setup
mesh = UnitSquareMesh(mesh_arg[0], mesh_arg[1])
nlevel = level
mh = MeshHierarchy(mesh, nlevel)
V = FunctionSpace(mh[-1], 'CG', 2)
u = function.Function(V)
f = function.Function(V)
v = TestFunction(V)
F = dot(grad(u), grad(v)) * dx - f * v * dx
bcs = DirichletBC(V, 0.0, (1, 2, 3, 4))
x = SpatialCoordinate(V.mesh())
with offloading(cuda):
with timed_stage("warmup"):
f.interpolate(-0.5 * pi * pi *
(4 * cos(pi * x[0]) - 5 * cos(pi * x[0] * 0.5) + 2) *
sin(pi * x[1]))
solve(F == 0, u, bcs=bcs, solver_parameters=parameters)
def apply(self, field):
"""
Applies the limiter on the given field (in place)
:arg field: :class:`Function` to limit
"""
with timed_stage('limiter'):
if self.is_vector:
tmp_func = self.P1DG.get_work_function()
fs = field.function_space()
for i in range(fs.value_size):
tmp_func.dat.data_with_halos[:] = field.dat.data_with_halos[:,
i]
#super(VertexBasedP1DGLimiter, self).apply(tmp_func)
self.compute_bounds(tmp_func)
self.apply_limiter(tmp_func)
field.dat.data_with_halos[:,
i] = tmp_func.dat.data_with_halos[:]
self.P1DG.restore_work_function(tmp_func)
else:
#super(VertexBasedP1DGLimiter, self).apply(field)
# is the same, but seems less confusing, WPan 2020-03-27
self.compute_bounds(field)
self.apply_limiter(field)
def run(self, t, tmax, pickup=False):
state = self.state
state.xn.assign(state.x_init)
xstar_fields = {name: func for (name, func) in
zip(state.fieldlist, state.xstar.split())}
xp_fields = {name: func for (name, func) in
zip(state.fieldlist, state.xp.split())}
dt = state.timestepping.dt
alpha = state.timestepping.alpha
if state.mu is not None:
mu_alpha = [0., dt]
else:
mu_alpha = [None, None]
with timed_stage("Dump output"):
t = state.dump(t, pickup)
while t < tmax + 0.5*dt:
if state.output.Verbose:
print "STEP", t, dt
t += dt
with timed_stage("Apply forcing terms"):
self.forcing.apply((1-alpha)*dt, state.xn, state.xn,
state.xstar, mu_alpha=mu_alpha[0])
state.xnp1.assign(state.xn)
for k in range(state.timestepping.maxk):
with timed_stage("Compute ubar"):
self._set_ubar() # computes state.ubar from state.xn and state.xnp1
with timed_stage("Advection"):
for field, advection in self.advection_dict.iteritems():
# advects a field from xstar and puts result in xp
advection.apply(xstar_fields[field], xp_fields[field])
state.xrhs.assign(0.) # xrhs is the residual which goes in the linear solve
for i in range(state.timestepping.maxi):
with timed_stage("Apply forcing terms"):
self.forcing.apply(alpha*dt, state.xp, state.xnp1,
state.xrhs, mu_alpha=mu_alpha[1],
incompressible=self.incompressible)
state.xrhs -= state.xnp1
with timed_stage("Implicit solve"):
self.linear_solver.solve() # solves linear system and places result in state.dy
state.xnp1 += state.dy
self._apply_bcs()
state.xn.assign(state.xnp1)
with timed_stage("Diffusion"):
for name, diffusion in self.diffusion_dict.iteritems():
diffusion.apply(state.field_dict[name], state.field_dict[name])
with timed_stage("Dump output"):
state.dump(t, pickup=False)
state.diagnostic_dump()
print "TIMELOOP complete. t= "+str(t-dt)+" tmax="+str(tmax)
def solve(self):
with timed_stage("vert_integral"):
self.solver.solve()
m,
vp_fields,
vp_coupling,
dt,
vp_bcs,
solver_parameters=vp_solver_parameters,
predictor_solver_parameters=predictor_solver_parameters,
picard_iterations=1)
# performs pseudo timestep to get good initial pressure
# this is to avoid inconsistencies in terms (viscosity and advection) that
# are meant to decouple from pressure projection, but won't if pressure is not initialised
# do this here, so we can see the initial pressure in pressure_0.pvtu
if not DUMP:
# should not be done when picking up
with timed_stage('initial_pressure'):
vp_timestepper.initialize_pressure()
#u_timestepper = DIRK33(u_eq, u, u_fields, dt, u_bcs, solver_parameters=u_solver_parameters)
temp_timestepper = DIRK33(temp_eq,
temp,
temp_fields,
dt,
temp_bcs,
solver_parameters=temp_solver_parameters)
sal_timestepper = DIRK33(sal_eq,
sal,
sal_fields,
dt,
sal_bcs,
solver_parameters=sal_solver_parameters)
depth_profile_to_csv(depth_profile1km, velocity_depth_profile1km, "1km", '0.0')
depth_profile_to_csv(depth_profile2km, velocity_depth_profile2km, "2km", '0.0')
depth_profile_to_csv(depth_profile4km, velocity_depth_profile4km, "4km", '0.0')
depth_profile_to_csv(depth_profile6km, velocity_depth_profile6km, "6km", '0.0')
########
# Begin time stepping
t = 0.0
step = 0
PROFILING = False
if PROFILING:
while t < T - 0.5 * dt:
with timed_stage('velocity-pressure'):
vp_timestepper.advance(t)
u_timestepper.advance(t)
with timed_stage('temperature'):
temp_timestepper.advance(t)
with timed_stage('salinity'):
sal_timestepper.advance(t)
step += 1
t += dt
#with timed_region('output'):
# Output files
# if step % output_step == 0:
else:
while t < T - 0.5 * dt:
vp_timestepper.advance(t)
def spcs(W,
mesh,
nue,
bc,
U_inf,
t,
dt,
T,
outfile,
order,
IP_stabilityparam_type=None,
u_init=None,
p_init=None,
output=False):
with PETSc.Log.Event("spcs configuration"):
#functions and normal
U = W.sub(0)
P = W.sub(1)
u, p = TrialFunctions(W)
v, q = TestFunctions(W)
f = Function(U)
#get solver parameters
#[parameters_pres_iter,parameters_corr,parameters_pres,parameters_pres_better,parameters_velo,parameters_velo_initial,nn]=defineSolverParameters()
parameters_iter = defineSolverParameters()[1]
parameters_velo = parameters_iter[0]
parameters_pres = parameters_iter[1]
parameters_corr = parameters_iter[2]
PETSc.Sys.Print("Solver parameter sets used: \n", parameters_iter)
#split up boundary conditions
[bc_norm, bc_tang, bc_expr_list] = bc
with PETSc.Log.Event("initial values"):
PETSc.Sys.Print(
"\nCALCULATE INITIAL VALUES"
) ########################################################
#check if analytical initial condition is given
if (u_init):
u_init_sol = Function(U).project(u_init)
else:
#calculate inital velocity with potential flow
u_init_sol = initial_velocity(W, dt, mesh, bc, nue, order,
IP_stabilityparam_type)
divtest = Function(P).project(div(u_init_sol))
PETSc.Sys.Print("Div error of initial velocity",
errornorm(divtest, Function(P)))
#check if analytical solutions is given
if p_init:
x, y = SpatialCoordinate(W.mesh())
p_init_sol = Function(P).project(p_init)
else:
#with the initial value calculate initial pressure
#with Poission euqation including some non-divergence free velocity
p_init_sol = initial_pressure(W, dt, mesh, nue, bc, u_init_sol,
order, IP_stabilityparam_type)
with PETSc.Log.Event("build forms"):
PETSc.Sys.Print(
"\nBUILD FORMS"
) #####################################################################
v_k = Function(U)
u_n = Function(U, name="Velocity")
p_n = Function(P, name="Pressure")
div_old = Function(P)
v_knew_hat = Function(U)
beta = Function(P)
eq_pred = build_predictor_form(W, dt, mesh, nue, bc_tang, v_k, u_n,
p_n, order, IP_stabilityparam_type)
eq_upd = build_update_form(W, dt, mesh, bc_tang, div_old)
eq_corr = build_corrector_form(W, dt, mesh, v_knew_hat, beta)
with PETSc.Log.Event("build problems and solvers"):
PETSc.Sys.Print(
"\nBUILD PROBLEM AND SOLVERS"
) ########################################################
nullspace = MixedVectorSpaceBasis(
W, [W.sub(0), VectorSpaceBasis(constant=True)])
def nullspace_basis(T):
return VectorSpaceBasis(constant=True)
appctx = {'trace_nullspace': nullspace_basis}
with PETSc.Log.Event("predictor"):
w_pred = Function(U)
predictor = LinearVariationalProblem(lhs(eq_pred), rhs(eq_pred),
w_pred, bc_norm)
solver_pred = LinearVariationalSolver(
predictor, solver_parameters=parameters_velo)
with PETSc.Log.Event("update"):
w_upd = Function(W)
nullspace = MixedVectorSpaceBasis(
W, [W.sub(0), VectorSpaceBasis(constant=True)])
update = LinearVariationalProblem(lhs(eq_upd), rhs(eq_upd), w_upd,
bc_norm)
solver_upd = LinearVariationalSolver(
update, solver_parameters=parameters_pres, appctx=appctx)
with PETSc.Log.Event("corrector"):
w_corr = Function(U)
corrector = LinearVariationalProblem(lhs(eq_corr), rhs(eq_corr),
w_corr, bc_norm)
solver_corr = LinearVariationalSolver(
corrector, solver_parameters=parameters_corr)
with PETSc.Log.Event("time progressing"):
#initialise time stepping
u_n.assign(u_init_sol)
p_n.assign(p_init_sol)
#save & divtest
if output:
outfile.write(u_n, p_n, time=0)
divtest = Function(P).project(div(u_n))
PETSc.Sys.Print("Div error of initial velocity %d" %
errornorm(divtest, Function(P)))
PETSc.Sys.Print(
"\nTIME PROGRESSING"
) ################################################################
#outerloop for time progress
n = 1
while n < (T + 1):
#update time-dependent boundary
t.assign(n)
PETSc.Sys.Print("t is: ", n * dt)
PETSc.Sys.Print("n is: ", n)
print(bc_expr_list[0])
if bc_tang:
bc_tang[0] = [bc_tang[0][0].project(bc_expr_list[0]), 1]
bc_tang[1] = [bc_tang[1][0].project(bc_expr_list[1]), 2]
bc_tang[2] = [bc_tang[2][0].project(bc_expr_list[2]), 3]
bc_tang[3] = [bc_tang[3][0].project(bc_expr_list[3]), 4]
#update start value of picard iteration
counter = 0
v_k.assign(u_n)
with PETSc.Log.Event("picard iteration"):
PETSc.Sys.Print(
"\n1)PREDICTOR"
) ##################################################################
#loop for non-linearity
while (True):
with timed_stage("predictor solve"):
solver_pred.solve()
#convergence criterion
eps = errornorm(v_k, w_pred) #l2 by default
counter += 1
PETSc.Sys.Print("Picard iteration counter: ", counter,
"Picard iteration norm: ", eps)
if (counter > 6): #eps<10**(-8)):
PETSc.Sys.Print("Picard iteration converged")
break
else:
v_k.assign(w_pred)
PETSc.Sys.Print(
"\n2) PRESSURE UPDATE"
) #########################################################
#first modify update form
div_old_temp = Function(P).project(div(w_pred))
div_old.assign(div_old_temp)
PETSc.Sys.Print("Div error of predictor velocity",
errornorm(div_old, Function(P)))
#solve update equation
with timed_stage("update solve"):
solver_upd.solve()
wsol, betasol = w_upd.split()
#update pressure
p_knew = Function(P).assign(p_n + betasol)
PETSc.Sys.Print(
"\n3) CORRECTOR"
) ##############################################################
#first modify corrector form
v_knew_hat.assign(w_pred)
beta.assign(betasol)
#then solve
with timed_stage("corrector solve"):
solver_corr.solve()
usol = Function(U).assign(w_corr)
#divtest
divtest = Function(P).project(div(usol))
PETSc.Sys.Print("Div error of corrector velocity",
errornorm(divtest, Function(P)))
#update for next time step
u_n.assign(usol)
p_n.assign(p_knew)
#write in vtk file
if output:
outfile.write(u_n, p_n, time=n)
n += 1
#final time step solution
sol = Function(W)
sol.sub(0).assign(u_n)
sol.sub(1).assign(p_n)
return sol
def func_wrapper(*args, **kwargs):
with timed_stage('{}'.format(func.__name__[:15])):
return func(*args, **kwargs)
def run_simulation(self, tmax, write=False, dumpfreq=100):
PETSc.Sys.Print("""
Running linear Boussinesq simulation with parameters:\n
Hybridization: %s,\n
Model order: %s,\n
Refinements: %s,\n
Layers: %s,\n
Lid height (m): %s,\n
Radius (m): %s,\n
Coriolis: %s,\n
Horizontal Courant number: %s,\n
Approx. Delta x (m): %s,\n
Time-step size (s): %s,\n
Stop time (s): %s.
""" % (self._hybridization, self._order, self._refinements,
self._nlayers, self._H, self._R, self._coriolis, self._nu_cfl,
self._dx_avg, self._dt, tmax))
t = 0.0
self._initialize()
un, pn, bn = self._state.split()
dumpcount = dumpfreq
if write:
dumpcount = self.write(dumpcount, dumpfreq)
self.up_solver.snes.setConvergenceHistory()
self.up_solver.snes.ksp.setConvergenceHistory()
self.b_solver.snes.setConvergenceHistory()
self.b_solver.snes.ksp.setConvergenceHistory()
while t < tmax:
t += self._dt
self._sim_time.append(t)
# Solve for new u and p field
self._up.assign(0.0)
with timed_stage("Velocity-Pressure-Solve"):
self.up_solver.solve()
# Update state with new u and p
un.assign(self._up.sub(0))
pn.assign(self._up.sub(1))
# Reconstruct b using new u and p
self._btmp.assign(0.0)
with timed_stage("Buoyancy-Solve"):
self.b_solver.solve()
bn.assign(assemble(bn - self._dt_half_N2 * self._btmp))
# Collect residual reductions for u-p system
r0 = self.up_solver.snes.ksp.getRhs()
# Assemble u-p residual
self._assemble_upr()
rn = self._up_residual
r0norm = r0.norm()
rnnorm = rn.dat.norm
reduction = rnnorm / r0norm
self._up_residual_reductions.append(reduction)
# Collect KSP iterations
outer_ksp = self.up_solver.snes.ksp
if self._hybridization:
ctx = outer_ksp.getPC().getPythonContext()
inner_ksp = ctx.trace_ksp
else:
ksps = outer_ksp.getPC().getFieldSplitSubKSP()
_, inner_ksp = ksps
outer_its = outer_ksp.getIterationNumber()
inner_its = inner_ksp.getIterationNumber()
self._outer_ksp_iterations.append(outer_its)
self._inner_ksp_iterations.append(inner_its)
if write:
with timed_stage("Dump output"):
dumpcount = self.write(dumpcount, dumpfreq)
# Set up time stepping routines
vp_timestepper = PressureProjectionTimeIntegrator([mom_eq, cty_eq], m, vp_fields, vp_coupling, dt, vp_bcs,
solver_parameters=vp_solver_parameters,
predictor_solver_parameters=predictor_solver_parameters,
picard_iterations=1,
pressure_nullspace=VectorSpaceBasis(constant=True))
# performs pseudo timestep to get good initial pressure
# this is to avoid inconsistencies in terms (viscosity and advection) that
# are meant to decouple from pressure projection, but won't if pressure is not initialised
# do this here, so we can see the initial pressure in pressure_0.pvtu
if not DUMP:
# should not be done when picking up
with timed_stage('initial_pressure'):
vp_timestepper.initialize_pressure()
##########
# Set up Vectorfolder
folder = "/data/ekman3D/extruded_meshes/"+str(args.date)+"_3d_ekman_dt"+str(dt)+\
"_dtOut"+str(output_dt)+"_T"+str(T)+"_ip2_Muh"+str(mu_h.values()[0])+"_Muv"+str(mu_v.values()[0])+"fromdump7.8days/"
###########
# Output files for velocity, pressure, temperature and salinity
v_file = File(folder+"velocity.pvd")
v_file.write(v_)
def stepper(self, t_start, t_end, w, t_visualization):
""" Timesteps the shallow water equations from t_start to t_end using a 3rd order SSP Runge-Kutta scheme
:param t_start: start time
:type t_start: float
:param t_end: end time
:type t_end: float
:param w: Current state vector function
:param t_visualization: time interval to write the free surface water depth to a .pvd file
:type t_visualization: float
"""
self.w = w
# setup the solver - this is a timed stage for profiling reasons!
with timed_stage("Setup of forms and solver"):
self.__solver_setup()
# used as the original state vector in each RK3 step
self.w_old = Function(self.V).assign(self.w)
# initial slope modification
self.__update_slope_modification()
hout = Function(self.v_h)
hout.rename("free surface depth")
# free surface depth
hout_file = File("h.pvd")
# bed depth
bout = Function(self.v_h).project(self.b_)
bout.rename("topography")
bout_file = File("b.pvd")
self.Project = Projector(
conditional(self.h <= (self.plot_tol * self.E), self.b_,
self.h + self.b_), hout)
self.Project.project()
hout_file.write(hout)
bout_file.write(bout)
self.t = t_start
# start counter of how many time dumps
self.c = 1
# warning boolean marker
self.wmark = 0
while self.t < t_end:
# find new timestep
with timed_stage("Finding adaptive time-step"):
self.dt = self.AT.FindTimestep(self.w)
self.Dt.assign(self.dt)
# check that prescribed timestep doesn't fall below minimum timestep
if self.dt < self.MinTimestep:
if self.wmark is False:
warning(RED % "Minimum timestep has been reached. " +
"Simulation might become unstable.")
self.wmark = 1
self.dt = self.MinTimestep
self.Dt.assign(self.dt)
# check if remaining time to next time dump is less than timestep
# correct if neeeded
if self.dt + self.t > self.c * t_visualization:
self.dt = (self.c * t_visualization) - self.t
self.Dt.assign(self.dt)
# check if remaining time to end time is less than timestep
# correct if needed
if (self.t <=
(self.c + 1) * t_visualization) and (self.dt + self.t > t_end):
self.dt = t_end - self.t
self.Dt.assign(self.dt)
with timed_stage("Runge-Kutta time-stepping scheme"):
self.solver.solve()
self.w.assign(self.w_)
# slope limiter
with timed_stage("Slope limiting"):
self.__update_slope_limiter()
# slope modification
with timed_stage("Slope modification"):
self.__update_slope_modification()
self.solver.solve()
self.w.assign((3.0 / 4.0) * self.w_old + (1.0 / 4.0) * self.w_)
# slope limiter
with timed_stage("Slope limiting"):
self.__update_slope_limiter()
# slope modification
with timed_stage("Slope modification"):
self.__update_slope_modification()
self.solver.solve()
self.w.assign((1.0 / 3.0) * self.w_old + (2.0 / 3.0) * self.w_)
# slope limiter
with timed_stage("Slope limiting"):
self.__update_slope_limiter()
# slope modification
with timed_stage("Slope modification"):
self.__update_slope_modification()
self.w_old.assign(self.w)
# timstep complete - dump realisation if needed
self.t += self.dt
with timed_stage("Visualization"):
if self.t == self.c * t_visualization:
self.Project.project()
hout_file.write(hout)
bout_file.write(bout)
self.c += 1
self.dt = self.initial_dt
self.Dt.assign(self.dt)
# return timestep
self.dt = self.initial_dt
self.Dt.assign(self.dt)
return self.w
def run_simulation(self, tmax, write=False, dumpfreq=100):
PETSc.Sys.Print("""
Running Williamson 5 test with parameters:\n
method: %s,\n
model degree: %d,\n
refinements: %d,\n
hybridization: %s,\n
tmax: %s
""" % (self.method, self.model_degree, self.refinement_level,
self.hybridization, tmax))
t = 0.0
un, Dn = self.state
up, Dp = self.updates
deltau, deltaD = self.DU.split()
dumpcount = dumpfreq
if write:
dumpcount = self.write(dumpcount, dumpfreq)
self.Dsolver.snes.setConvergenceHistory()
self.Dsolver.snes.ksp.setConvergenceHistory()
self.Usolver.snes.setConvergenceHistory()
self.Usolver.snes.ksp.setConvergenceHistory()
self.DUsolver.snes.setConvergenceHistory()
self.DUsolver.snes.ksp.setConvergenceHistory()
while t < tmax - self.Dt / 2:
t += self.Dt
up.assign(un)
Dp.assign(Dn)
for i in range(4):
self.sim_time.append(t)
self.picard_seq.append(i + 1)
with timed_stage("DU Residuals"):
self.Dsolver.solve()
self.Usolver.solve()
with timed_stage("Linear solve"):
self.DUsolver.solve()
# Here we collect the reductions in the linear residual.
# Get rhs from ksp
r0 = self.DUsolver.snes.ksp.getRhs()
# Assemble the problem residual (b - Ax)
res = assemble(self.FuD, mat_type="aij")
bnorm = r0.norm()
rnorm = res.dat.norm
r_factor = rnorm / bnorm
self.reductions.append(r_factor)
up += deltau
Dp += deltaD
outer_ksp = self.DUsolver.snes.ksp
if self.hybridization:
ctx = outer_ksp.getPC().getPythonContext()
inner_ksp = ctx.trace_ksp
else:
ksps = outer_ksp.getPC().getFieldSplitSubKSP()
_, inner_ksp = ksps
# Collect ksp iterations
self.ksp_outer_its.append(outer_ksp.getIterationNumber())
self.ksp_inner_its.append(inner_ksp.getIterationNumber())
un.assign(up)
Dn.assign(Dp)
if write:
with timed_stage("Dump output"):
dumpcount = self.write(dumpcount, dumpfreq)
def apply(self, field):
with timed_stage('limiter'):
super(VertexBasedP1DGLimiter, self).apply(field)
Planet radius (m): %s,\n
Atmospheric lid (m) %s,\n
Mixed method: %s,\n
Model degree: %s,\n
Hybridization: %s,\n
Horizontal CFL: %s,\n
Dt (s): %s,\n
Dx (km): %s,\n
Dz (m): %s,\n
Solver rtol: %s,\n
KSP monitor: %s.
""" % (solver._R, solver.H, solver.method, solver.model_degree,
solver.hybridization, solver.courant, solver.Dt, solver.dx_max / 1000,
solver.dz, solver.rtol, solver.monitor))
with timed_stage("Warm up"):
solver.warmup()
PETSc.Sys.Print("""Warm up complete. Profiling linear solver.""")
solver.run_profile()
PETSc.Sys.Print("""Run complete. Extracting data.""")
PETSc.Log.Stage("UP Solver").push()
comm = solver.comm
snes = PETSc.Log.Event("SNESSolve").getPerfInfo()
ksp = PETSc.Log.Event("KSPSolve").getPerfInfo()
pcsetup = PETSc.Log.Event("PCSetUp").getPerfInfo()
type=int,
default=1,
help='Telescope factor for telescoping solvers (set to number of NODES)')
args, unknown = parser.parse_known_args()
results = Path(args.resultsdir)
if (not results.is_dir()) and (COMM_WORLD.rank == 0):
try:
results.mkdir()
except FileExistsError:
print('File', cwd,
'already exists, cannot create directory with the same name')
csvfile = ResultsCSV(args)
with timed_stage('Mesh_hierarchy'):
distribution_parameters = {
"partition": True,
"overlap_type": (DistributedMeshOverlapType.VERTEX, 1)
}
mesh = BoxMesh(args.baseN,
args.baseN,
args.baseN,
1,
1,
1,
distribution_parameters=distribution_parameters)
hierarchy = MeshHierarchy(mesh, args.nref)
mesh = hierarchy[-1]
with timed_stage('Problem_setup'):
tp.M = amd.p1metric
else:
tp.indicate_error()
print("Error estimator: {:.4e}".format(tp.estimators['dwr']))
if i > 0 and np.abs(tp.estimators['dwr'] -
estimator_old) < estimator_rtol * estimator_old:
print("Number of elements: ", tp.mesh.num_cells())
print("Number of dofs: ", sum(tp.V.dof_count))
print("Converged error estimator!")
break
qoi_old = tp.qoi
num_cells_old = tp.mesh.num_cells()
estimator_old = tp.estimators['dwr']
# Adapt mesh
with timed_stage('mesh adapt {:d}'.format(i)):
tp.adapt_mesh()
mh = MeshHierarchy(tp.mesh, 1)
sol = tp.solution
print('\n' + 80 * '#')
print('SUMMARY')
print(80 * '#' + '\n')
print('Approach: {:s}'.format(op.approach))
print('Target: {:.1f}'.format(op.target))
print("Number of elements: {:d}".format(tp.mesh.num_cells()))
print("Number of dofs: {:d}".format(sum(tp.V.dof_count)))
print("Quantity of interest: {:.4e}".format(tp.qoi))
print(80 * '#')
#depth_profile_to_csv(depth_profile500m, velocity_depth_profile500m, "500m", '0.0')
#depth_profile_to_csv(depth_profile1km, velocity_depth_profile1km, "1km", '0.0')
#depth_profile_to_csv(depth_profile2km, velocity_depth_profile2km, "2km", '0.0')
#depth_profile_to_csv(depth_profile4km, velocity_depth_profile4km, "4km", '0.0')
#depth_profile_to_csv(depth_profile6km, velocity_depth_profile6km, "6km", '0.0')
########
# Begin time stepping
t = 0.0
step = 0
while t < T - 0.5*dt:
with timed_stage('velocity-pressure'):
vp_timestepper.advance(t)
#u_timestepper.advance(t)
with timed_stage('temperature'):
temp_timestepper.advance(t)
with timed_stage('salinity'):
sal_timestepper.advance(t)
step += 1
t += dt
with timed_stage('output'):
if step % output_step == 0:
# dumb checkpoint for starting from spin up
tape.add_block(DiagnosticBlock(adj_s_file, sal))
tape.add_block(DiagnosticBlock(adj_t_file, temp))
def stepper(self, t_start, t_end, w, t_visualization):
""" Timesteps the shallow water equations from t_start to t_end using a 3rd order SSP Runge-Kutta scheme
:param t_start: start time
:type t_start: float
:param t_end: end time
:type t_end: float
:param w: Current state vector function
:param t_visualization: time interval to write the free surface water depth to a .pvd file
:type t_visualization: float
"""
self.w = w
# setup the solver - this is a timed stage for profiling reasons!
with timed_stage("Setup of forms and solver"):
self.__solver_setup()
# used as the original state vector in each RK3 step
self.w_old = Function(self.V).assign(self.w)
# initial slope modification
self.__update_slope_modification()
hout = Function(self.v_h)
hout.rename("free surface depth")
# free surface depth
hout_file = File("h.pvd")
# bed depth
bout = Function(self.v_h).project(self.b_)
bout.rename("topography")
bout_file = File("b.pvd")
self.Project = Projector(conditional(self.h <= (self.plot_tol * self.E), self.b_, self.h + self.b_), hout)
self.Project.project()
hout_file.write(hout)
bout_file.write(bout)
self.t = t_start
# start counter of how many time dumps
self.c = 1
# warning boolean marker
self.wmark = 0
while self.t < t_end:
# find new timestep
with timed_stage("Finding adaptive time-step"):
self.dt = self.AT.FindTimestep(self.w)
self.Dt.assign(self.dt)
# check that prescribed timestep doesn't fall below minimum timestep
if self.dt < self.MinTimestep:
if self.wmark is False:
warning(RED % "Minimum timestep has been reached. " +
"Simulation might become unstable.")
self.wmark = 1
self.dt = self.MinTimestep
self.Dt.assign(self.dt)
# check if remaining time to next time dump is less than timestep
# correct if neeeded
if self.dt + self.t > self.c * t_visualization:
self.dt = (self.c * t_visualization) - self.t
self.Dt.assign(self.dt)
# check if remaining time to end time is less than timestep
# correct if needed
if (self.t <= (self.c + 1) * t_visualization) and (self.dt + self.t > t_end):
self.dt = t_end - self.t
self.Dt.assign(self.dt)
with timed_stage("Runge-Kutta time-stepping scheme"):
self.solver.solve()
self.w.assign(self.w_)
# slope limiter
with timed_stage("Slope limiting"):
self.__update_slope_limiter()
# slope modification
with timed_stage("Slope modification"):
self.__update_slope_modification()
self.solver.solve()
self.w.assign((3.0 / 4.0) * self.w_old + (1.0 / 4.0) * self.w_)
# slope limiter
with timed_stage("Slope limiting"):
self.__update_slope_limiter()
# slope modification
with timed_stage("Slope modification"):
self.__update_slope_modification()
self.solver.solve()
self.w.assign((1.0 / 3.0) * self.w_old + (2.0 / 3.0) * self.w_)
# slope limiter
with timed_stage("Slope limiting"):
self.__update_slope_limiter()
# slope modification
with timed_stage("Slope modification"):
self.__update_slope_modification()
self.w_old.assign(self.w)
# timstep complete - dump realisation if needed
self.t += self.dt
with timed_stage("Visualization"):
if self.t == self.c * t_visualization:
self.Project.project()
hout_file.write(hout)
bout_file.write(bout)
self.c += 1
self.dt = self.initial_dt
self.Dt.assign(self.dt)
# return timestep
self.dt = self.initial_dt
self.Dt.assign(self.dt)
return self.w
def run_williamson5(problem_cls,
Dt,
refinements,
method,
model_degree,
mesh_degree,
nsteps,
hybridization,
write=False,
cold=False):
# Radius of the Earth (m)
R = 6371220.0
# Max depth height (m)
H = 5960.0
if cold:
PETSc.Sys.Print("""
Running cold initialization for the problem set:\n
method: %s,\n
model degree: %s,\n
hybridization: %s,\n
""" % (method, model_degree, bool(hybridization)))
problem = problem_cls(refinement_level=refinements,
R=R,
H=H,
Dt=Dt,
method=method,
hybridization=hybridization,
model_degree=model_degree,
mesh_degree=mesh_degree,
monitor=args.monitor)
problem.warmup()
return
problem = problem_cls(refinement_level=refinements,
R=R,
H=H,
Dt=Dt,
method=method,
hybridization=hybridization,
model_degree=model_degree,
mesh_degree=mesh_degree,
monitor=args.monitor)
cfl = problem.courant
dx_max = problem.dx_max
PETSc.Sys.Print("""
Dt = %s,\n
Courant number (approximate): %s,\n
Dx (max): %s km,\n
nsteps: %s.
""" % (Dt, cfl, dx_max / 1000, nsteps))
comm = problem.comm
day = 24. * 60. * 60.
if args.profile:
tmax = nsteps * Dt
else:
tmax = 15 * day
PETSc.Sys.Print("Running 15 day simulation\n")
# If writing simulation output, write out fields in 5-day intervals
dumpfreq = 5 * day / Dt
PETSc.Sys.Print("Warm up with one-step.\n")
with timed_stage("Warm up"):
problem.warmup()
PETSc.Log.Stage("Warm up: Linear solve").push()
prepcsetup = PETSc.Log.Event("PCSetUp").getPerfInfo()
pre_res_eval = PETSc.Log.Event("SNESFunctionEval").getPerfInfo()
pre_jac_eval = PETSc.Log.Event("SNESJacobianEval").getPerfInfo()
pre_res_eval_time = comm.allreduce(pre_res_eval["time"],
op=MPI.SUM) / comm.size
pre_jac_eval_time = comm.allreduce(pre_jac_eval["time"],
op=MPI.SUM) / comm.size
pre_setup_time = comm.allreduce(prepcsetup["time"],
op=MPI.SUM) / comm.size
if problem.hybridization:
prehybridinit = PETSc.Log.Event("HybridInit").getPerfInfo()
prehybridinit_time = comm.allreduce(prehybridinit["time"],
op=MPI.SUM) / comm.size
PETSc.Log.Stage("Warm up: Linear solve").pop()
PETSc.Sys.Print("Warm up done. Profiling run for %d steps.\n" % nsteps)
problem.initialize()
problem.run_simulation(tmax, write=write, dumpfreq=dumpfreq)
PETSc.Sys.Print("Simulation complete.\n")
PETSc.Log.Stage("Linear solve").push()
snes = PETSc.Log.Event("SNESSolve").getPerfInfo()
ksp = PETSc.Log.Event("KSPSolve").getPerfInfo()
pcsetup = PETSc.Log.Event("PCSetUp").getPerfInfo()
pcapply = PETSc.Log.Event("PCApply").getPerfInfo()
jac_eval = PETSc.Log.Event("SNESJacobianEval").getPerfInfo()
residual = PETSc.Log.Event("SNESFunctionEval").getPerfInfo()
snes_time = comm.allreduce(snes["time"], op=MPI.SUM) / comm.size
ksp_time = comm.allreduce(ksp["time"], op=MPI.SUM) / comm.size
pc_setup_time = comm.allreduce(pcsetup["time"], op=MPI.SUM) / comm.size
pc_apply_time = comm.allreduce(pcapply["time"], op=MPI.SUM) / comm.size
jac_eval_time = comm.allreduce(jac_eval["time"], op=MPI.SUM) / comm.size
res_eval_time = comm.allreduce(residual["time"], op=MPI.SUM) / comm.size
ref = problem.refinement_level
num_cells = comm.allreduce(problem.num_cells, op=MPI.SUM)
if problem.hybridization:
results_data = "results/hybrid_%s_data_W5_ref%d_Dt%s_NS%d.csv" % (
problem.method, ref, Dt, nsteps)
results_timings = "results/hybrid_%s_profile_W5_ref%d_Dt%s_NS%d.csv" % (
problem.method, ref, Dt, nsteps)
RHS = PETSc.Log.Event("HybridRHS").getPerfInfo()
trace = PETSc.Log.Event("SCSolve").getPerfInfo()
proj = PETSc.Log.Event("HybridProject").getPerfInfo()
full_recon = PETSc.Log.Event("SCBackSub").getPerfInfo()
hybridbreak = PETSc.Log.Event("HybridBreak").getPerfInfo()
hybridupdate = PETSc.Log.Event("HybridUpdate").getPerfInfo()
hybridinit = PETSc.Log.Event("HybridInit").getPerfInfo()
# Time to reconstruct (backsub) and project
full_recon_time = comm.allreduce(full_recon["time"],
op=MPI.SUM) / comm.size
# Project only
projection = comm.allreduce(proj["time"], op=MPI.SUM) / comm.size
# Backsub only = Total Recon time - projection time
recon_time = full_recon_time - projection
transfer = comm.allreduce(hybridbreak["time"], op=MPI.SUM) / comm.size
update_time = comm.allreduce(hybridupdate["time"],
op=MPI.SUM) / comm.size
trace_solve = comm.allreduce(trace["time"], op=MPI.SUM) / comm.size
rhstime = comm.allreduce(RHS["time"], op=MPI.SUM) / comm.size
inittime = comm.allreduce(hybridinit["time"], op=MPI.SUM) / comm.size
other = ksp_time - (trace_solve + transfer + projection + recon_time +
rhstime)
full_solve = (transfer + trace_solve + rhstime + recon_time +
projection + update_time)
else:
results_data = "results/gmres_%s_data_W5_ref%d_Dt%s_NS%d.csv" % (
problem.method, ref, Dt, nsteps)
results_timings = "results/gmres_%s_profile_W5_ref%d_Dt%s_NS%d.csv" % (
problem.method, ref, Dt, nsteps)
KSPSchur = PETSc.Log.Event("KSPSolve_FS_Schu").getPerfInfo()
KSPF0 = PETSc.Log.Event("KSPSolve_FS_0").getPerfInfo()
KSPLow = PETSc.Log.Event("KSPSolve_FS_Low").getPerfInfo()
schur_time = comm.allreduce(KSPSchur["time"], op=MPI.SUM) / comm.size
f0_time = comm.allreduce(KSPF0["time"], op=MPI.SUM) / comm.size
ksplow_time = comm.allreduce(KSPLow["time"], op=MPI.SUM) / comm.size
other = ksp_time - (schur_time + f0_time + ksplow_time)
PETSc.Log.Stage("Linear solve").pop()
if COMM_WORLD.rank == 0:
if not os.path.exists(os.path.dirname('results/')):
os.makedirs(os.path.dirname('results/'))
data = {
"OuterIters": problem.ksp_outer_its,
"InnerIters": problem.ksp_inner_its,
"PicardIters": problem.picard_seq,
"SimTime": problem.sim_time,
"ResidualReductions": problem.reductions
}
dofs = problem.DU.dof_dset.layout_vec.getSize()
time_data = {
"PETSCLogKSPSolve": ksp_time,
"PETSCLogPCApply": pc_apply_time,
"PETSCLogPCSetup": pc_setup_time,
"PETSCLogPreSetup": pre_setup_time,
"PETSCLogPreSNESJacobianEval": pre_jac_eval_time,
"PETSCLogPreSNESFunctionEval": pre_res_eval_time,
"SNESSolve": snes_time,
"SNESFunctionEval": res_eval_time,
"SNESJacobianEval": jac_eval_time,
"num_processes": problem.comm.size,
"method": problem.method,
"model_degree": problem.model_degree,
"refinement_level": problem.refinement_level,
"total_dofs": dofs,
"num_cells": num_cells,
"Dt": Dt,
"CFL": cfl,
"nsteps": nsteps,
"DxMax": dx_max
}
if problem.hybridization:
updates = {
"HybridTraceSolve": trace_solve,
"HybridRHS": rhstime,
"HybridBreak": transfer,
"HybridReconstruction": recon_time,
"HybridProjection": projection,
"HybridFullRecovery": full_recon_time,
"HybridUpdate": update_time,
"HybridInit": inittime,
"PreHybridInit": prehybridinit_time,
"HybridFullSolveTime": full_solve,
"HybridKSPOther": other
}
else:
updates = {
"KSPSchur": schur_time,
"KSPF0": f0_time,
"KSPFSLow": ksplow_time,
"KSPother": other
}
time_data.update(updates)
df_data = pd.DataFrame(data)
df_data.to_csv(results_data, index=False, mode="w", header=True)
df_time = pd.DataFrame(time_data, index=[0])
df_time.to_csv(results_timings, index=False, mode="w", header=True)
