def init_problem(self):
# Initial conditions
self.u = self.solver.state['u']['g']
self.v = self.solver.state['v']['g']
self.w = self.solver.state['w']['g']
z = self.domain.grid(2)
b = self.solver.state['b']
bz = self.solver.state['bz']
# Random perturbations, initialized globally for same results in parallel
gshape = self.domain.dist.grid_layout.global_shape(scales=1)
slices = self.domain.dist.grid_layout.slices(scales=1)
rand = np.random.RandomState(seed=23)
noise = rand.standard_normal(gshape)[slices]
# Linear background + perturbations damped at walls
zb, zt = self.z_basis.interval
pert = 1e-3 * noise * (zt - z) * (z - zb)
b['g'] = -self.F * (z - pert)
b.differentiate('z', out=bz)
# Flow properties
self.CFL = flow_tools.CFL(self.solver,
initial_dt=1e-4,
cadence=5,
safety=0.5,
max_change=1.5,
min_change=0.5,
max_dt=0.05)
self.CFL.add_velocities(('u', 'v', 'w'))
self.flow = flow_tools.GlobalFlowProperty(self.solver, cadence=10)
self.flow.add_property("sqrt(u*u + v*v + w*w) / R", name='Re')
def run(self, initial_dt=1e-4, sim_time=100):
# Integration parameters
self.solver.stop_sim_time = self.solver.sim_time + sim_time
self.solver.stop_wall_time = np.inf # 60 * 60.
self.solver.stop_iteration = np.inf
# CFL
CFL = flow_tools.CFL(self.solver,
initial_dt=initial_dt,
cadence=5,
safety=1.5,
max_change=1.5,
min_change=0.5,
max_dt=0.05)
CFL.add_velocities(('u', 'v', 'w'))
flow = flow_tools.GlobalFlowProperty(self.solver, cadence=10)
flow.add_property("0.5*(u*u + v*v + w*w)", name='KE')
self.CFL = CFL
self.flow = flow
try:
logger.info('Starting loop')
start_run_time = time.time()
while self.solver.ok:
dt = CFL.compute_dt()
self.solver.step(dt)
if (self.solver.iteration - 1) % 100 == 0:
logger.info(
'Iteration: %i, Time: %e, dt: %e' %
(self.solver.iteration, self.solver.sim_time, dt))
logger.info('Average KE = %e' % flow.volume_average('KE'))
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
end_run_time = time.time()
logger.info('Iterations: %i' % self.solver.iteration)
logger.info('Sim end time: %f' % self.solver.sim_time)
logger.info('Run time: %.2f sec' % (end_run_time - start_run_time))
logger.info('Run time: %f cpu-hr' %
((end_run_time - start_run_time) / hour *
self.domain.dist.comm_cart.size))
def Rayleigh_Benard(Rayleigh=1e6,
Prandtl=1,
nz=64,
nx=None,
aspect=4,
fixed_flux=False,
fixed_T=True,
viscous_heating=False,
restart=None,
run_time=23.5,
run_time_buoyancy=50,
run_time_iter=np.inf,
max_writes=10,
max_slice_writes=10,
data_dir='./',
coeff_output=True,
verbose=False,
no_join=False):
import os
from dedalus.tools.config import config
config['logging']['filename'] = os.path.join(data_dir, 'logs/dedalus_log')
config['logging']['file_level'] = 'DEBUG'
import mpi4py.MPI
if mpi4py.MPI.COMM_WORLD.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.makedirs('{:s}/'.format(data_dir))
logdir = os.path.join(data_dir, 'logs')
if not os.path.exists(logdir):
os.mkdir(logdir)
logger = logging.getLogger(__name__)
logger.info("saving run in: {}".format(data_dir))
import time
from dedalus import public as de
from dedalus.extras import flow_tools
from dedalus.tools import post
# input parameters
logger.info("Ra = {}, Pr = {}".format(Rayleigh, Prandtl))
# Parameters
Lz = 1.
Lx = aspect * Lz
if nx is None:
nx = int(nz * aspect)
logger.info("resolution: [{}x{}]".format(nx, nz))
# Create bases and domain
x_basis = de.Fourier('x', nx, interval=(0, Lx), dealias=3 / 2)
z_basis = de.Chebyshev('z', nz, interval=(0, Lz), dealias=3 / 2)
domain = de.Domain([x_basis, z_basis], grid_dtype=np.float64)
if fixed_flux:
T_bc_var = 'Tz'
elif fixed_T:
T_bc_var = 'T'
# 2D Boussinesq hydrodynamics
problem = de.IVP(domain, variables=['Tz', 'T', 'p', 'u', 'w', 'Oy'])
problem.meta['p', T_bc_var, 'Oy', 'w']['z']['dirichlet'] = True
T0_z = domain.new_field()
T0_z.meta['x']['constant'] = True
T0_z['g'] = -1
T0 = domain.new_field()
T0.meta['x']['constant'] = True
T0['g'] = Lz / 2 - domain.grid(-1)
problem.parameters['T0'] = T0
problem.parameters['T0_z'] = T0_z
problem.parameters['P'] = (Rayleigh * Prandtl)**(-1 / 2)
problem.parameters['R'] = (Rayleigh / Prandtl)**(-1 / 2)
problem.parameters['F'] = F = 1
problem.parameters['Lx'] = Lx
problem.parameters['Lz'] = Lz
problem.substitutions['plane_avg(A)'] = 'integ(A, "x")/Lx'
problem.substitutions['vol_avg(A)'] = 'integ(A)/Lx/Lz'
problem.substitutions['v'] = '0'
problem.substitutions['Ox'] = '0'
problem.substitutions['Oz'] = '(dx(v) )'
problem.substitutions['Kx'] = '( -dz(Oy))'
problem.substitutions['Ky'] = '(dz(Ox) - dx(Oz))'
problem.substitutions['Kz'] = '(dx(Oy) )'
problem.substitutions['vorticity'] = 'Oy'
problem.substitutions['enstrophy'] = 'Oy**2'
problem.substitutions['u_fluc'] = '(u - plane_avg(u))'
problem.substitutions['w_fluc'] = '(w - plane_avg(w))'
problem.substitutions['KE'] = '(0.5*(u*u+w*w))'
problem.substitutions['sigma_xz'] = '(dx(w) + Oy + dx(w))'
problem.substitutions['sigma_xx'] = '(2*dx(u))'
problem.substitutions['sigma_zz'] = '(2*dz(w))'
if viscous_heating:
problem.substitutions[
'visc_heat'] = 'R*(sigma_xz**2 + sigma_xx*dx(u) + sigma_zz*dz(w))'
problem.substitutions['visc_flux_z'] = 'R*(u*sigma_xz + w*sigma_zz)'
else:
problem.substitutions['visc_heat'] = '0'
problem.substitutions['visc_flux_z'] = '0'
problem.substitutions['conv_flux_z'] = '(w*T + visc_flux_z)/P'
problem.substitutions['kappa_flux_z'] = '(-Tz)'
problem.add_equation("Tz - dz(T) = 0")
problem.add_equation(
"dt(T) - P*(dx(dx(T)) + dz(Tz)) + w*T0_z = -(u*dx(T) + w*Tz) - visc_heat"
)
# O == omega = curl(u); K = curl(O)
problem.add_equation("dt(u) + R*Kx + dx(p) = v*Oz - w*Oy ")
problem.add_equation("dt(w) + R*Kz + dz(p) -T = u*Oy - v*Ox ")
problem.add_equation("dx(u) + dz(w) = 0")
problem.add_equation("Oy - dz(u) + dx(w) = 0")
problem.add_bc("right(p) = 0", condition="(nx == 0)")
if fixed_flux:
problem.add_bc("left(Tz) = 0")
problem.add_bc("right(Tz) = 0")
elif fixed_T:
problem.add_bc("left(T) = 0")
problem.add_bc("right(T) = 0")
problem.add_bc("left(Oy) = 0")
problem.add_bc("right(Oy) = 0")
problem.add_bc("left(w) = 0")
problem.add_bc("right(w) = 0", condition="(nx != 0)")
# Build solver
ts = de.timesteppers.RK443
cfl_safety = 1
solver = problem.build_solver(ts)
logger.info('Solver built')
checkpoint = solver.evaluator.add_file_handler(data_dir + 'checkpoints',
wall_dt=8 * 3600,
max_writes=1)
checkpoint.add_system(solver.state, layout='c')
# Initial conditions
x = domain.grid(0)
z = domain.grid(1)
T = solver.state['T']
Tz = solver.state['Tz']
# Random perturbations, initialized globally for same results in parallel
noise = global_noise(domain, scale=1, frac=0.25)
if restart is None:
# Linear background + perturbations damped at walls
zb, zt = z_basis.interval
pert = 1e-3 * noise * (zt - z) * (z - zb)
T['g'] = pert
T.differentiate('z', out=Tz)
else:
logger.info("restarting from {}".format(restart))
checkpoint.restart(restart, solver)
# Integration parameters
solver.stop_sim_time = run_time_buoyancy
solver.stop_wall_time = run_time * 3600.
solver.stop_iteration = run_time_iter
# Analysis
analysis_tasks = []
snapshots = solver.evaluator.add_file_handler(data_dir + 'slices',
sim_dt=0.1,
max_writes=max_slice_writes)
snapshots.add_task("T + T0", name='T')
snapshots.add_task("enstrophy")
snapshots.add_task("vorticity")
analysis_tasks.append(snapshots)
snapshots_small = solver.evaluator.add_file_handler(
data_dir + 'slices_small', sim_dt=0.1, max_writes=max_slice_writes)
snapshots_small.add_task("T + T0", name='T', scales=0.25)
snapshots_small.add_task("enstrophy", scales=0.25)
snapshots_small.add_task("vorticity", scales=0.25)
analysis_tasks.append(snapshots_small)
if coeff_output:
coeffs = solver.evaluator.add_file_handler(data_dir + 'coeffs',
sim_dt=0.1,
max_writes=max_slice_writes)
coeffs.add_task("T+T0", name="T", layout='c')
coeffs.add_task("T - plane_avg(T)", name="T'", layout='c')
coeffs.add_task("w", layout='c')
coeffs.add_task("u", layout='c')
coeffs.add_task("enstrophy", layout='c')
coeffs.add_task("vorticity", layout='c')
analysis_tasks.append(coeffs)
profiles = solver.evaluator.add_file_handler(data_dir + 'profiles',
sim_dt=0.1,
max_writes=max_writes)
profiles.add_task("plane_avg(T+T0)", name="T")
profiles.add_task("plane_avg(T)", name="T'")
profiles.add_task("plane_avg(u)", name="u")
profiles.add_task("plane_avg(w)", name="w")
profiles.add_task("plane_avg(enstrophy)", name="enstrophy")
# This may have an error:
profiles.add_task("plane_avg(conv_flux_z)/plane_avg(kappa_flux_z) + 1",
name="Nu")
profiles.add_task("plane_avg(conv_flux_z) + plane_avg(kappa_flux_z)",
name="Nu_2")
analysis_tasks.append(profiles)
scalar = solver.evaluator.add_file_handler(data_dir + 'scalar',
sim_dt=0.1,
max_writes=max_writes)
scalar.add_task("vol_avg(T)", name="IE")
scalar.add_task("vol_avg(KE)", name="KE")
scalar.add_task("vol_avg(-T*z)", name="PE_fluc")
scalar.add_task("vol_avg(T) + vol_avg(KE) + vol_avg(-T*z)", name="TE")
scalar.add_task("0.5*vol_avg(u_fluc*u_fluc+w_fluc*w_fluc)", name="KE_fluc")
scalar.add_task("0.5*vol_avg(u*u)", name="KE_x")
scalar.add_task("0.5*vol_avg(w*w)", name="KE_z")
scalar.add_task("0.5*vol_avg(u_fluc*u_fluc)", name="KE_x_fluc")
scalar.add_task("0.5*vol_avg(w_fluc*w_fluc)", name="KE_z_fluc")
scalar.add_task("vol_avg(plane_avg(u)**2)", name="u_avg")
scalar.add_task("vol_avg((u - plane_avg(u))**2)", name="u1")
scalar.add_task("vol_avg(conv_flux_z) + 1.", name="Nu")
analysis_tasks.append(scalar)
# workaround for issue #29
problem.namespace['enstrophy'].store_last = True
# CFL
CFL = flow_tools.CFL(solver,
initial_dt=0.1,
cadence=1,
safety=cfl_safety,
max_change=1.5,
min_change=0.5,
max_dt=0.1,
threshold=0.1)
CFL.add_velocities(('u', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("sqrt(u*u + w*w) / R", name='Re')
first_step = True
# Main loop
try:
logger.info('Starting loop')
Re_avg = 0
while solver.ok and np.isfinite(Re_avg):
dt = CFL.compute_dt()
solver.step(dt) #, trim=True)
Re_avg = flow.grid_average('Re')
log_string = 'Iteration: {:5d}, '.format(solver.iteration)
log_string += 'Time: {:8.3e}, dt: {:8.3e}, '.format(
solver.sim_time, dt)
log_string += 'Re: {:8.3e}/{:8.3e}'.format(Re_avg, flow.max('Re'))
logger.info(log_string)
if first_step:
if verbose:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.spy(solver.pencils[0].L,
markersize=1,
markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern.png", dpi=1200)
import scipy.sparse.linalg as sla
LU = sla.splu(solver.pencils[0].LHS.tocsc(),
permc_spec='NATURAL')
fig = plt.figure()
ax = fig.add_subplot(1, 2, 1)
ax.spy(LU.L.A, markersize=1, markeredgewidth=0.0)
ax = fig.add_subplot(1, 2, 2)
ax.spy(LU.U.A, markersize=1, markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern_LU.png", dpi=1200)
logger.info("{} nonzero entries in LU".format(LU.nnz))
logger.info("{} nonzero entries in LHS".format(
solver.pencils[0].LHS.tocsc().nnz))
logger.info("{} fill in factor".format(
LU.nnz / solver.pencils[0].LHS.tocsc().nnz))
first_step = False
start_time = time.time()
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
end_time = time.time()
main_loop_time = end_time - start_time
n_iter_loop = solver.iteration - 1
logger.info('Iterations: {:d}'.format(n_iter_loop))
logger.info('Sim end time: {:f}'.format(solver.sim_time))
logger.info('Run time: {:f} sec'.format(main_loop_time))
logger.info('Run time: {:f} cpu-hr'.format(main_loop_time / 60 / 60 *
domain.dist.comm_cart.size))
logger.info('iter/sec: {:f} (main loop only)'.format(n_iter_loop /
main_loop_time))
final_checkpoint = solver.evaluator.add_file_handler(
data_dir + 'final_checkpoint', iter=1)
final_checkpoint.add_system(solver.state, layout='c')
solver.step(dt) #clean this up in the future...works for now.
post.merge_analysis(data_dir + 'final_checkpoint')
if not no_join:
logger.info('beginning join operation')
post.merge_analysis(data_dir + 'checkpoints')
for task in analysis_tasks:
logger.info(task.base_path)
post.merge_analysis(task.base_path)
logger.info(40 * "=")
logger.info('Iterations: {:d}'.format(n_iter_loop))
logger.info('Sim end time: {:f}'.format(solver.sim_time))
logger.info('Run time: {:f} sec'.format(main_loop_time))
logger.info('Run time: {:f} cpu-hr'.format(main_loop_time / 60 / 60 *
domain.dist.comm_cart.size))
logger.info('iter/sec: {:f} (main loop only)'.format(n_iter_loop /
main_loop_time))
timeseries = IVP.evaluator.add_file_handler(os.path.join(
data_dir, 'timeseries'),
iter=10,
max_writes=np.inf)
timeseries.add_task("vol_avg(sqrt(u*u))", name='urms')
timeseries.add_task("vol_avg(sqrt(v*v))", name='vrms')
timeseries.add_task("vol_avg(0.5*(u*u+v*v))", name='Ekin')
timeseries.add_task("vol_avg((u*u)/(u*u+v*v))", name='Ekin_x_ratio')
if mhd:
timeseries.add_task("vol_avg(0.5*(Bx*Bx+By*By))", name='Mag_en')
timeseries.add_task("vol_avg(sqrt(Bx*Bx))", name='Bx')
timeseries.add_task("vol_avg(sqrt(By*By))", name='By')
analysis_tasks = [snapshots, slices, timeseries]
flow = flow_tools.GlobalFlowProperty(IVP, cadence=10)
flow.add_property("0.5*(u*u + v*v)", name="Ekin")
flow.add_property("u", name="u")
flow.add_property("v", name="v")
# Main loop
try:
logger.info('Starting loop')
start_time = time.time()
while IVP.ok:
dt = CFL.compute_dt()
dt = IVP.step(dt)
if (IVP.iteration - 1) % 10 == 0:
logger.info(
'Iteration: %i, Time: %e, dt: %e, Kinetic Energy: %e, u max: %e, v max: %e'
% (IVP.iteration, IVP.sim_time, dt,
flow.volume_average('Ekin'), flow.max('u'), flow.max('v')))
def main():
args = get_args()
# Parameters
Lx, Lz = (args.lx, args.lz)
Prandtl = args.prandtl
Rayleigh = args.rayleigh
seed = args.seed
# Create bases and domain
x_basis = de.Fourier('x', args.res_x, interval=(0, Lx), dealias=3/2) # 256
z_basis = de.Chebyshev('z', args.res_z, interval=(-Lz/2, Lz/2), dealias=3/2) # 64
domain = de.Domain([x_basis, z_basis], grid_dtype=np.float64)
# 2D Boussinesq hydrodynamics
problem = de.IVP(domain, variables=['p','b','u','w','bz','uz','wz'])
problem.meta['p','b','u','w']['z']['dirichlet'] = True
problem.parameters['P'] = (Rayleigh * Prandtl)**(-1/2)
problem.parameters['R'] = (Rayleigh / Prandtl)**(-1/2)
problem.parameters['F'] = F = 1
problem.add_equation("dx(u) + wz = 0")
# problem.add_equation("dt(b) - P*(dx(dx(b)) + dz(bz)) - F*w = -(u*dx(b) + w*bz)")
problem.add_equation("dt(b) - P*(dx(dx(b)) + dz(bz)) = -(u*dx(b) + w*bz)")
problem.add_equation("dt(u) - R*(dx(dx(u)) + dz(uz)) + dx(p) = -(u*dx(u) + w*uz)")
problem.add_equation("dt(w) - R*(dx(dx(w)) + dz(wz)) + dz(p) - b = -(u*dx(w) + w*wz)")
problem.add_equation("bz - dz(b) = 0")
problem.add_equation("uz - dz(u) = 0")
problem.add_equation("wz - dz(w) = 0")
problem.add_bc("left(b) = 0.5")
problem.add_bc("left(u) = 0")
problem.add_bc("left(w) = 0")
problem.add_bc("right(b) = -0.5")
problem.add_bc("right(u) = 0")
problem.add_bc("right(w) = 0", condition="(nx != 0)")
problem.add_bc("right(p) = 0", condition="(nx == 0)")
# Build solver
solver = problem.build_solver(de.timesteppers.RK222)
logger.info('Solver built')
# Initial conditions or restart
if not pathlib.Path('restart.h5').exists():
# Initial conditions
x = domain.grid(0)
z = domain.grid(1)
b = solver.state['b']
bz = solver.state['bz']
# Random perturbations, initialized globally for same results in parallel
gshape = domain.dist.grid_layout.global_shape(scales=1)
slices = domain.dist.grid_layout.slices(scales=1)
rand = np.random.RandomState(seed=seed)
noise = rand.standard_normal(gshape)[slices]
# Linear background + perturbations damped at walls
zb, zt = z_basis.interval
pert = 1e-3 * noise * (zt - z) * (z - zb)
b['g'] += F * pert
b.differentiate('z', out=bz)
# Timestepping and output
dt = args.dt
stop_sim_time = args.stop_sim_time
fh_mode = 'overwrite'
else:
# Restart
write, last_dt = solver.load_state('restart.h5', -1)
# Timestepping and output
dt = last_dt
stop_sim_time = args.stop_sim_time
fh_mode = 'append'
# Integration parameters
solver.stop_sim_time = stop_sim_time
# Analysis
snapshots = solver.evaluator.add_file_handler('snapshots', sim_dt=0.05, max_writes=250, mode=fh_mode)
snapshots.add_system(solver.state)
# CFL
CFL = flow_tools.CFL(solver, initial_dt=dt, cadence=1, safety=1,
max_change=1.5, min_change=0., max_dt=0.125, threshold=0.05)
CFL.add_velocities(('u', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=10)
flow.add_property("sqrt(u*u + w*w) / R", name='Re')
# Main loop
try:
logger.info('Starting loop')
start_time = time.time()
while solver.proceed:
dt = CFL.compute_dt()
dt = solver.step(dt)
if (solver.iteration-1) % 10 == 0:
logger.info('Iteration: %i, Time: %e, dt: %e' %(solver.iteration, solver.sim_time, dt))
logger.info('Max Re = %f' %flow.max('Re'))
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
end_time = time.time()
logger.info('Iterations: %i' %solver.iteration)
logger.info('Sim end time: %f' %solver.sim_time)
logger.info('Run time: %.2f sec' %(end_time-start_time))
logger.info('Run time: %f cpu-hr' %((end_time-start_time)/60/60*domain.dist.comm_cart.size))
traces = solver.evaluator.add_file_handler(data_dir + '/traces',
sim_dt=0.1,
max_writes=np.inf)
traces.add_task(avg(0.5 * ρ * dot(u, u)), name='KE')
traces.add_task(avg(IE), name='IE')
traces.add_task(avg(Re), name='Re')
traces.add_task(avg(ω**2), name='enstrophy')
traces.add_task(x_avg(np.sqrt(dot(τu1, τu1))), name='τu1')
traces.add_task(x_avg(np.sqrt(dot(τu2, τu2))), name='τu2')
traces.add_task(x_avg(np.sqrt(τs1**2)), name='τs1')
traces.add_task(x_avg(np.sqrt(τs2**2)), name='τs2')
report_cadence = 10
good_solution = True
flow = flow_tools.GlobalFlowProperty(solver, cadence=report_cadence)
flow.add_property(Re, name='Re')
flow.add_property(KE, name='KE')
flow.add_property(IE, name='IE')
flow.add_property(τu1, name='τu1')
flow.add_property(τu2, name='τu2')
flow.add_property(τs1, name='τs1')
flow.add_property(τs2, name='τs2')
KE_avg = 0
while solver.proceed and good_solution:
# advance
solver.step(Δt)
if solver.iteration % report_cadence == 0:
KE_avg = flow.grid_average('KE')
IE_avg = flow.grid_average('IE')
def FC_convection(Rayleigh=1e6, Prandtl=1, stiffness=1e4, m_rz=3, gamma=5/3,
n_rho_cz=3.5, n_rho_rz=1,
nz_cz=128, nz_rz=128,
nx = None,
width=None,
single_chebyshev=False,
rk222=False,
superstep=False,
dense=False, nz_dense=64,
oz=False,
fixed_flux=False,
run_time=23.5, run_time_buoyancies=np.inf, run_time_iter=np.inf,
dynamic_diffusivities=False,
max_writes=20,out_cadence=0.1, no_coeffs=False, no_join=False,
restart=None, data_dir='./', verbose=False, label=None):
def format_number(number, no_format_min=0.1, no_format_max=10):
if number > no_format_max or number < no_format_min:
try:
mantissa = "{:e}".format(number).split("+")[0].split("e")[0].rstrip("0") or "0"
power = "{:e}".format(number).split("+")[1].lstrip("0") or "0"
except:
mantissa = "{:e}".format(number).split("-")[0].split("e")[0].rstrip("0") or "0"
power = "{:e}".format(number).split("-")[1].lstrip("0") or "0"
power = "-"+power
if mantissa[-1]==".":
mantissa = mantissa[:-1]
mantissa += "e"
else:
mantissa = "{:f}".format(number).rstrip("0") or "0"
if mantissa[-1]==".":
mantissa = mantissa[:-1]
power = ""
number_string = mantissa+power
return number_string
# save data in directory named after script
if data_dir[-1] != '/':
data_dir += '/'
data_dir += sys.argv[0].split('.py')[0]
if fixed_flux:
data_dir += '_flux'
if dynamic_diffusivities:
data_dir += '_dynamic'
if oz:
data_dir += '_oz'
data_dir += "_nrhocz{}_Ra{}_S{}".format(format_number(n_rho_cz), format_number(Rayleigh), format_number(stiffness))
if width:
data_dir += "_erf{}".format(format_number(width))
if label:
data_dir += "_{}".format(label)
data_dir += '/'
from dedalus.tools.config import config
config['logging']['filename'] = os.path.join(data_dir,'logs/dedalus_log')
config['logging']['file_level'] = 'DEBUG'
import mpi4py.MPI
if mpi4py.MPI.COMM_WORLD.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.makedirs('{:s}/'.format(data_dir))
logdir = os.path.join(data_dir,'logs')
if not os.path.exists(logdir):
os.mkdir(logdir)
logger = logging.getLogger(__name__)
logger.info("saving run in: {}".format(data_dir))
import dedalus.public as de
from dedalus.tools import post
from dedalus.extras import flow_tools
from dedalus.core.future import FutureField
from stratified_dynamics import multitropes
from tools.checkpointing import Checkpoint
checkpoint_min = 30
initial_time = time.time()
logger.info("Starting Dedalus script {:s}".format(sys.argv[0]))
constant_Prandtl=True
mixed_temperature_flux=None
if oz:
stable_top=True
if not fixed_flux:
mixed_temperature_flux=True
else:
stable_top=False
# Set domain
if nx is None:
nx = nz_cz*4
if single_chebyshev:
nz = nz_cz
nz_list = [nz_cz]
else:
if dense:
nz = nz_rz+nz_dense+nz_cz
#nz_list = [nz_rz, int(nz_dense/2), int(nz_dense/2), nz_cz]
nz_list = [nz_rz, nz_dense, nz_cz]
else:
nz = nz_rz+nz_cz
nz_list = [nz_rz, nz_cz]
if dynamic_diffusivities:
atmosphere = multitropes.FC_multitrope_2d_kappa_mu(nx=nx, nz=nz_list, stiffness=stiffness, m_rz=m_rz, gamma=gamma,
n_rho_cz=n_rho_cz, n_rho_rz=n_rho_rz,
verbose=verbose, width=width,
constant_Prandtl=constant_Prandtl,
stable_top=stable_top)
else:
atmosphere = multitropes.FC_multitrope(nx=nx, nz=nz_list, stiffness=stiffness, m_rz=m_rz, gamma=gamma,
n_rho_cz=n_rho_cz, n_rho_rz=n_rho_rz,
verbose=verbose, width=width,
constant_Prandtl=constant_Prandtl,
stable_top=stable_top)
atmosphere.set_IVP_problem(Rayleigh, Prandtl)
atmosphere.set_BC(mixed_temperature_flux=mixed_temperature_flux, fixed_flux=fixed_flux)
problem = atmosphere.get_problem()
if atmosphere.domain.distributor.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.makedirs('{:s}/'.format(data_dir))
if rk222:
logger.info("timestepping using RK222")
ts = de.timesteppers.RK222
cfl_safety_factor = 0.2*2
else:
logger.info("timestepping using RK443")
ts = de.timesteppers.RK443
cfl_safety_factor = 0.2*4
# Build solver
solver = problem.build_solver(ts)
# initial conditions
if restart is None:
mode = "overwrite"
else:
mode = "append"
logger.info("checkpointing in {}".format(data_dir))
checkpoint = Checkpoint(data_dir)
if restart is None:
atmosphere.set_IC(solver)
dt = None
else:
logger.info("restarting from {}".format(restart))
dt = checkpoint.restart(restart, solver)
checkpoint.set_checkpoint(solver, wall_dt=checkpoint_min*60, mode=mode)
logger.info("thermal_time = {:g}, top_thermal_time = {:g}".format(atmosphere.thermal_time, atmosphere.top_thermal_time))
max_dt = atmosphere.min_BV_time
max_dt = atmosphere.buoyancy_time*out_cadence
if dt is None: dt = max_dt/5
report_cadence = 1
output_time_cadence = out_cadence*atmosphere.buoyancy_time
solver.stop_sim_time = solver.sim_time + run_time_buoyancies*atmosphere.buoyancy_time
solver.stop_iteration = solver.iteration + run_time_iter
solver.stop_wall_time = run_time*3600
logger.info("output cadence = {:g}".format(output_time_cadence))
analysis_tasks = atmosphere.initialize_output(solver, data_dir, coeffs_output=not(no_coeffs), sim_dt=output_time_cadence, max_writes=max_writes, mode=mode)
cfl_cadence = 1
CFL = flow_tools.CFL(solver, initial_dt=dt, cadence=cfl_cadence, safety=cfl_safety_factor,
max_change=1.5, min_change=0.5, max_dt=max_dt, threshold=0.1)
if superstep:
CFL_traditional = flow_tools.CFL(solver, initial_dt=max_dt, cadence=cfl_cadence, safety=cfl_safety_factor,
max_change=1.5, min_change=0.5, max_dt=max_dt, threshold=0.1)
CFL_traditional.add_velocities(('u', 'w'))
vel_u = FutureField.parse('u', CFL.solver.evaluator.vars, CFL.solver.domain)
delta_x = atmosphere.Lx/nx
CFL.add_frequency(vel_u/delta_x)
vel_w = FutureField.parse('w', CFL.solver.evaluator.vars, CFL.solver.domain)
mean_delta_z_cz = atmosphere.Lz_cz/nz_cz
CFL.add_frequency(vel_w/mean_delta_z_cz)
else:
CFL.add_velocities(('u', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("Re_rms", name='Re')
try:
logger.info("starting main loop")
start_time = time.time()
start_iter = solver.iteration
good_solution = True
first_step = True
while solver.ok and good_solution:
dt = CFL.compute_dt()
# advance
solver.step(dt)
effective_iter = solver.iteration - start_iter
# update lists
if effective_iter % report_cadence == 0:
Re_avg = flow.grid_average('Re')
log_string = 'Iteration: {:5d}, Time: {:8.3e} ({:8.3e}), '.format(solver.iteration, solver.sim_time, solver.sim_time/atmosphere.buoyancy_time)
log_string += 'dt: {:8.3e}'.format(dt)
if superstep:
dt_traditional = CFL_traditional.compute_dt()
log_string += ' (vs {:8.3e})'.format(dt_traditional)
log_string += ', '
log_string += 'Re: {:8.3e}/{:8.3e}'.format(Re_avg, flow.max('Re'))
logger.info(log_string)
if not np.isfinite(Re_avg):
good_solution = False
logger.info("Terminating run. Trapped on Reynolds = {}".format(Re_avg))
if first_step:
if verbose:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(1,1,1)
ax.spy(solver.pencils[0].L, markersize=0.5, markeredgewidth=0.0)
fig.savefig(data_dir+"sparsity_pattern.png", dpi=2400)
#fig.savefig(data_dir+"sparsity_pattern.svg", format="svg")
import scipy.sparse.linalg as sla
LU = sla.splu(solver.pencils[0].LHS.tocsc(), permc_spec='NATURAL')
fig = plt.figure()
ax = fig.add_subplot(1,2,1)
ax.spy(LU.L.A, markersize=1, markeredgewidth=0.0)
ax = fig.add_subplot(1,2,2)
ax.spy(LU.U.A, markersize=1, markeredgewidth=0.0)
fig.savefig(data_dir+"sparsity_pattern_LU.png", dpi=1200)
#fig.savefig(data_dir+"sparsity_pattern_LU.svg", format="svg")
logger.info("{} nonzero entries in LU".format(LU.nnz))
logger.info("{} nonzero entries in LHS".format(solver.pencils[0].LHS.tocsc().nnz))
logger.info("{} fill in factor".format(LU.nnz/solver.pencils[0].LHS.tocsc().nnz))
first_step = False
start_time = time.time()
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
end_time = time.time()
# Print statistics
elapsed_time = end_time - start_time
elapsed_sim_time = solver.sim_time
N_iterations = solver.iteration - 1
logger.info('main loop time: {:e}'.format(elapsed_time))
logger.info('Iterations: {:d}'.format(N_iterations))
logger.info('iter/sec: {:g}'.format(N_iterations/(elapsed_time)))
if N_iterations > 0:
logger.info('Average timestep: {:e}'.format(elapsed_sim_time / N_iterations))
logger.info('beginning join operation')
try:
final_checkpoint = Checkpoint(data_dir, checkpoint_name='final_checkpoint')
final_checkpoint.set_checkpoint(solver, wall_dt=1, mode="append")
solver.step(dt) #clean this up in the future...works for now.
post.merge_process_files(data_dir+'/final_checkpoint/')
except:
print('cannot save final checkpoint')
if not(no_join):
logger.info(data_dir+'/checkpoint/')
post.merge_process_files(data_dir+'/checkpoint/')
for task in analysis_tasks:
logger.info(analysis_tasks[task].base_path)
post.merge_process_files(analysis_tasks[task].base_path)
if (atmosphere.domain.distributor.rank==0):
N_TOTAL_CPU = atmosphere.domain.distributor.comm_cart.size
# Print statistics
print('-' * 40)
total_time = end_time-initial_time
main_loop_time = end_time - start_time
startup_time = start_time-initial_time
n_steps = solver.iteration-1
print(' startup time:', startup_time)
print('main loop time:', main_loop_time)
print(' total time:', total_time)
if n_steps > 0:
print(' iterations:', n_steps)
print(' loop sec/iter:', main_loop_time/n_steps)
print(' average dt:', solver.sim_time/n_steps)
print(" N_cores, Nx, Nz, startup main loop, main loop/iter, main loop/iter/grid, n_cores*main loop/iter/grid")
print('scaling:',
' {:d} {:d} {:d}'.format(N_TOTAL_CPU,nx,nz),
' {:8.3g} {:8.3g} {:8.3g} {:8.3g} {:8.3g}'.format(startup_time,
main_loop_time,
main_loop_time/n_steps,
main_loop_time/n_steps/(nx*nz),
N_TOTAL_CPU*main_loop_time/n_steps/(nx*nz)))
print('-' * 40)
return data_dir
def FC_polytrope(Rayleigh=1e4,
Prandtl=1,
aspect_ratio=4,
Taylor=None,
theta=0,
nz=128,
nx=None,
ny=None,
threeD=False,
mesh=None,
n_rho_cz=3,
epsilon=1e-4,
gamma=5 / 3,
run_time=23.5,
run_time_buoyancies=None,
run_time_iter=np.inf,
fixed_T=False,
fixed_flux=False,
mixed_flux_T=False,
const_mu=True,
const_kappa=True,
dynamic_diffusivities=False,
split_diffusivities=False,
restart=None,
start_new_files=False,
rk222=False,
safety_factor=0.2,
max_writes=20,
no_slip=False,
data_dir='./',
out_cadence=0.1,
no_coeffs=False,
no_volumes=False,
no_join=False,
verbose=False):
import dedalus.public as de
from dedalus.tools import post
from dedalus.extras import flow_tools
import time
import os
import sys
from stratified_dynamics import polytropes
from tools.checkpointing import Checkpoint
checkpoint_min = 30
initial_time = time.time()
logger.info("Starting Dedalus script {:s}".format(sys.argv[0]))
if nx is None:
nx = int(np.round(nz * aspect_ratio))
if threeD and ny is None:
ny = nx
if threeD:
atmosphere = polytropes.FC_polytrope_3d(nx=nx, ny=ny, nz=nz, mesh=mesh, constant_kappa=const_kappa, constant_mu=const_mu,\
epsilon=epsilon, gamma=gamma, n_rho_cz=n_rho_cz, aspect_ratio=aspect_ratio,\
fig_dir=data_dir)
else:
if dynamic_diffusivities:
atmosphere = polytropes.FC_polytrope_2d_kappa_mu(nx=nx, nz=nz, constant_kappa=const_kappa, constant_mu=const_mu,\
epsilon=epsilon, gamma=gamma, n_rho_cz=n_rho_cz, aspect_ratio=aspect_ratio,\
fig_dir=data_dir)
else:
atmosphere = polytropes.FC_polytrope_2d(nx=nx, nz=nz, constant_kappa=const_kappa, constant_mu=const_mu,\
epsilon=epsilon, gamma=gamma, n_rho_cz=n_rho_cz, aspect_ratio=aspect_ratio,\
fig_dir=data_dir)
if epsilon < 1e-4:
ncc_cutoff = 1e-14
elif epsilon > 1e-1:
ncc_cutoff = 1e-6
else:
ncc_cutoff = 1e-10
if threeD:
atmosphere.set_IVP_problem(Rayleigh,
Prandtl,
Taylor=Taylor,
theta=theta,
ncc_cutoff=ncc_cutoff,
split_diffusivities=split_diffusivities)
else:
atmosphere.set_IVP_problem(Rayleigh,
Prandtl,
ncc_cutoff=ncc_cutoff,
split_diffusivities=split_diffusivities)
bc_dict = {
'stress_free': False,
'no_slip': False,
'fixed_flux': False,
'mixed_flux_temperature': False,
'fixed_temperature': False
}
if no_slip:
bc_dict['no_slip'] = True
else:
bc_dict['stress_free'] = True
if fixed_flux:
bc_dict['fixed_flux'] = True
elif mixed_flux_T:
bc_dict['mixed_flux_temperature'] = True
else:
bc_dict['fixed_temperature'] = True
atmosphere.set_BC(**bc_dict)
problem = atmosphere.get_problem()
if atmosphere.domain.distributor.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.mkdir('{:s}/'.format(data_dir))
if rk222:
logger.info("timestepping using RK222")
ts = de.timesteppers.RK222
cfl_safety_factor = safety_factor * 2
else:
logger.info("timestepping using RK443")
ts = de.timesteppers.RK443
cfl_safety_factor = safety_factor * 4
# Build solver
solver = problem.build_solver(ts)
#Check atmosphere
logger.info("thermal_time = {:g}, top_thermal_time = {:g}".format(atmosphere.thermal_time,\
atmosphere.top_thermal_time))
logger.info("full atm HS check")
atmosphere.check_atmosphere(make_plots=False,
rho=atmosphere.get_full_rho(solver),
T=atmosphere.get_full_T(solver))
if restart is None or start_new_files:
mode = "overwrite"
else:
mode = "append"
logger.info('checkpointing in {}'.format(data_dir))
checkpoint = Checkpoint(data_dir)
if restart is None:
atmosphere.set_IC(solver)
dt = None
else:
logger.info("restarting from {}".format(restart))
dt = checkpoint.restart(restart, solver)
checkpoint.set_checkpoint(solver, wall_dt=checkpoint_min * 60, mode=mode)
if run_time_buoyancies != None:
solver.stop_sim_time = solver.sim_time + run_time_buoyancies * atmosphere.buoyancy_time
else:
solver.stop_sim_time = 100 * atmosphere.thermal_time
solver.stop_iteration = solver.iteration + run_time_iter
solver.stop_wall_time = run_time * 3600
report_cadence = 1
output_time_cadence = out_cadence * atmosphere.buoyancy_time
Hermitian_cadence = 100
logger.info("stopping after {:g} time units".format(solver.stop_sim_time))
logger.info("output cadence = {:g}".format(output_time_cadence))
if threeD:
analysis_tasks = atmosphere.initialize_output(
solver,
data_dir,
sim_dt=output_time_cadence,
coeffs_output=not (no_coeffs),
mode=mode,
max_writes=max_writes,
volumes_output=not (no_volumes))
else:
analysis_tasks = atmosphere.initialize_output(
solver,
data_dir,
sim_dt=output_time_cadence,
coeffs_output=not (no_coeffs),
mode=mode,
max_writes=max_writes)
#Set up timestep defaults
max_dt = output_time_cadence
if dt is None: dt = max_dt
cfl_cadence = 1
cfl_threshold = 0.1
CFL = flow_tools.CFL(solver,
initial_dt=dt,
cadence=cfl_cadence,
safety=cfl_safety_factor,
max_change=1.5,
min_change=0.5,
max_dt=max_dt,
threshold=cfl_threshold)
if threeD:
CFL.add_velocities(('u', 'v', 'w'))
else:
CFL.add_velocities(('u', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("Re_rms", name='Re')
if verbose:
flow.add_property("Pe_rms", name='Pe')
flow.add_property("Nusselt_AB17", name='Nusselt')
start_iter = solver.iteration
start_sim_time = solver.sim_time
try:
start_time = time.time()
start_iter = solver.iteration
logger.info('starting main loop')
good_solution = True
first_step = True
while solver.ok and good_solution:
dt = CFL.compute_dt()
# advance
solver.step(dt)
effective_iter = solver.iteration - start_iter
Re_avg = flow.grid_average('Re')
if threeD and effective_iter % Hermitian_cadence == 0:
for field in solver.state.fields:
field.require_grid_space()
# update lists
if effective_iter % report_cadence == 0:
log_string = 'Iteration: {:5d}, Time: {:8.3e} ({:8.3e}), dt: {:8.3e}, '.format(
solver.iteration - start_iter, solver.sim_time,
(solver.sim_time - start_sim_time) /
atmosphere.buoyancy_time, dt)
if verbose:
log_string += '\n\t\tRe: {:8.5e}/{:8.5e}'.format(
Re_avg, flow.max('Re'))
log_string += '; Pe: {:8.5e}/{:8.5e}'.format(
flow.grid_average('Pe'), flow.max('Pe'))
log_string += '; Nu: {:8.5e}/{:8.5e}'.format(
flow.grid_average('Nusselt'), flow.max('Nusselt'))
else:
log_string += 'Re: {:8.3e}/{:8.3e}'.format(
Re_avg, flow.max('Re'))
logger.info(log_string)
if not np.isfinite(Re_avg):
good_solution = False
logger.info(
"Terminating run. Trapped on Reynolds = {}".format(
Re_avg))
if first_step:
if verbose:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.spy(solver.pencils[0].L,
markersize=1,
markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern.png", dpi=1200)
import scipy.sparse.linalg as sla
LU = sla.splu(solver.pencils[0].LHS.tocsc(),
permc_spec='NATURAL')
fig = plt.figure()
ax = fig.add_subplot(1, 2, 1)
ax.spy(LU.L.A, markersize=1, markeredgewidth=0.0)
ax = fig.add_subplot(1, 2, 2)
ax.spy(LU.U.A, markersize=1, markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern_LU.png", dpi=1200)
logger.info("{} nonzero entries in LU".format(LU.nnz))
logger.info("{} nonzero entries in LHS".format(
solver.pencils[0].LHS.tocsc().nnz))
logger.info("{} fill in factor".format(
LU.nnz / solver.pencils[0].LHS.tocsc().nnz))
first_step = False
start_time = time.time()
except:
logger.error('Exception raised, triggering end of main loop.')
finally:
end_time = time.time()
# Print statistics
elapsed_time = end_time - start_time
elapsed_sim_time = solver.sim_time
N_iterations = solver.iteration - 1
logger.info('main loop time: {:e}'.format(elapsed_time))
logger.info('Iterations: {:d}'.format(N_iterations))
logger.info('iter/sec: {:g}'.format(N_iterations / (elapsed_time)))
if N_iterations > 0:
logger.info('Average timestep: {:e}'.format(elapsed_sim_time /
N_iterations))
if not no_join:
logger.info('beginning join operation')
try:
final_checkpoint = Checkpoint(
data_dir, checkpoint_name='final_checkpoint')
final_checkpoint.set_checkpoint(solver,
wall_dt=1,
mode="append")
solver.step(dt) #clean this up in the future...works for now.
post.merge_process_files(data_dir + '/final_checkpoint/',
cleanup=False)
except:
print('cannot save final checkpoint')
logger.info(data_dir + '/checkpoint/')
post.merge_process_files(data_dir + '/checkpoint/', cleanup=False)
for task in analysis_tasks.keys():
logger.info(analysis_tasks[task].base_path)
post.merge_process_files(analysis_tasks[task].base_path,
cleanup=False)
if (atmosphere.domain.distributor.rank == 0):
logger.info('main loop time: {:e}'.format(elapsed_time))
if start_iter > 1:
logger.info('Iterations (this run): {:d}'.format(N_iterations -
start_iter))
logger.info('Iterations (total): {:d}'.format(N_iterations -
start_iter))
logger.info('iter/sec: {:g}'.format(N_iterations / (elapsed_time)))
if N_iterations > 0:
logger.info('Average timestep: {:e}'.format(elapsed_sim_time /
N_iterations))
N_TOTAL_CPU = atmosphere.domain.distributor.comm_cart.size
# Print statistics
print('-' * 40)
total_time = end_time - initial_time
main_loop_time = end_time - start_time
startup_time = start_time - initial_time
n_steps = solver.iteration - 1
print(' startup time:', startup_time)
print('main loop time:', main_loop_time)
print(' total time:', total_time)
if n_steps > 0:
print(' iterations:', n_steps)
print(' loop sec/iter:', main_loop_time / n_steps)
print(' average dt:', solver.sim_time / n_steps)
print(
" N_cores, Nx, Nz, startup main loop, main loop/iter, main loop/iter/grid, n_cores*main loop/iter/grid"
)
print(
'scaling:', ' {:d} {:d} {:d}'.format(N_TOTAL_CPU, nx, nz),
' {:8.3g} {:8.3g} {:8.3g} {:8.3g} {:8.3g}'.format(
startup_time, main_loop_time, main_loop_time / n_steps,
main_loop_time / n_steps / (nx * nz),
N_TOTAL_CPU * main_loop_time / n_steps / (nx * nz)))
print('-' * 40)
def FC_convection(Rayleigh=1e6, Prandtl=1,
ChemicalPrandtl=1, ChemicalReynolds=10, stiffness=1e4,
n_rho_cz=3.5, n_rho_rz=1,
nz_cz=128, nz_rz=128,
nx = None,
width=None,
single_chebyshev=False,
rk222=False,
superstep=False,
dense=False, nz_dense=64,
oz=False,
restart=None, data_dir='./', verbose=False):
import numpy as np
import time
from stratified_dynamics import multitropes
import os
from dedalus.core.future import FutureField
initial_time = time.time()
logger.info("Starting Dedalus script {:s}".format(sys.argv[0]))
if oz:
constant_Prandtl=False
stable_top=True
mixed_temperature_flux=True
else:
constant_Prandtl=True
stable_top=False
mixed_temperature_flux=None
# Set domain
if nx is None:
nx = nz_cz*4
if single_chebyshev:
nz = nz_cz
nz_list = [nz_cz]
else:
if dense:
nz = nz_rz+nz_dense+nz_cz
#nz_list = [nz_rz, int(nz_dense/2), int(nz_dense/2), nz_cz]
nz_list = [nz_rz, nz_dense, nz_cz]
else:
nz = nz_rz+nz_cz
nz_list = [nz_rz, nz_cz]
atmosphere = multitropes.FC_multitrope_rxn(nx=nx, nz=nz_list, stiffness=stiffness,
n_rho_cz=n_rho_cz, n_rho_rz=n_rho_rz,
verbose=verbose, width=width,
constant_Prandtl=constant_Prandtl,
stable_top=stable_top)
atmosphere.set_IVP_problem(Rayleigh, Prandtl, ChemicalPrandtl, ChemicalReynolds)
atmosphere.set_BC(mixed_temperature_flux=mixed_temperature_flux)
problem = atmosphere.get_problem()
#atmosphere.plot_atmosphere()
#atmosphere.plot_scaled_atmosphere()
if atmosphere.domain.distributor.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.mkdir('{:s}/'.format(data_dir))
if rk222:
logger.info("timestepping using RK222")
ts = de.timesteppers.RK222
cfl_safety_factor = 0.2*2
else:
logger.info("timestepping using RK443")
ts = de.timesteppers.RK443
cfl_safety_factor = 0.2*4
# Build solver
solver = problem.build_solver(ts)
if do_checkpointing:
checkpoint = Checkpoint(data_dir)
checkpoint.set_checkpoint(solver, wall_dt=1800)
# initial conditions
if restart is None:
atmosphere.set_IC(solver)
else:
if do_checkpointing:
logger.info("restarting from {}".format(restart))
checkpoint.restart(restart, solver)
else:
logger.error("No checkpointing capability in this branch of Dedalus. Aborting.")
raise
logger.info("thermal_time = {:g}, top_thermal_time = {:g}".format(atmosphere.thermal_time, atmosphere.top_thermal_time))
max_dt = atmosphere.min_BV_time
max_dt = atmosphere.buoyancy_time*0.25
report_cadence = 1
output_time_cadence = 0.1*atmosphere.buoyancy_time
solver.stop_sim_time = np.inf
solver.stop_iteration= np.inf
solver.stop_wall_time = 23.5*3600
logger.info("output cadence = {:g}".format(output_time_cadence))
analysis_tasks = atmosphere.initialize_output(solver, data_dir, sim_dt=output_time_cadence)
cfl_cadence = 1
CFL = flow_tools.CFL(solver, initial_dt=max_dt, cadence=cfl_cadence, safety=cfl_safety_factor,
max_change=1.5, min_change=0.5, max_dt=max_dt, threshold=0.1)
if superstep:
CFL_traditional = flow_tools.CFL(solver, initial_dt=max_dt, cadence=cfl_cadence, safety=cfl_safety_factor,
max_change=1.5, min_change=0.5, max_dt=max_dt, threshold=0.1)
CFL_traditional.add_velocities(('u', 'w'))
vel_u = FutureField.parse('u', CFL.solver.evaluator.vars, CFL.solver.domain)
delta_x = atmosphere.Lx/nx
CFL.add_frequency(vel_u/delta_x)
vel_w = FutureField.parse('w', CFL.solver.evaluator.vars, CFL.solver.domain)
mean_delta_z_cz = atmosphere.Lz_cz/nz_cz
CFL.add_frequency(vel_w/mean_delta_z_cz)
else:
CFL.add_velocities(('u', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("Re_rms", name='Re')
try:
start_time = time.time()
while solver.ok:
dt = CFL.compute_dt()
# advance
solver.step(dt)
# update lists
if solver.iteration % report_cadence == 0:
log_string = 'Iteration: {:5d}, Time: {:8.3e} ({:8.3e}), '.format(solver.iteration, solver.sim_time, solver.sim_time/atmosphere.buoyancy_time)
log_string += 'dt: {:8.3e}'.format(dt)
if superstep:
dt_traditional = CFL_traditional.compute_dt()
log_string += ' (vs {:8.3e})'.format(dt_traditional)
log_string += ', '
log_string += 'Re: {:8.3e}/{:8.3e}'.format(flow.grid_average('Re'), flow.max('Re'))
logger.info(log_string)
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
end_time = time.time()
# Print statistics
elapsed_time = end_time - start_time
elapsed_sim_time = solver.sim_time
N_iterations = solver.iteration
logger.info('main loop time: {:e}'.format(elapsed_time))
logger.info('Iterations: {:d}'.format(N_iterations))
logger.info('iter/sec: {:g}'.format(N_iterations/(elapsed_time)))
logger.info('Average timestep: {:e}'.format(elapsed_sim_time / N_iterations))
logger.info('beginning join operation')
if do_checkpointing:
logger.info(data_dir+'/checkpoint/')
post.merge_analysis(data_dir+'/checkpoint/')
for task in analysis_tasks:
logger.info(analysis_tasks[task].base_path)
post.merge_analysis(analysis_tasks[task].base_path)
if (atmosphere.domain.distributor.rank==0):
logger.info('main loop time: {:e}'.format(elapsed_time))
logger.info('Iterations: {:d}'.format(N_iterations))
logger.info('iter/sec: {:g}'.format(N_iterations/(elapsed_time)))
logger.info('Average timestep: {:e}'.format(elapsed_sim_time / N_iterations))
N_TOTAL_CPU = atmosphere.domain.distributor.comm_cart.size
# Print statistics
print('-' * 40)
total_time = end_time-initial_time
main_loop_time = end_time - start_time
startup_time = start_time-initial_time
n_steps = solver.iteration-1
print(' startup time:', startup_time)
print('main loop time:', main_loop_time)
print(' total time:', total_time)
print('Iterations:', solver.iteration)
print('Average timestep:', solver.sim_time / n_steps)
print(" N_cores, Nx, Nz, startup main loop, main loop/iter, main loop/iter/grid, n_cores*main loop/iter/grid")
print('scaling:',
' {:d} {:d} {:d}'.format(N_TOTAL_CPU,nx,nz),
' {:8.3g} {:8.3g} {:8.3g} {:8.3g} {:8.3g}'.format(startup_time,
main_loop_time,
main_loop_time/n_steps,
main_loop_time/n_steps/(nx*nz),
N_TOTAL_CPU*main_loop_time/n_steps/(nx*nz)))
print('-' * 40)
def FC_polytrope(dynamics_file,
Rayleigh=1e4,
Prandtl=1,
aspect_ratio=4,
Taylor=None,
theta=0,
nz=128,
nx=None,
ny=None,
threeD=False,
mesh=None,
n_rho_cz=3,
epsilon=1e-4,
gamma=5 / 3,
run_time=23.5,
run_time_buoyancies=None,
run_time_iter=np.inf,
fixed_T=False,
fixed_flux=False,
mixed_flux_T=False,
const_mu=True,
const_kappa=True,
dynamic_diffusivities=False,
split_diffusivities=False,
chemistry=True,
ChemicalPrandtl=1,
Qu_0=5e-8,
phi_0=10,
restart=None,
start_new_files=False,
scalar_file=None,
rk222=False,
safety_factor=0.2,
max_writes=20,
data_dir='./',
out_cadence=0.1,
no_coeffs=False,
no_join=False,
verbose=False):
import dedalus.public as de
from dedalus.tools import post
from dedalus.extras import flow_tools
import time
import os
import sys
from stratified_dynamics import polytropes
from tools.checkpointing import Checkpoint
checkpoint_min = 30
initial_time = time.time()
logger.info("Starting Dedalus script {:s}".format(sys.argv[0]))
if nx is None:
nx = int(np.round(nz * aspect_ratio))
if threeD and ny is None:
ny = nx
eqn_dict = {
'nx': nx,
'nz': nz,
'constant_kappa': const_kappa,
'constant_mu': const_mu,
'epsilon': epsilon,
'gamma': gamma,
'n_rho_cz': n_rho_cz,
'aspect_ratio': aspect_ratio,
'fig_dir': data_dir
}
if threeD:
eqn_dict['mesh'] = mesh
eqn_dict['nz'] = nz
atmosphere = polytropes.FC_polytrope_rxn_3d(**eqn_dict)
else:
if dynamic_diffusivities:
atmosphere = polytropes.FC_polytrope_2d_kappa(**eqn_dict)
else:
if chemistry:
atmosphere = polytropes.FC_polytrope_rxn_2d(**eqn_dict)
else:
atmosphere = polytropes.FC_polytrope_2d(**eqn_dict)
if epsilon < 1e-4:
ncc_cutoff = 1e-14
elif epsilon > 1e-1:
ncc_cutoff = 1e-6
else:
ncc_cutoff = 1e-10
problem_dict = {
'ncc_cutoff': ncc_cutoff,
'split_diffusivities': split_diffusivities
}
if threeD:
problem_dict['Taylor'] = Taylor
problem_dict['theta'] = theta
if chemistry:
problem_dict['ChemicalPrandtl'] = ChemicalPrandtl
problem_dict['Qu_0'] = Qu_0
problem_dict['phi_0'] = phi_0
atmosphere.set_IVP_problem(Rayleigh, Prandtl, **problem_dict)
if fixed_flux:
atmosphere.set_BC(fixed_flux=True, stress_free=True)
elif mixed_flux_T:
atmosphere.set_BC(mixed_flux_temperature=True, stress_free=True)
else:
atmosphere.set_BC(fixed_temperature=True, stress_free=True)
problem = atmosphere.get_problem()
if atmosphere.domain.distributor.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.mkdir('{:s}/'.format(data_dir))
if rk222:
logger.info("timestepping using RK222")
ts = de.timesteppers.RK222
cfl_safety_factor = safety_factor * 2
else:
logger.info("timestepping using RK443")
ts = de.timesteppers.RK443
cfl_safety_factor = safety_factor * 4
# Build solver
solver = problem.build_solver(ts)
#Check atmosphere
logger.info("thermal_time = {:g}, top_thermal_time = {:g}".format(atmosphere.thermal_time,\
atmosphere.top_thermal_time))
logger.info("full atm HS check")
atmosphere.check_atmosphere(make_plots=False,
rho=atmosphere.get_full_rho(solver),
T=atmosphere.get_full_T(solver))
if restart is None:
mode = "overwrite"
else:
mode = "append"
logger.info('checkpointing in {}'.format(data_dir))
checkpoint = Checkpoint(data_dir)
if restart is None:
atmosphere.set_IC(solver)
dt = None
else:
logger.info("restarting from {}".format(restart))
dt = checkpoint.restart(restart, solver)
checkpoint.set_checkpoint(solver, wall_dt=checkpoint_min * 60, mode=mode)
if run_time_buoyancies != None:
solver.stop_sim_time = solver.sim_time + run_time_buoyancies * atmosphere.buoyancy_time
else:
solver.stop_sim_time = 100 * atmosphere.thermal_time
solver.stop_iteration = solver.iteration + run_time_iter
solver.stop_wall_time = run_time * 3600
report_cadence = 1
output_time_cadence = out_cadence * atmosphere.buoyancy_time
Hermitian_cadence = 100
logger.info("stopping after {:g} time units".format(solver.stop_sim_time))
logger.info("output cadence = {:g}".format(output_time_cadence))
analysis_tasks = atmosphere.initialize_output(
solver,
data_dir,
sim_dt=output_time_cadence,
coeffs_output=not (no_coeffs),
mode=mode,
max_writes=max_writes)
# Reinjecting dynamics and tracers if desired
logger.info("Re-injecting scalars")
def reset_variable(key, h5_val, grid=False, grad=False):
# Get variable
k = solver.state[key]
k.set_scales(1, keep_data=True)
# Set grid or coefficient space
if grid:
gc = 'g'
slice = solver.domain.dist.grid_layout.slices(k.meta[:]['scale'])
else:
gc = 'c'
slice = solver.domain.dist.coeff_layout.slices(k.meta[:]['scale'])
# Set initial value of variable
k[gc] = h5_val[slice]
if grad: # Set gradient if called for
k.differentiate('z', out=k)
logger.info("Re-injecting: {}".format(key))
return k
objects = {}
# Set all the dynamic variables
if threeD:
keys = ['u', 'u_z', 'v', 'v_z', 'w', 'w_z', 'T1', 'T1_z', 'ln_rho1']
grads = [
False, True, False, True, False, True, False, True, False, False,
True, False, True
]
h5_keys = ['u', 'u', 'v', 'v', 'w', 'w', 'T', 'T', 'ln_rho']
else:
keys = ['u', 'u_z', 'w', 'w_z', 'T1', 'T1_z', 'ln_rho1']
grads = [
False, True, False, True, False, True, False, False, True, False,
True
]
h5_keys = ['u', 'u', 'w', 'w', 'T', 'T', 'ln_rho']
h5File_c = h5py.File(dynamics_file, 'r')
dt = h5File_c['scales']['timestep'][-1]
h5File_c = h5File_c['tasks']
for i, K in enumerate(keys):
# Restarting with final profile of run
objects[K] = reset_variable(K, h5File_c[h5_keys[i]][-1], grad=grads[i])
if scalar_file != 'None':
keys = ['f', 'f_z', 'C', 'C_z', 'G', 'G_z']
grads = [False, True, False, True]
h5_keys = ['f', 'f', 'C', 'C', 'G', 'G']
h5File_g = h5py.File(scalar_file, 'r')['tasks']
for i, K in enumerate(keys):
# Restarting with initial profile
objects[K] = reset_variable(K,
h5File_g[h5_keys[i]][0],
grid=True,
grad=grads[i])
#Set up timestep defaults
max_dt = output_time_cadence / 2
if dt is None: dt = max_dt
cfl_cadence = 1
cfl_threshold = 0.1
CFL = flow_tools.CFL(solver,
initial_dt=dt,
cadence=cfl_cadence,
safety=cfl_safety_factor,
max_change=1.5,
min_change=0.5,
max_dt=max_dt,
threshold=cfl_threshold)
if threeD:
CFL.add_velocities(('u', 'v', 'w'))
else:
CFL.add_velocities(('u', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("Re_rms", name='Re')
if verbose:
flow.add_property("Pe_rms", name='Pe')
flow.add_property("Nusselt_AB17", name='Nusselt')
start_iter = solver.iteration
start_sim_time = solver.sim_time
try:
start_time = time.time()
start_iter = solver.iteration
logger.info('starting main loop')
good_solution = True
first_step = True
while solver.ok and good_solution:
dt = CFL.compute_dt()
# advance
solver.step(dt)
effective_iter = solver.iteration - start_iter
if threeD and effective_iter % Hermitian_cadence == 0:
for field in solver.state.fields:
field.require_grid_space()
# update lists
if effective_iter % report_cadence == 0:
Re_avg = flow.grid_average('Re')
log_string = 'Iteration: {:5d}, Time: {:8.3e} ({:8.3e}), dt: {:8.3e}, '.format(
solver.iteration - start_iter, solver.sim_time,
(solver.sim_time - start_sim_time) /
atmosphere.buoyancy_time, dt)
if verbose:
log_string += '\n\t\tRe: {:8.5e}/{:8.5e}'.format(
Re_avg, flow.max('Re'))
log_string += '; Pe: {:8.5e}/{:8.5e}'.format(
flow.grid_average('Pe'), flow.max('Pe'))
log_string += '; Nu: {:8.5e}/{:8.5e}'.format(
flow.grid_average('Nusselt'), flow.max('Nusselt'))
else:
log_string += 'Re: {:8.3e}/{:8.3e}'.format(
Re_avg, flow.max('Re'))
logger.info(log_string)
if not np.isfinite(Re_avg):
good_solution = False
logger.info(
"Terminating run. Trapped on Reynolds = {}".format(
Re_avg))
if first_step:
if verbose:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.spy(solver.pencils[0].L,
markersize=1,
markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern.png", dpi=1200)
import scipy.sparse.linalg as sla
LU = sla.splu(solver.pencils[0].LHS.tocsc(),
permc_spec='NATURAL')
fig = plt.figure()
ax = fig.add_subplot(1, 2, 1)
ax.spy(LU.L.A, markersize=1, markeredgewidth=0.0)
ax = fig.add_subplot(1, 2, 2)
ax.spy(LU.U.A, markersize=1, markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern_LU.png", dpi=1200)
logger.info("{} nonzero entries in LU".format(LU.nnz))
logger.info("{} nonzero entries in LHS".format(
solver.pencils[0].LHS.tocsc().nnz))
logger.info("{} fill in factor".format(
LU.nnz / solver.pencils[0].LHS.tocsc().nnz))
first_step = False
start_time = time.time()
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
end_time = time.time()
# Print statistics
elapsed_time = end_time - start_time
elapsed_sim_time = solver.sim_time
N_iterations = solver.iteration - 1
logger.info('main loop time: {:e}'.format(elapsed_time))
logger.info('Iterations: {:d}'.format(N_iterations))
logger.info('iter/sec: {:g}'.format(N_iterations / (elapsed_time)))
if N_iterations > 0:
logger.info('Average timestep: {:e}'.format(elapsed_sim_time /
N_iterations))
if not no_join:
logger.info('beginning join operation')
try:
final_checkpoint = Checkpoint(
data_dir, checkpoint_name='final_checkpoint')
final_checkpoint.set_checkpoint(solver,
wall_dt=1,
mode="append")
solver.step(dt) #clean this up in the future...works for now.
post.merge_process_files(data_dir + '/final_checkpoint/',
cleanup=True)
except:
print('cannot save final checkpoint')
logger.info(data_dir + '/checkpoint/')
post.merge_process_files(data_dir + '/checkpoint/', cleanup=True)
for task in analysis_tasks.keys():
logger.info(analysis_tasks[task].base_path)
post.merge_process_files(analysis_tasks[task].base_path,
cleanup=True)
if (atmosphere.domain.distributor.rank == 0):
logger.info('main loop time: {:e}'.format(elapsed_time))
if start_iter > 1:
logger.info('Iterations (this run): {:d}'.format(N_iterations -
start_iter))
logger.info('Iterations (total): {:d}'.format(N_iterations -
start_iter))
logger.info('iter/sec: {:g}'.format(N_iterations / (elapsed_time)))
if N_iterations > 0:
logger.info('Average timestep: {:e}'.format(elapsed_sim_time /
N_iterations))
N_TOTAL_CPU = atmosphere.domain.distributor.comm_cart.size
# Print statistics
print('-' * 40)
total_time = end_time - initial_time
main_loop_time = end_time - start_time
startup_time = start_time - initial_time
n_steps = solver.iteration - 1
print(' startup time:', startup_time)
print('main loop time:', main_loop_time)
print(' total time:', total_time)
if n_steps > 0:
print(' iterations:', n_steps)
print(' loop sec/iter:', main_loop_time / n_steps)
print(' average dt:', solver.sim_time / n_steps)
print(
" N_cores, Nx, Nz, startup main loop, main loop/iter, main loop/iter/grid, n_cores*main loop/iter/grid"
)
print(
'scaling:', ' {:d} {:d} {:d}'.format(N_TOTAL_CPU, nx, nz),
' {:8.3g} {:8.3g} {:8.3g} {:8.3g} {:8.3g}'.format(
startup_time, main_loop_time, main_loop_time / n_steps,
main_loop_time / n_steps / (nx * nz),
N_TOTAL_CPU * main_loop_time / n_steps / (nx * nz)))
print('-' * 40)
def RUN(self,
scheme=de.timesteppers.RK443,
adding=False,
sim_time=2,
wall_time=np.inf,
tight=False,
save=20):
self.solver = self.problem.build_solver(scheme)
if adding:
t = load_last_value('times.txt')
temp = load_last_array('temperatures.txt', shape=self.saving_shape)
Variables = load_arrays('variables.txt',
frequency=1,
shape=self.saving_shape)
T = self.solver.state['T']
Psi = self.solver.state['psi']
Curl = self.solver.state['curl']
T['g'] = temp
T.differentiate('z', out=self.solver.state['Tz'])
Psi['g'] = Variables[0]
Psi.differentiate('z', out=self.solver.state['psiz'])
Curl['g'] = Variables[1]
Curl.differentiate('z', out=self.solver.state['curlz'])
else:
t = 0
print("Clearing old data ...")
clear_file('temperatures.txt')
clear_file('times.txt')
clear_file('variables.txt')
# Initial conditions ----------------------------------------------------------
print("Initializing Values ...")
eps = 1e-4
k = 3.117
x, z = self.problem.domain.grids(scales=1)
T = self.solver.state['T']
T['g'] = 1 - z + eps * np.sin(k * x) * np.sin(2 * np.pi * z)
T.differentiate('z', out=self.solver.state['Tz'])
# Stopping Parameters ---------------------------------------------------------
self.solver.stop_sim_time = sim_time # Length of simulation.
self.solver.stop_wall_time = wall_time # Real time allowed to compute.
self.solver.stop_iteration = np.inf # Maximum iterations allowed.
# Control Flow ----------------------------------------------------------------
dt = 1e-4
if tight:
cfl = flow_tools.CFL(self.solver,
initial_dt=dt,
cadence=1,
safety=1,
max_change=1.5,
min_change=0.01,
max_dt=0.01,
min_dt=1e-10)
else:
cfl = flow_tools.CFL(self.solver,
initial_dt=dt,
cadence=10,
safety=1,
max_change=1.5,
min_change=0.5,
max_dt=0.01,
min_dt=1e-6)
cfl.add_velocities(('u', 'v'))
# Flow properties (print during run; not recorded in the records files)
flow = flow_tools.GlobalFlowProperty(self.solver, cadence=1)
flow.add_property("sqrt(u **2 + v **2) / Ra", name='Re')
# MAIN COMPUTATION LOOP -------------------------------------------------------
try:
print("####### BEGINNING CALCULATIONS #######")
print("Starting main loop")
start_time = time.time()
while self.solver.ok:
# Recompute time step and iterate.
dt = self.solver.step(cfl.compute_dt())
t = t + dt
if self.solver.iteration % 10 == 0:
info = "Iteration {:>5d}, Time: {:.7f}, dt: {:.2e}".format(
self.solver.iteration, self.solver.sim_time, dt)
Re = flow.max("Re")
info += ", Max Re = {:f}".format(Re)
print(info)
if np.isnan(Re):
raise ValueError("Reynolds number went to infinity!!"
"\nRe = {}".format(Re))
if save:
if self.solver.iteration % save == 0:
T = self.solver.state['T']
append_unique_array('temperatures.txt', T['g'])
append_unique_value('times.txt', t)
except BaseException as e:
print("Exception raised, triggering end of main loop.")
raise
finally:
print("####### CALCULATIONS FINISHED #######")
total_time = time.time() - start_time
print("Iterations: {:d}".format(self.solver.iteration))
print("Sim end time: {:.3e}".format(self.solver.sim_time))
print("Run time: {:.3e} sec".format(total_time))
print("END OF SIMULATION\n")
T = self.solver.state['T']
Psi = self.solver.state['psi']
Curl = self.solver.state['curl']
append_unique_array('temperatures.txt', T['g'])
append_unique_value('times.txt', t)
append_unique_array('variables.txt', Psi['g'])
append_unique_array('variables.txt', Curl['g'])
# Analysis
snap = model.solver.evaluator.add_file_handler('snapshots', sim_dt=0.2,
max_writes=10)
snap.add_task("interp(b, z=0)", scales=1, name='b midplane')
snap.add_task("interp(u, z=0)", scales=1, name='u midplane')
snap.add_task("interp(v, z=0)", scales=1, name='v midplane')
snap.add_task("interp(w, z=0)", scales=1, name='w midplane')
# CFL
CFL = flow_tools.CFL(model.solver, initial_dt=1e-4, cadence=5,
safety=1.5, max_change=1.5, min_change=0.5, max_dt=0.05)
CFL.add_velocities(('u', 'v', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(model.solver, cadence=10)
flow.add_property("sqrt(u*u + v*v + w*w) / ν", name='Re')
# Main loop
try:
logger.info('Starting loop')
start_run_time = time.time()
while model.solver.ok:
dt = CFL.compute_dt()
model.solver.step(dt)
if (model.solver.iteration-1) % 100 == 0:
logger.info('Iteration: %i, Time: %e, dt: %e' %(
model.solver.iteration, model.solver.sim_time, dt))
logger.info('Max Re = %f' %flow.max('Re'))
except:
logger.error('Exception raised, triggering end of main loop.')
def FC_convection(Rayleigh=1e6,
Prandtl=1,
stiffness=3,
m_rz=3,
gamma=5 / 3,
MHD=False,
MagneticPrandtl=1,
B0_amplitude=1,
n_rho_cz=1,
n_rho_rz=5,
nz_cz=128,
nz_rz=128,
nx=None,
width=None,
single_chebyshev=False,
rk222=False,
superstep=False,
dense=False,
nz_dense=64,
oz=False,
fixed_flux=False,
run_time=23.5,
run_time_buoyancies=np.inf,
run_time_iter=np.inf,
dynamic_diffusivities=False,
max_writes=20,
out_cadence=0.1,
no_coeffs=False,
no_join=False,
restart=None,
data_dir='./',
verbose=False,
label=None):
def format_number(number, no_format_min=0.1, no_format_max=10):
if number > no_format_max or number < no_format_min:
try:
mantissa = "{:e}".format(number).split("+")[0].split(
"e")[0].rstrip("0") or "0"
power = "{:e}".format(number).split("+")[1].lstrip("0") or "0"
except:
mantissa = "{:e}".format(number).split("-")[0].split(
"e")[0].rstrip("0") or "0"
power = "{:e}".format(number).split("-")[1].lstrip("0") or "0"
power = "-" + power
if mantissa[-1] == ".":
mantissa = mantissa[:-1]
mantissa += "e"
else:
mantissa = "{:f}".format(number).rstrip("0") or "0"
if mantissa[-1] == ".":
mantissa = mantissa[:-1]
power = ""
number_string = mantissa + power
return number_string
data_dir = './'
# save data in directory named after script
if data_dir[-1] != '/':
data_dir += '/'
data_dir += sys.argv[0].split('.py')[0]
data_dir += "_nrhocz{}_Ra{}_S{}".format(format_number(n_rho_cz),
format_number(Rayleigh),
format_number(stiffness))
if width:
data_dir += "_erf{}".format(format_number(width))
if args['--MHD']:
data_dir += '_MHD'
if label:
data_dir += "_{}".format(label)
data_dir += '/'
from dedalus.tools.config import config
config['logging']['filename'] = os.path.join(data_dir, 'logs/dedalus_log')
config['logging']['file_level'] = 'DEBUG'
import mpi4py.MPI
if mpi4py.MPI.COMM_WORLD.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.makedirs('{:s}/'.format(data_dir))
logdir = os.path.join(data_dir, 'logs')
if not os.path.exists(logdir):
os.mkdir(logdir)
logger = logging.getLogger(__name__)
logger.info("saving run in: {}".format(data_dir))
import dedalus.public as de
from dedalus.tools import post
from dedalus.extras import flow_tools
from dedalus.core.future import FutureField
from stratified_dynamics import multitropes
from tools.checkpointing import Checkpoint
checkpoint_min = 30
initial_time = time.time()
logger.info("Starting Dedalus script {:s}".format(sys.argv[0]))
constant_Prandtl = True
stable_top = True
mixed_temperature_flux = True
# Set domain
if nx is None:
nx = nz_cz * 4
if single_chebyshev:
nz = nz_cz
nz_list = [nz_cz]
else:
nz = nz_rz + nz_cz
nz_list = [nz_rz, nz_cz]
eqns_dict = {
'stiffness': stiffness,
'nx': nx,
'nz': nz_list,
'n_rho_cz': n_rho_cz,
'n_rho_rz': n_rho_rz,
'verbose': verbose,
'width': width,
'constant_Prandtl': constant_Prandtl,
'stable_top': stable_top,
'gamma': gamma,
'm_rz': m_rz
}
if MHD:
atmosphere = multitropes.FC_MHD_multitrope_guidefield_2d(**eqns_dict)
atmosphere.set_IVP_problem(Rayleigh,
Prandtl,
MagneticPrandtl,
guidefield_amplitude=B0_amplitude)
else:
atmosphere = multitropes.FC_multitrope(**eqns_dict)
atmosphere.set_IVP_problem(Rayleigh, Prandtl)
atmosphere.set_BC()
problem = atmosphere.get_problem()
if atmosphere.domain.distributor.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.mkdir('{:s}/'.format(data_dir))
if rk222:
logger.info("timestepping using RK222")
ts = de.timesteppers.RK222
cfl_safety_factor = 0.2 * 2
else:
logger.info("timestepping using RK443")
ts = de.timesteppers.RK443
cfl_safety_factor = 0.2 * 4
# Build solver
solver = problem.build_solver(ts)
if restart is None:
mode = "overwrite"
else:
mode = "append"
checkpoint = Checkpoint(data_dir)
# initial conditions
if restart is None:
atmosphere.set_IC(solver)
dt = None
else:
logger.info("restarting from {}".format(restart))
dt = checkpoint.restart(restart, solver)
checkpoint.set_checkpoint(solver, wall_dt=checkpoint_min * 60, mode=mode)
logger.info("thermal_time = {:g}, top_thermal_time = {:g}".format(
atmosphere.thermal_time, atmosphere.top_thermal_time))
max_dt = atmosphere.min_BV_time
max_dt = atmosphere.buoyancy_time * out_cadence
if dt is None: dt = max_dt
report_cadence = 1
output_time_cadence = out_cadence * atmosphere.buoyancy_time
solver.stop_sim_time = solver.sim_time + run_time_buoyancies * atmosphere.buoyancy_time
solver.stop_iteration = solver.iteration + run_time_iter
solver.stop_wall_time = run_time * 3600
logger.info("output cadence = {:g}".format(output_time_cadence))
analysis_tasks = atmosphere.initialize_output(
solver,
data_dir,
coeffs_output=not (no_coeffs),
sim_dt=output_time_cadence,
max_writes=max_writes,
mode=mode)
cfl_cadence = 1
CFL = flow_tools.CFL(solver,
initial_dt=dt,
cadence=cfl_cadence,
safety=cfl_safety_factor,
max_change=1.5,
min_change=0.5,
max_dt=max_dt,
threshold=0.1)
CFL.add_velocities(('u', 'w'))
if MHD:
CFL.add_velocities(
('Bx/sqrt(4*pi*rho_full)', 'Bz/sqrt(4*pi*rho_full)'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("Re_rms", name='Re')
if MHD:
flow.add_property("abs(dx(Bx) + dz(Bz))", name='divB')
try:
start_time = time.time()
while solver.ok:
dt = CFL.compute_dt()
# advance
solver.step(dt)
# update lists
if solver.iteration % report_cadence == 0:
Re_avg = flow.grid_average('Re')
if not np.isfinite(Re_avg):
solver.ok = False
log_string = 'Iteration: {:5d}, Time: {:8.3e} ({:8.3e}), dt: {:8.3e}, '.format(
solver.iteration, solver.sim_time,
solver.sim_time / atmosphere.buoyancy_time, dt)
log_string += 'Re: {:8.3e}/{:8.3e}'.format(
Re_avg, flow.max('Re'))
if MHD:
log_string += ', divB: {:8.3e}/{:8.3e}'.format(
flow.grid_average('divB'), flow.max('divB'))
logger.info(log_string)
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
end_time = time.time()
# Print statistics
elapsed_time = end_time - start_time
elapsed_sim_time = solver.sim_time
N_iterations = solver.iteration
logger.info('main loop time: {:e}'.format(elapsed_time))
logger.info('Iterations: {:d}'.format(N_iterations))
logger.info('iter/sec: {:g}'.format(N_iterations / (elapsed_time)))
logger.info('Average timestep: {:e}'.format(elapsed_sim_time /
N_iterations))
if not no_join:
logger.info('beginning join operation')
logger.info(data_dir + '/checkpoint/')
post.merge_process_files(data_dir + '/checkpoint/', cleanup=False)
for task in analysis_tasks:
logger.info(analysis_tasks[task].base_path)
post.merge_process_files(analysis_tasks[task].base_path,
cleanup=False)
if (atmosphere.domain.distributor.rank == 0):
logger.info('main loop time: {:e}'.format(elapsed_time))
logger.info('Iterations: {:d}'.format(N_iterations))
logger.info('iter/sec: {:g}'.format(N_iterations / (elapsed_time)))
logger.info('Average timestep: {:e}'.format(elapsed_sim_time /
N_iterations))
N_TOTAL_CPU = atmosphere.domain.distributor.comm_cart.size
# Print statistics
print('-' * 40)
total_time = end_time - initial_time
main_loop_time = end_time - start_time
startup_time = start_time - initial_time
n_steps = solver.iteration - 1
print(' startup time:', startup_time)
print('main loop time:', main_loop_time)
print(' total time:', total_time)
print('Iterations:', solver.iteration)
print('Average timestep:', solver.sim_time / n_steps)
print(
" N_cores, Nx, Nz, startup main loop, main loop/iter, main loop/iter/grid, n_cores*main loop/iter/grid"
)
print(
'scaling:', ' {:d} {:d} {:d}'.format(N_TOTAL_CPU, nx, nz),
' {:8.3g} {:8.3g} {:8.3g} {:8.3g} {:8.3g}'.format(
startup_time, main_loop_time, main_loop_time / n_steps,
main_loop_time / n_steps / (nx * nz),
N_TOTAL_CPU * main_loop_time / n_steps / (nx * nz)))
print('-' * 40)
snapshots = solver.evaluator.add_file_handler('snapshots',
sim_dt=snapshotStep,
max_writes=600,
mode=fh_mode)
snapshots.add_system(solver.state)
output_cadence = 5
# CFL
#CFL = flow_tools.CFL(solver, initial_dt=dt, cadence=10, safety=10,
# max_change=1.5, min_change=0.1, max_dt=max_timestep, threshold=0.05)
#CFL.add_velocities(('vx', 'vy'))
#CFL.add_velocities(('vx2', 'vy2'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=output_cadence)
flow.add_property("dw(n)*dw(n) + dw(T)*dw(T)", name='dw_enstrophy')
curr_time = time.time()
# Main loop
try:
logger.info('Starting loop')
start_time = time.time()
while solver.proceed:
#dt = CFL.compute_dt()
dt = solver.step(dt)
if (solver.iteration - 1) % output_cadence == 0:
next_time = time.time()
logger.info('Iteration: %i, Time: %e, dt: %e' %
(solver.iteration, solver.sim_time, dt))
spinup_model.set_fields(u=noise,
v=noise,
w=noise,
b=spinup_model.z / Lz - noise)
spinup_model.stop_at(sim_time=spinup_time)
# Make CFL
spinup_CFL = flow_tools.CFL(spinup_model.solver,
initial_dt=dt0,
cadence=10,
max_change=1.5,
safety=0.5)
spinup_CFL.add_velocities(('u', 'v', 'w'))
# Flow properties: Reynolds number and three Nusselt numbers
spinup_stats = flow_tools.GlobalFlowProperty(spinup_model.solver,
cadence=spinup_log_cadence)
add_reynolds_number(spinup_stats)
add_nusselt_numbers(spinup_stats)
# Spinup loop with coarse model
try:
logger.info("Starting coarse spinup. Ra: {}, closure: {}".format(
Ra, closure_name))
start_run_time = time.time()
log_time = time.time()
while spinup_model.solver.ok:
dt = spinup_CFL.compute_dt()
spinup_model.solver.step(dt)
if (spinup_model.solver.iteration - 1) % spinup_log_cadence == 0:
compute_time = time.time() - log_time
log_time = time.time()
an1.add_task('p1+p2', name='diff_p1', layout='g')
an1.add_task('u1+u2', name='diff_u', layout='g')
an1.add_task('w1+w2', name='diff_w', layout='g')
an2 = solver.evaluator.add_file_handler('data_scalars',
sim_dt=param.scalar_sim_dt,
max_writes=100)
an2.add_task('integ(((u1+u2)**2 + (w1+w2)**2) /(a0+a1+a2)/2)',
name='KE',
layout='g')
an2.add_task('integ((u2*u2+w2*w2)/(a0+a1+a2)/2)', name='KE_pert', layout='g')
an2.add_task('-integ(u2*dx(t2xx) + u2*dz(t2xz) + w2*dx(t2xz) + w2*dz(t2zz))',
name='D',
layout='g')
# Monitoring
flow = flow_tools.GlobalFlowProperty(solver, cadence=param.CFL['cadence'])
flow.add_property('integ((u2*u2+w2*w2)/(a0+a1+a2)/2)', name='KE_pert')
# CFL calculator
CFL = flow_tools.CFL(solver, **param.CFL)
CFL.add_velocities(('u1+u2', 'w1+w2'))
# Main loop
dt_floor = 2**(np.log2(param.CFL['min_dt']) // 1)
try:
logger.info('Starting loop')
start_time = time.time()
while solver.ok:
dt = CFL.compute_dt()
dt = dt_floor * (dt // dt_floor)
dt = solver.step(dt, trim=param.trim)
ny=ny,
nz=nz,
ν=1,
κ=1,
closure=closure)
model.set_bc("no penetration", "top", "bottom")
model.set_bc("no slip", "top", "bottom")
model.set_bc("no flux", "top", "bottom")
model.build_solver()
# Initial condition
fields = {}
for field in ['u', 'v', 'w', 'b']:
noise = dedaLES.random_noise(model.domain)
pert = a * noise * model.z * (1 - model.z)
fields[field] = pert
model.set_fields(**fields)
model.stop_at(iteration=3)
# Flow properties
flow = flow_tools.GlobalFlowProperty(model.solver)
flow.add_property("sqrt(u*u + v*v + w*w) / ν", name='Re_domain')
model.solver.step(dt)
logger.info('Max domain Re = %e' % flow.max("Re_domain"))
model.solver.step(dt)
logger.info('Max domain Re = %e' % flow.max("Re_domain"))
initial_b = initial_N2*model.z
model.set_fields(
u = noise,
v = noise,
b = initial_b + np.sqrt(initial_N2) * noise
#w = noise,
)
model.stop_at(sim_time=run_time)
dt_gizmo = dedaLES.TimeStepGizmo(model.solver, initial_dt=initial_dt, cadence=dt_cadence, max_change=1.5, safety=dt_safety)
dt_gizmo.add_velocities(('u', 'v', 'w'))
dt_gizmo.add_diffusivity('ν_sgs')
dt_gizmo.add_diffusivity('κb_sgs')
stats = flow_tools.GlobalFlowProperty(model.solver, cadence=stats_cadence)
stats.add_property("w*b", name="wb")
stats.add_property("ε + ε_sgs", name="epsilon")
stats.add_property("ν_sgs", name="eddy viscosity")
stats.add_property("κb_sgs", name="eddy diffusivity")
stats.add_property("χ + χ_sgs", name="chi")
stats.add_property("w*w", name="w_sq")
stats.add_property("sqrt(u*u + v*v + w*w) / ν", name='Re')
analysis = model.solver.evaluator.add_file_handler("mixed_layer_analysis_{}".format(identifier(model, closure=closure)),
iter=analysis_cadence, max_writes=max_writes)
analysis.add_system(model.solver.state, layout='g')
analysis.add_task("interp(b, y=0)", scales=1, name='b midplane')
analysis.add_task("interp(u, y=0)", scales=1, name='u midplane')
analysis.add_task("interp(v, y=0)", scales=1, name='v midplane')
def solve_IVP(self,
dt,
CFL,
data_dir,
analysis_tasks,
task_args=(),
pre_loop_args=(),
task_kwargs={},
pre_loop_kwargs={},
time_div=None,
track_fields=['Pe'],
threeD=False,
Hermitian_cadence=100,
no_join=False,
mode='append'):
"""Logic for a while-loop that solves an initial value problem.
Parameters
----------
dt : float
The initial timestep of the simulation
CFL : a Dedalus CFL object
A CFL object that calculates the timestep of the simulation on the fly
data_dir : string
The parent directory of output files
analysis_tasks : OrderedDict()
An OrderedDict of dedalus FileHandler objects
task_args, task_kwargs : list, dict, optional
arguments & keyword arguments to the self._special_tasks() function
pre_loop_args, pre_loop_kwargs: list, dict, optional
arguments & keyword arguments to the self.pre_loop_setup() function
time_div : float, optional
A siulation time to divide the normal time by for easier output tracking
threeD : bool, optional
If True, occasionally force the solution to grid space to remove Hermitian errors
Hermitian_cadence : int, optional
The number of timesteps between grid space forcings in 3D.
no_join : bool, optional
If True, do not join files at the end of the simulation run.
mode : string, optional
Dedalus output mode for final checkpoint. "append" or "overwrite"
args, kwargs : list and dictionary
Additional arguments and keyword arguments to be passed to the self.special_tasks() function
"""
# Flow properties
self.flow = flow_tools.GlobalFlowProperty(self.solver, cadence=1)
for f in track_fields:
self.flow.add_property(f, name=f)
self.pre_loop_setup(*pre_loop_args, **pre_loop_kwargs)
start_time = time.time()
# Main loop
count = 0
try:
logger.info('Starting loop')
init_time = self.solver.sim_time
start_iter = self.solver.iteration
while (self.solver.ok):
dt = CFL.compute_dt()
self.solver.step(dt) #, trim=True)
# prevents blow-up over long timescales in 3D due to hermitian-ness
effective_iter = self.solver.iteration - start_iter
if threeD and effective_iter % Hermitian_cadence == 0:
for field in self.solver.state.fields:
field.require_grid_space()
self.special_tasks(*task_args, **task_kwargs)
#reporting string
self.iteration_report(dt, track_fields, time_div=time_div)
if not np.isfinite(self.flow.grid_average(track_fields[0])):
break
except:
raise
logger.error('Exception raised, triggering end of main loop.')
finally:
end_time = time.time()
main_loop_time = end_time - start_time
n_iter_loop = self.solver.iteration - 1
logger.info('Iterations: {:d}'.format(n_iter_loop))
logger.info('Sim end time: {:f}'.format(self.solver.sim_time))
logger.info('Run time: {:f} sec'.format(main_loop_time))
logger.info('Run time: {:f} cpu-hr'.format(
main_loop_time / 60 / 60 *
self.de_domain.domain.dist.comm_cart.size))
logger.info('iter/sec: {:f} (main loop only)'.format(
n_iter_loop / main_loop_time))
try:
final_checkpoint = Checkpoint(
data_dir, checkpoint_name='final_checkpoint')
final_checkpoint.set_checkpoint(self.solver,
wall_dt=1,
mode=mode)
self.solver.step(
dt) #clean this up in the future...works for now.
post.merge_process_files(data_dir + '/final_checkpoint/',
cleanup=False)
except:
raise
print('cannot save final checkpoint')
finally:
if not no_join:
logger.info('beginning join operation')
post.merge_analysis(data_dir + 'checkpoint')
for key, task in analysis_tasks.items():
logger.info(task.base_path)
post.merge_analysis(task.base_path)
logger.info(40 * "=")
logger.info('Iterations: {:d}'.format(n_iter_loop))
logger.info('Sim end time: {:f}'.format(self.solver.sim_time))
logger.info('Run time: {:f} sec'.format(main_loop_time))
logger.info('Run time: {:f} cpu-hr'.format(
main_loop_time / 60 / 60 *
self.de_domain.domain.dist.comm_cart.size))
logger.info('iter/sec: {:f} (main loop only)'.format(
n_iter_loop / main_loop_time))
def Rayleigh_Benard(Rayleigh=1e6,
Prandtl=1,
nz=64,
nx=None,
ny=None,
aspect=4,
fixed_flux=False,
fixed_T=False,
mixed_flux_T=True,
stress_free=False,
no_slip=True,
restart=None,
run_time=23.5,
run_time_buoyancy=None,
run_time_iter=np.inf,
run_time_therm=1,
max_writes=20,
max_slice_writes=20,
output_dt=0.2,
data_dir='./',
coeff_output=True,
verbose=False,
no_join=False,
do_bvp=False,
num_bvps=10,
bvp_convergence_factor=1e-3,
bvp_equil_time=10,
bvp_resolution_factor=1,
bvp_transient_time=30,
bvp_final_equil_time=None,
min_bvp_time=50,
first_bvp_time=20,
first_bvp_convergence_factor=1e-2,
threeD=False,
seed=42,
mesh=None,
overwrite=False):
import os
from dedalus.tools.config import config
config['logging']['filename'] = os.path.join(data_dir, 'logs/dedalus_log')
config['logging']['file_level'] = 'DEBUG'
import mpi4py.MPI
if mpi4py.MPI.COMM_WORLD.rank == 0:
if not os.path.exists('{:s}/'.format(data_dir)):
os.makedirs('{:s}/'.format(data_dir))
logdir = os.path.join(data_dir, 'logs')
if not os.path.exists(logdir):
os.mkdir(logdir)
logger = logging.getLogger(__name__)
logger.info("saving run in: {}".format(data_dir))
import time
from dedalus import public as de
from dedalus.extras import flow_tools
from dedalus.tools import post
# input parameters
logger.info("Ra = {}, Pr = {}".format(Rayleigh, Prandtl))
# Parameters
Lz = 1.
Lx = aspect * Lz
Ly = aspect * Lz
if nx is None:
nx = int(nz * aspect)
if ny is None:
ny = int(nz * aspect)
if threeD:
logger.info("resolution: [{}x{}x{}]".format(nx, ny, nz))
equations = BoussinesqEquations3D(nx=nx,
ny=ny,
nz=nz,
Lx=Lx,
Ly=Ly,
Lz=Lz,
mesh=mesh)
else:
logger.info("resolution: [{}x{}]".format(nx, nz))
equations = BoussinesqEquations2D(nx=nx, nz=nz, Lx=Lx, Lz=Lz)
equations.set_IVP(Rayleigh, Prandtl)
bc_dict = {
'fixed_flux': None,
'fixed_temperature': None,
'mixed_flux_temperature': None,
'mixed_temperature_flux': None,
'stress_free': None,
'no_slip': None
}
if mixed_flux_T:
bc_dict['mixed_flux_temperature'] = True
elif fixed_T:
bc_dict['fixed_temperature'] = True
elif fixed_flux:
bc_dict['fixed_flux'] = True
if stress_free:
bc_dict['stress_free'] = True
elif no_slip:
bc_dict['no_slip'] = True
supercrit = Rayleigh / RA_CRIT
equations.set_BC(**bc_dict)
# Build solver
ts = de.timesteppers.RK443
cfl_safety = 0.8
solver = equations.problem.build_solver(ts)
logger.info('Solver built')
checkpoint = Checkpoint(data_dir)
if isinstance(restart, type(None)):
equations.set_IC(solver, seed=seed)
dt = None
mode = 'overwrite'
else:
logger.info("restarting from {}".format(restart))
checkpoint.restart(restart, solver)
if overwrite:
mode = 'overwrite'
else:
mode = 'append'
checkpoint.set_checkpoint(solver, wall_dt=checkpoint_min * 60, mode=mode)
# Integration parameters
# if not isinstance(run_time_therm, type(None)):
# solver.stop_sim_time = run_time_therm*equations.thermal_time + solver.sim_time
# elif not isinstance(run_time_buoyancy, type(None)):
# solver.stop_sim_time = run_time_buoyancy + solver.sim_time
# else:
solver.stop_sim_time = np.inf
solver.stop_wall_time = run_time * 3600.
solver.stop_iteration = run_time_iter
Hermitian_cadence = 100
# Analysis
max_dt = output_dt
analysis_tasks = equations.initialize_output(solver,
data_dir,
coeff_output=coeff_output,
output_dt=output_dt,
mode=mode)
# CFL
CFL = flow_tools.CFL(solver,
initial_dt=0.1,
cadence=1,
safety=cfl_safety,
max_change=1.5,
min_change=0.5,
max_dt=max_dt,
threshold=0.1)
if threeD:
CFL.add_velocities(('u', 'v', 'w'))
else:
CFL.add_velocities(('u', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("Re", name='Re')
# flow.add_property("interp(w, z=0.95)", name='w near top')
# u, v, w = solver.state['u'], solver.state['v'], solver.state['w']
if do_bvp:
if not threeD:
ny = 0
atmo_class = BoussinesqEquations2D
else:
atmo_class = BoussinesqEquations3D
bvp_solver = BoussinesqBVPSolver(atmo_class, nx, ny, nz, \
flow, equations.domain.dist.comm_cart, \
solver, num_bvps, bvp_equil_time, \
threeD=threeD,
bvp_transient_time=bvp_transient_time, \
bvp_run_threshold=bvp_convergence_factor, \
bvp_l2_check_time=1, mesh=mesh,\
first_bvp_time=first_bvp_time,
first_run_threshold=first_bvp_convergence_factor,\
plot_dir='{}/bvp_plots/'.format(data_dir),\
min_avg_dt=1e-10, final_equil_time=bvp_final_equil_time,
min_bvp_time=min_bvp_time)
bc_dict.pop('stress_free')
bc_dict.pop('no_slip')
# print(equations.domain.grid(0), equations.domain.grid(1), equations.domain.grid(2))
first_step = True
# Main loop
try:
logger.info('Starting loop')
Re_avg = 0
continue_bvps = True
not_corrected_times = True
init_time = solver.sim_time
start_iter = solver.iteration
while (solver.ok and np.isfinite(Re_avg)) and continue_bvps:
dt = CFL.compute_dt()
solver.step(dt) #, trim=True)
# Solve for blow-up over long timescales in 3D due to hermitian-ness
effective_iter = solver.iteration - start_iter
if threeD and effective_iter % Hermitian_cadence == 0:
for field in solver.state.fields:
field.require_grid_space()
Re_avg = flow.grid_average('Re')
log_string = 'Iteration: {:5d}, '.format(solver.iteration)
log_string += 'Time: {:8.3e} ({:8.3e} therm), dt: {:8.3e}, '.format(
solver.sim_time, solver.sim_time / equations.thermal_time, dt)
log_string += 'Re: {:8.3e}/{:8.3e}'.format(Re_avg, flow.max('Re'))
logger.info(log_string)
if not_corrected_times and Re_avg > 1:
if not isinstance(run_time_therm, type(None)):
solver.stop_sim_time = run_time_therm * equations.thermal_time + solver.sim_time
elif not isinstance(run_time_buoyancy, type(None)):
solver.stop_sim_time = run_time_buoyancy + solver.sim_time
not_corrected_times = False
if do_bvp:
bvp_solver.update_avgs(dt, Re_avg, min_Re=np.sqrt(supercrit))
if bvp_solver.check_if_solve():
atmo_kwargs = {'nz': nz * bvp_resolution_factor, 'Lz': Lz}
diff_args = [Rayleigh, Prandtl]
bvp_solver.solve_BVP(atmo_kwargs, diff_args, bc_dict)
if bvp_solver.terminate_IVP():
continue_bvps = False
if first_step:
if verbose:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.spy(solver.pencils[0].L,
markersize=1,
markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern.png", dpi=1200)
import scipy.sparse.linalg as sla
LU = sla.splu(solver.pencils[0].LHS.tocsc(),
permc_spec='NATURAL')
fig = plt.figure()
ax = fig.add_subplot(1, 2, 1)
ax.spy(LU.L.A, markersize=1, markeredgewidth=0.0)
ax = fig.add_subplot(1, 2, 2)
ax.spy(LU.U.A, markersize=1, markeredgewidth=0.0)
fig.savefig(data_dir + "sparsity_pattern_LU.png", dpi=1200)
logger.info("{} nonzero entries in LU".format(LU.nnz))
logger.info("{} nonzero entries in LHS".format(
solver.pencils[0].LHS.tocsc().nnz))
logger.info("{} fill in factor".format(
LU.nnz / solver.pencils[0].LHS.tocsc().nnz))
first_step = False
start_time = time.time()
except:
raise
logger.error('Exception raised, triggering end of main loop.')
finally:
end_time = time.time()
main_loop_time = end_time - start_time
n_iter_loop = solver.iteration - 1
logger.info('Iterations: {:d}'.format(n_iter_loop))
logger.info('Sim end time: {:f}'.format(solver.sim_time))
logger.info('Run time: {:f} sec'.format(main_loop_time))
logger.info('Run time: {:f} cpu-hr'.format(
main_loop_time / 60 / 60 * equations.domain.dist.comm_cart.size))
logger.info('iter/sec: {:f} (main loop only)'.format(n_iter_loop /
main_loop_time))
try:
final_checkpoint = Checkpoint(data_dir,
checkpoint_name='final_checkpoint')
final_checkpoint.set_checkpoint(solver, wall_dt=1, mode=mode)
solver.step(dt) #clean this up in the future...works for now.
post.merge_process_files(data_dir + '/final_checkpoint/',
cleanup=False)
except:
raise
print('cannot save final checkpoint')
finally:
if not no_join:
logger.info('beginning join operation')
post.merge_analysis(data_dir + 'checkpoints')
for task in analysis_tasks:
logger.info(task.base_path)
post.merge_analysis(task.base_path)
logger.info(40 * "=")
logger.info('Iterations: {:d}'.format(n_iter_loop))
logger.info('Sim end time: {:f}'.format(solver.sim_time))
logger.info('Run time: {:f} sec'.format(main_loop_time))
logger.info('Run time: {:f} cpu-hr'.format(
main_loop_time / 60 / 60 *
equations.domain.dist.comm_cart.size))
logger.info('iter/sec: {:f} (main loop only)'.format(
n_iter_loop / main_loop_time))
def simulate(self,
initial_conditions=None,
scheme=de.timesteppers.RK443,
sim_time=2,
wall_time=np.inf,
tight=False):
"""Load initial conditions, run the simulation, and merge results.
Parameters:
initial_conditions (None, str): determines from what source to
draw the initial conditions. Valid options are as follows:
- None: use trivial conditions (T_ = 1 - z, T = 1 - z + eps).
- 'resume': use the most recent state file in the
records directory (load both model and DA system).
- An .h5 filename: load state variables for the model and
reset the data assimilation state variables to zero.
scheme (de.timesteppers): The kind of solver to use. Options are
RK443 (de.timesteppers.RK443), RK111, RK222, RKSMR, etc.
sim_time (float): The maximum amount of simulation time allowed
(in seconds) before ending the simulation.
wall_time (float): The maximum amound of computing time allowed
(in seconds) before ending the simulation.
tight (bool): If True, set a low cadence and min_dt for refined
simulation. If False, set a higher cadence and min_dt for a
more coarse (but faster) simulation.
"""
if not self.problem:
raise TypeError("problem not initialized (run setup())")
self.logger.debug("\n")
self.logger.debug("NEW SIMULATION")
solver = self.problem.build_solver(scheme)
self.logger.info("Solver built")
N = int(self.problem.parameters['N'])
# Initial conditions --------------------------------------------------
if initial_conditions is None: # "Trivial" conditions.
dt = 1e-4
eps = 1e-4
k = 3.117
x, z = self.problem.domain.grids(scales=1)
T, T_ = solver.state['T'], solver.state['T_']
# Start T from rest plus a small perturbation.
T['g'] = 1 - z + eps * np.sin(k * x) * np.sin(2 * np.pi * z)
T.differentiate('z', out=solver.state['Tz'])
# Start T_ from rest.
T_['g'] = 1 - z
T_.differentiate('z', out=solver.state['Tz_'])
self.logger.info("Using trivial initial conditions")
elif isinstance(initial_conditions, str): # Load data from a file.
# Resume: load the state of the last (merged) state file.
resume = initial_conditions == "resume"
if resume:
initial_conditions = self._get_merged_file("states")
if not initial_conditions.endswith(".h5"):
raise ValueError(
"'{}' is not an h5 file".format(initial_conditions))
# Load the data from the specified h5 file into the system.
self.logger.info("Loading initial conditions from {}".format(
initial_conditions))
with h5py.File(initial_conditions, 'r') as infile:
dt = infile["scales/timestep"][-1] * .01 # initial dt
errs = []
tasks = ["T", "Tz", "psi", "psiz", "zeta", "zetaz"]
if resume: # Only load assimilating variables to resume.
tasks += ["T_", "Tz_", "psi_", "psiz_", "zeta_", "zetaz_"]
solver.sim_time = infile["scales/sim_time"][-1]
niters = infile["scales/iteration"][-1]
solver.initial_iteration = niters
solver.iteration = niters
for name in tasks:
# Get task data from the h5 file (recording failures).
try:
data = infile["tasks/" + name][-1, :, :]
except KeyError as e:
errs.append("tasks/" + name)
continue
# Determine the chunk belonging to this process.
chunk = data.shape[1] // SIZE
subset = data[:, RANK * chunk:(RANK + 1) * chunk]
# Change the corresponding state variable.
scale = solver.state[name]['g'].shape[0] / \
self.problem.parameters["xsize"]
solver.state[name].set_scales(1)
solver.state[name]['g'] = subset
solver.state[name].set_scales(scale)
if errs:
raise KeyError("Missing keys in '{}': '{}'".format(
initial_conditions, "', '".join(errs)))
# Initial conditions for assimilating system: T_0 = P_4(T0).
if not resume:
G = self.problem.domain.new_field()
G['c'] = solver.state['T']['c'].copy()
solver.state['T_']['g'] = BoussinesqDataAssimilation2D.P_N(
G, 4, True)
solver.state['T_'].differentiate('z', out=solver.state['Tz_'])
# Driving / projection function arguments -----------------------------
dT = solver.state['T_'] - solver.state['T']
self.problem.parameters["driving"].args = [dT, N]
self.problem.parameters["driving"].original_args = [dT, N]
# Stopping Parameters -------------------------------------------------
solver.stop_sim_time = sim_time # Length of simulation.
solver.stop_wall_time = wall_time # Real time allowed to compute.
solver.stop_iteration = np.inf # Maximum iterations allowed.
# State snapshots -----------------------------------------------------
# Save the entire state in states/ files. USE iter, NOT sim_dt.
# NOTE: This is where BoussinesqDataAssimilation2Dmovie differs.
snaps = solver.evaluator.add_file_handler(os.path.join(
self.records_dir, "states"),
iter=1,
max_writes=5000,
mode="append")
snaps.add_task("T")
snaps.add_task("T_")
snaps.add_task("driving", name="P_N")
# Control Flow --------------------------------------------------------
if tight: # Tighter control flow (slower but safer).
cfl = flow_tools.CFL(solver,
initial_dt=dt,
cadence=1,
safety=1,
max_change=1.5,
min_change=0.01,
max_dt=0.01,
min_dt=1e-10)
else: # Looser control flow (faster but risky).
cfl = flow_tools.CFL(solver,
initial_dt=dt,
cadence=10,
safety=1,
max_change=1.5,
min_change=0.5,
max_dt=0.01,
min_dt=1e-6)
cfl.add_velocities(('v', 'w'))
cfl.add_velocities(('v_', 'w_'))
# Flow properties (print during run; not recorded in the records files)
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("sqrt(v **2 + w **2) / Ra", name='Re')
flow.add_property("sqrt(v_**2 + w_**2) / Ra", name='Re_')
# MAIN COMPUTATION LOOP -----------------------------------------------
try:
self.logger.info("Starting main loop")
start_time = time.time()
while solver.ok:
# Recompute time step and iterate.
dt = cfl.compute_dt()
dt = solver.step(dt)
# Print info to the screen every 10 iterations.
if solver.iteration % 10 == 0:
info = "Iteration {:>5d}, Time: {:.7f}, dt: {:.2e}".format(
solver.iteration, solver.sim_time, dt)
Re = flow.max("Re")
Re_ = flow.max("Re_")
info += ", Max Re = {:f}".format(Re)
info += ", Max Re_ = {:f}".format(Re_)
self.logger.info(info)
# Make sure the simulation hasn't blown up.
if np.isnan(Re) or np.isnan(Re_):
raise ValueError("Reynolds number went to infinity!!"
"\nRe = {}, Re_ = {}".format(Re, Re_))
except BaseException as e:
self.logger.error("Exception raised, triggering end of main loop.")
raise
finally:
total_time = time.time() - start_time
cpu_hr = total_time / 60 / 60 * SIZE
self.logger.info("Iterations: {:d}".format(solver.iteration))
self.logger.info("Sim end time: {:.3e}".format(solver.sim_time))
self.logger.info("Run time: {:.3e} sec".format(total_time))
self.logger.info("Run time: {:.3e} cpu-hr".format(cpu_hr))
self.logger.debug("END OF SIMULATION\n")
max_writes=200)
integ.add_task("Avg_x(vx)", name='<vx>_x', scales=1)
integ.add_task("Avg_x(vy)", name='<vy>_x', scales=1)
integ.add_task("Avg_x(Bx)", name='<Bx>_x', scales=1)
integ.add_task("Avg_x(By)", name='<By>_x', scales=1)
analysis_tasks.append(integ)
timeseries = solver.evaluator.add_file_handler(os.path.join(
data_dir, 'timeseries'),
sim_dt=1e-2)
timeseries.add_task("0.5*integ(vx**2 + vy**2)", name='Ekin')
timeseries.add_task("0.5*integ(Bx**2 + By**2)", name='Emag')
analysis_tasks.append(timeseries)
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver, cadence=1)
flow.add_property("0.5*(vx**2 + vy**2)", name='Ekin')
try:
logger.info('Starting loop')
start_run_time = time.time()
while solver.ok:
solver.step(dt)
if (solver.iteration - 1) % 1 == 0:
logger.info('Iteration: %i, Time: %e, dt: %e' %
(solver.iteration, solver.sim_time, dt))
logger.info('Max E_kin = %17.12e' % flow.max('Ekin'))
except:
logger.error('Exception raised, triggering end of main loop.')
raise
def Dedalus_Inelastic_S(Ra,Pr,Np,Ta,end_time,snaps=True):
''' Builds a problem with its equations, parameters and boundary conditions and solves it, saving values in a h5py '''
m=1.5
theta = 1-np.exp(-Np/m)
Ly, Lz = 2,1
Ny, Nz = 192,96
Lat=np.pi/4
initial_timestep = 1.5e-4 # Initial timestep
snapshot_skip=10
analysis_freq = 1.5e-3 # Frequency analysis files are outputted
end_sim_time = end_time # Stop time in simulations units
end_wall_time = np.inf # Stop time in wall time
end_iterations = np.inf # Stop time in iterations
max_dt=0.005
save_direc = "raw_data/"
pathlib.Path(save_direc).mkdir(parents=True, exist_ok=True)
# Create bases and domain
y_basis = de.Fourier('y', Ny, interval=(0, Ly), dealias=3/2) # Fourier basis in the x
z_basis = de.Chebyshev('z', Nz, interval=(0, Lz), dealias=3/2) # Chebyshev basis in the z
domain = de.Domain([y_basis, z_basis], grid_dtype=np.float64) # Defining our domain
z = domain.grid(1, scales=1) # accessing the z values
# 2D Anelastic hydrodynamics
problem = de.IVP(domain, variables=['p', 's', 'u', 'v', 'w', 'sz', 'uz', 'vz', 'wz'])
problem.meta['p','s','u','w']['z']['dirichlet'] = True
# Defining model parameters
problem.parameters['Ly'] = Ly
problem.parameters['Lz'] = Lz
problem.parameters['Ra'] = Ra
problem.parameters['Pr'] = Pr
problem.parameters['Ta'] = Ta
problem.parameters['Lat'] = Lat
problem.parameters['m'] = m
problem.parameters['theta'] = theta
problem.parameters['X'] = Ra/Pr
problem.parameters['Y'] = (Pr*Pr*theta) / Ra
problem.parameters['T'] = Ta**(1/2)
Sfilename='S-Ra:'+str(Ra)+'-Pr:'+str(Pr)+'-Ta:'+str(Ta)+'-Np:'+str(Np)
KEfilename='KE-Ra:'+str(Ra)+'-Pr:'+str(Pr)+'-Ta:'+str(Ta)+'-Np:'+str(Np)
# Non-constant coeffiecents
rho_ref = domain.new_field(name='rho_ref')
rho_ref['g'] = (1-theta*z)**m
rho_ref.meta['y']['constant'] = True
problem.parameters['rho_ref'] = rho_ref # Background state for rho
T_ref = domain.new_field(name='T_ref')
T_ref['g'] = 1-theta*z
T_ref.meta['y']['constant'] = True
problem.parameters['T_ref'] = T_ref # Background state for T
dz_rho_ref = domain.new_field(name='dz_rho_ref')
dz_rho_ref['g'] = -theta*m*((1-theta*z)**(m-1))
dz_rho_ref.meta['y']['constant'] = True
problem.parameters['dz_rho_ref'] = dz_rho_ref # z-derivative of rho_ref
# Defining d/dz of s, u, and w for reducing our equations to first order
problem.add_equation("sz - dz(s) = 0")
problem.add_equation("uz - dz(u) = 0")
problem.add_equation("vz - dz(v) = 0")
problem.add_equation("wz - dz(w) = 0")
# mass continuity with rho_ref and dz(rho_ref) expanded analytically
problem.add_equation(" (1-theta*z)*(dy(v) + wz) - theta*m*w = 0 ")
# x-component of the momentum equation
problem.add_equation(" rho_ref*( dt(u) - dy(dy(u)) - dz(uz) + T*(w*cos(Lat) - v*sin(Lat)) ) - dz_rho_ref*uz = -rho_ref*( v*dy(u) + w*uz ) ")
# y-component of the momentum equation
problem.add_equation(" rho_ref*( dt(v) - (4/3)*dy(dy(v)) - dz(vz) - (1/3)*dy(wz) + T*u*sin(Lat) ) + dy(p) - dz_rho_ref*(vz + dy(w)) = -rho_ref*( v*dy(v) + w*vz )")
# z-component of the momentum equation
problem.add_equation(" rho_ref*T_ref*( dt(w) - X*s - dy(dy(w)) - (4/3)*dz(wz) - (1/3)*dy(vz) - T*u*cos(Lat) ) + T_ref*dz(p) + theta*m*p + (2/3)*theta*m*rho_ref*( 2*wz - dy(v) ) = -rho_ref*T_ref*( v*dy(w) + w*wz )")
# entropy diffusion equation
problem.add_equation(" T_ref*( Pr*dt(s) - dy(dy(s)) - dz(sz) ) + theta*(m+1)*sz = -Pr*T_ref*( v*dy(s) + w*sz ) + 2*Y*( dy(v)*dy(v) + wz*wz + vz*dy(w) - (1/3)*(dy(v) + wz)*(dy(v) + wz) + (1/2)*(dy(u)*dy(u) + uz*uz + vz*vz + dy(w)*dy(w)) )")
# Flux equations for use in analysis outputs
problem.add_bc("left(w) = 0") # Impermeable bottom boundary
problem.add_bc("right(w) = 0", condition="(ny != 0)") # Impermeable top boundary
problem.add_bc("right(p) = 0", condition="(ny == 0)") # Required for equations to be well-posed - see https://bit.ly/2nPVWIg for a related discussion
problem.add_bc("left(uz) = 0") # Stress-free bottom boundary
problem.add_bc("right(uz) = 0") # Stress-free top boundary
problem.add_bc("left(vz) = 0")
problem.add_bc("right(vz) = 0")
problem.add_bc("right(s) = 0") # Fixed entropy at upper boundary, arbitarily set to 0
problem.add_bc("left(sz) = -1") # Fixed flux at bottom boundary, F = F_cond
# Build solver
solver = problem.build_solver(de.timesteppers.RK222)
logger.info('Solver built')
# Initial conditions
x = domain.grid(0)
z = domain.grid(1)
s = solver.state['s']
w = solver.state['w']
sz = solver.state['sz']
# Random perturbations, initialized globally for same results in parallel
gshape = domain.dist.grid_layout.global_shape(scales=1)
slices = domain.dist.grid_layout.slices(scales=1)
rand = np.random.RandomState(seed=42)
noise = rand.standard_normal(gshape)[slices]
# Linear background + perturbations damped at walls
zb, zt = z_basis.interval
pert = 1e-5*noise*(zt - z)*(z - zb)
s['g'] = pert
s.differentiate('z', out=sz)
dt = initial_timestep # Initial timestep
# Integration parameters --- Note if these are all set to np.inf, simulation will perpetually run.
solver.stop_sim_time = end_sim_time
solver.stop_wall_time = end_wall_time
solver.stop_iteration = end_iterations
# CFL criterion
CFL = flow_tools.CFL(solver,initial_dt=dt,cadence=1,safety=1,max_change=1.5,min_change=0.01,max_dt=0.01,min_dt=1e-10)
CFL.add_velocities(('v', 'w'))
# Flow properties
flow = flow_tools.GlobalFlowProperty(solver,cadence=10)
flow.add_property("sqrt(u*u + v*v + w*w)",name='Re')
# Analysis tasks
analysis = solver.evaluator.add_file_handler(save_direc +'analysis',sim_dt=analysis_freq,max_writes=5000)
analysis.add_task("s", layout='g', name='entropy')
# Flux decomposition - Internal energy equation
analysis.add_task("integ( integ( sqrt(u*u + v*v + w*w) , 'y')/Ly, 'z')/Lz", layout='g', name='Re') # Mean Reynolds number
analysis.add_task(" integ( (integ(0.5*(u*u + v*v + w*w)*rho_ref,'y')/Ly), 'z')/Lz", layout='g', name='KE') # Mean KE
# Creating a parameter file
run_parameters = solver.evaluator.add_file_handler(save_direc+'run_parameters',wall_dt=1e20,max_writes=1)
run_parameters.add_task(Ly,name="Ly")
run_parameters.add_task(Lz,name="Lz")
run_parameters.add_task(Ra,name="Ra")
run_parameters.add_task(Pr,name="Pr")
run_parameters.add_task(Np,name="Np")
run_parameters.add_task(m,name="m")
run_parameters.add_task(Ny,name="Ny")
run_parameters.add_task(Nz,name="Nz")
run_parameters.add_task("z",layout='g',name="z_grid")
run_parameters.add_task(analysis_freq,name="ana_freq")
run_parameters.add_task(max_dt,name="max_dt")
try: # Main loop
logger.info('Starting loop')
start_time = time.time()
while solver.ok:
dt = CFL.compute_dt()
dt = solver.step(dt)
time.sleep(0.02)
if (solver.iteration) == 1:
# Prints various parameters to terminal upon starting the simulation
logger.info('Parameter values imported form run_param_file.py:')
logger.info('Ly = {}, Lz = {}; (Resolution of {},{})'.format(Ly, Lz, Ny, Nz))
logger.info('Ra = {}, Pr = {}, Np = {}'.format(Ra, Pr, Np))
logger.info('Analysis files outputted every {}'.format(analysis_freq))
if end_sim_time != np.inf:
logger.info('Simulation finishes at sim_time = {}'.format(end_sim_time))
elif end_wall_time != np.inf:
logger.info('Simulation finishes at wall_time = {}'.format(end_wall_time))
elif end_iterations != np.inf:
logger.info('Simulation finishes at iteration {}'.format(end_iterations))
else:
logger.info('No clear end point defined. Simulation may run perpetually.')
if (solver.iteration-1) % 10 == 0:
# Prints progress information include maximum Reynolds number every 10 iterations
logger.info('Iteration: %i, Time: %e, dt: %e' %(solver.iteration, solver.sim_time, dt))
logger.info('Max Re = %f' %flow.max('Re'))
except:
logger.error('Exception raised, triggering end of main loop.')
raise
finally:
# Prints concluding information upon reaching the end of the simulation.
end_time = time.time()
if snaps==True:
with open(Sfilename,'w') as file:
file.write(str(gshape[0])+' '+str(gshape[1]))
logger.info('Iterations: %i' %solver.iteration)
logger.info('Sim end time: %f' %solver.sim_time)
logger.info('Run time: %.2f sec' %(end_time-start_time))
logger.info('Run time: %f cpu-hr' %((end_time-start_time)/60/60*domain.dist.comm_cart.size))
with h5py.File("raw_data/analysis/analysis_s1/analysis_s1_p0.h5", mode='r') as file:
times=file['scales']['sim_time'][:]
data=file['tasks']['entropy'][:,:,:]
ke=file['tasks']['KE'][:]
ke=np.squeeze(ke)
times=np.squeeze(times)
n=times.shape[0]
if snaps==True:
for i in range(n):
if i%snapshot_skip==0:
append_unique_value(Sfilename,times[i])
append_unique_array(Sfilename,data[i])
save_fct_txt(times,ke,KEfilename)
