def echoRD_job(mcinif='mcini',
mcpick='mc.pickle3',
runname='test',
wdir='./',
pathdir='../echoRD/',
saveDT=True,
aref='Shipitalo',
LTEdef='instant',
infilM='MDA',
exfilM='Ediss',
model_restart=False,
parallel=False,
hdf5pick=True,
update_mf=False,
update_prec=False,
update_part=False,
update_stochsoil=False,
update_md_area=False,
update_timenow=False,
legacy_pick=False,
column=False,
colref=False,
prec2D=False,
macscale=1.,
fsize=(7, 4),
w_rat=[4, 0.6],
h_rat=[1, 9]):
'''
This is a wrapper for running echoRD easily.
Be warned that some definitions are implicit in this wrapper
mcini -- echoRD ini file of run
mcpick -- pickled mode setup
runname -- name of model run
wdir -- working directory path
pathdir -- path to echoRD model
Model parameters
aref -- Advection reference [Shipitalo | Weiler | Zehe | geogene | geogene2 | obs]
infilM -- Infiltration handling [MDA | MED | rand]
LTEdef -- Infiltration assumption [instant | ks | random]
exfilM -- Exfiltration from macropores [Ediss | RWdiff]
saveDT -- optional modified time steps [True | int (factor) | double (static step)]
model_restart- True: setup new model base and start running, 'only': setup new model base and run one time step, False: reload existing model base, 'joint': reload common base setup
parallel-- flag for parallel initialisation of particles
hdf5pick-- flag using hdf5 for pickling
update_mf- change referenced ini moist file
update_part- change particle size factor (mc.part_sizefac)
update_prec- change total precip amount
update_stochsoil- change stochsoil reference file
legacy_pick- caompatibility to former runs with different state pickles
column -- True if referiing to soil column test cases
colref -- reference to circular column instead of 2D pane
prec2D -- precipitation definition over area
macscale-- scale factor for macropore coating (if other than 1)
fsize -- plotting parameter canvas size
w_rat -- plotting parameter width ratio plot and marginals
h_rat -- plotting parameter height ratio marginals and plot
'''
import numpy as np
import pandas as pd
import scipy as sp
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os, sys
try:
import cPickle as pickle
except:
import pickle
try:
import h5py
except:
hdf5pick = False
print('h5py not available - falling back to pickle use.')
#connect echoRD Tools
lib_path = os.path.abspath(pathdir)
sys.path.append(lib_path)
import vG_conv as vG
from hydro_tools import plotparticles_t, hydroprofile, plotparticles_gen #connect to echoRD
import run_echoRD as rE
#connect and load project
[dr, mc, mcp, pdyn, cinf, vG] = rE.loadconnect(pathdir='../',
mcinif=mcinif,
experimental=True)
if (model_restart == True) | (model_restart == 'only'):
mc = rE.preproc_echoRD(mc, dr, mcp, mcpick)
import macropore_ini as mpo
mpo.mac_plot(mc.macP, mc, 'macs_' + runname + '.pdf')
else:
mc = mcp.mcpick_out(mc, mcpick)
mc.advectref = aref
if update_stochsoil:
mc.stochsoil = update_stochsoil
try:
if mc.stochsoil == False:
hdf5pick = False
print(
'h5py not used as mc.stochsoil set False - falling back to pickle use.'
)
except:
mc.stochsoil = False
hdf5pick = False
print(
'h5py not used as mc.stochsoil not given - falling back to pickle use.'
)
precTS = pd.read_csv(mc.precf, sep=',', skiprows=3)
mc.prects = False
#check for generic versionings of setup:
if update_prec:
precTS.total = update_prec
precTS.intense = precTS.total / (precTS.tend - precTS.tstart)
if update_part:
mc.part_sizefac = update_part
if update_md_area:
mc.md_area *= update_md_area
if column:
total_volume = np.pi * 0.5**3
mc.particleV = total_volume / (mc.mgrid.vertgrid[0] *
mc.mgrid.latgrid[0] *
(2 * mc.part_sizefac))
mc.prects = 'column2'
mc.cloref = colref
if mc.nomac == True:
#add macropore into soilmatrix for column test
idx = int(np.shape(mc.soilgrid)[1] / 2)
mc.soilgrid[:, idx - 1:idx + 1] = 13
#check for previous runs to hook into
try:
if model_restart == True:
raise ValueError('I will overwrite any existing runfile.')
#unpickle:
if hdf5pick:
particles = pd.read_hdf('./results/P_' + runname + '.h5', 'table')
with h5py.File('./results/S_' + runname + '.h5', 'r') as f:
dset = f['states'][0]
[t, len_parts, leftover, len_drain, ix] = dset[:]
if ix > 0:
ix += 1
ix = int(ix)
leftover = int(leftover)
else:
with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']),
'rb') as handle:
pickle_l = pickle.load(handle)
dummyx = pickle.loads(pickle_l)
particles = pickle.loads(dummyx[0])
if legacy_pick:
[leftover, drained, t, TSstore,
ix] = pickle.loads(dummyx[1])
dummy = np.floor(mc.t_end / mc.t_out)
mc.mgrid['cells'] = np.shape(TSstore)[0]
[thS, npart] = pdyn.gridupdate_thS(particles.lat,
particles.z, mc)
thetastore = np.zeros((int(dummy), mc.mgrid.vertgrid[0],
mc.mgrid.latgrid[0]))
mc.t_end = 24. * 3600.
mc.t_out = 60
else:
[leftover, drained, t, TSstore, thetastore, npart,
ix] = pickle.loads(dummyx[1])
if ix > 0:
ix += 1
print('resuming into stored run at t=' + str(t) + '...')
if model_restart == 'joint':
#rename runfile now
runname = runname + str(int(np.round(macscale * 10))).zfill(2)
# define particle size
# WARNING: as in any model so far, we have a volume problem here.
# we consider all parts of the domain as static in volume at this stage.
# however, we will work on a revision of this soon.
mc.gridcellA = mc.mgrid.vertfac * mc.mgrid.latfac
mc.particleA = abs(mc.gridcellA.values) / (
2 * mc.part_sizefac
) #assume average ks at about 0.5 as reference of particle size
mc.particleD = 2. * np.sqrt(mc.particleA / np.pi)
mc.particleV = 3. / 4. * np.pi * (mc.particleD / 2.)**3.
mc.particleD /= np.sqrt(abs(mc.gridcellA.values))
mc.particleV /= np.sqrt(abs(
mc.gridcellA.values)) #assume grid size as 3rd dimension
mc.particlemass = dr.waterdensity(np.array(20), np.array(
-9999)) * mc.particleV #assume 20C as reference for particle mass
#DEBUG: a) we assume 2D=3D; b) change 20C to annual mean T?
#initialise bins and slopes
mc = dr.ini_bins(mc)
mc = dr.mc_diffs(mc, np.max(np.max(mc.mxbin)))
# estimate macropore capacity as relative share of space (as particle volume is not well-defined for the 1D-2D-3D references)
mc.maccap = np.ceil(
(2. * mc.md_area /
(-mc.gridcellA.values * mc.mgrid.latgrid.values)) *
mc.part_sizefac)
mc.mactopfill = np.ceil(
(2. * mc.md_area /
(-mc.gridcellA.values * mc.mgrid.latgrid.values)) *
mc.part_sizefac)[:, 0] * 0. #all empty
# assumption: the pore space is converted into particles through mc.part_sizefac. this is now reprojected to the macropore by using the areal share of the macropores
# DEBUG: there is still some inconcistency about the areas and volumes, but it may be a valid estimate with rather few assumptions
mc.mgrid['cells'] = len(mc.mxbin.ravel())
if hdf5pick:
[thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc)
except:
if update_mf:
mc.inimf = update_mf
#initialise particles
input()
[mc, particles, npart] = dr.particle_setup(mc, paral=parallel)
t = 0.
ix = 0
leftover = 0
try:
mc.t_out = mc.t_out
except:
mc.t_out = 60.
dummy = np.floor(mc.t_end / mc.t_out)
TSstore = np.zeros((int(dummy), mc.mgrid.cells[0], 2))
thetastore = np.zeros(
(int(dummy), mc.mgrid.vertgrid[0], mc.mgrid.latgrid[0]))
drained = pd.DataFrame(np.array([]))
if hdf5pick:
particles.to_hdf('./results/P_' + runname + '.h5', 'table')
with h5py.File('./results/S_' + runname + '.h5', 'w') as f:
dset = f.create_dataset("states", (1, 5), dtype='f')
dset[:] = [t, len(particles), leftover, len(drained), ix]
print('starting new run...')
#check for NaNs in lookup table for first bins and set to next filled ones:
#DEBUG: actually this should not happen. but it did.
for i in np.arange(np.shape(mc.D)[1]):
idx = np.where(np.isnan(mc.D[:, i]))[0]
mc.D[idx, i] = mc.D[np.amax(idx) + 1, i]
#define bin assignment mode for infiltration particles
mc.LTEdef = LTEdef #'instant'#'ks' #'instant' #'random'
mc.LTEmemory = mc.soilgrid.ravel() * 0.
#macropore reference
mc.maccon = np.where(
mc.macconnect.ravel() > 0)[0] #index of all connected cells
mc.md_macdepth = np.abs(mc.md_macdepth)
#############
# Run Model #
#############
mc.LTEpercentile = 70 #new parameter
t_end = mc.t_end
output = mc.t_out #mind to set also in TXstore.index definition
dummy = np.floor(t_end / output)
infiltmeth = infilM
exfiltmeth = exfilM
film = True
clogswitch = False
infiltscale = False
#just store the base files
if model_restart == 'only':
i = 0
plotparticles_gen(particles,
mc,
pdyn,
vG,
runname,
t,
i,
saving=True,
relative=False,
wdir=wdir,
fsize=fsize,
w_rat=w_rat,
h_rat=h_rat)
[thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc)
if hdf5pick:
with h5py.File(mc.stochsoil, 'r+') as f:
dset = f["theta"]
dset[:, :, i] = np.reshape(
(mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] +
(mc.soilmatrix.ts - mc.soilmatrix.tr
)[mc.soilgrid.ravel() - 1] * thS.ravel() * 0.01).values,
np.shape(thS))
particles.to_hdf('./results/P_' + runname + '.h5', 'table')
with h5py.File('./results/S_' + runname + '.h5', 'r+') as f:
dset = f["states"]
dset = [t, len(particles), leftover, len(drained), ix]
print('wrote initial states to P' + runname + 'h5 and S_' +
runname + '.h5')
else:
TSstore[i, :, :] = rE.part_store(particles, mc)
thetastore[i, :, :] = np.reshape(
(mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] +
(mc.soilmatrix.ts - mc.soilmatrix.tr)[mc.soilgrid.ravel() - 1]
* thS.ravel() * 0.01).values, np.shape(thS))
with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']),
'wb') as handle:
pickle.dump(pickle.dumps([
pickle.dumps(particles),
pickle.dumps(
[leftover, drained, t, TSstore, thetastore, npart, i])
]),
handle,
protocol=2)
print('wrote initial states to Z' + runname + '_Mstat.pick')
else:
if update_timenow:
ix = int(update_timenow / mc.t_out)
print('Time set manually to ' + str(ix * mc.t_out) + 's')
drained = pd.DataFrame(np.array(
[])) #new drained dataframe as it is not stored yet
# loop through plot cycles
for i in np.arange(dummy.astype(int))[ix:]:
plotparticles_gen(particles,
mc,
pdyn,
vG,
runname,
t,
i,
saving=True,
relative=False,
wdir=wdir)
[particles, npart, thS, leftover, drained,
t] = rE.CAOSpy_rundx1(i * output, (i + 1) * output,
mc,
pdyn,
cinf,
precTS,
particles,
leftover,
drained,
6.,
splitfac=4,
prec_2D=prec2D,
maccoat=macscale,
saveDT=saveDT,
clogswitch=clogswitch,
infilt_method=infiltmeth,
exfilt_method=exfiltmeth,
film=film,
infiltscale=infiltscale)
if hdf5pick:
with h5py.File(mc.stochsoil, 'r+') as f:
dset = f["theta"]
dset[:, :, i] = np.reshape(
(mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] +
(mc.soilmatrix.ts -
mc.soilmatrix.tr)[mc.soilgrid.ravel() - 1] *
thS.ravel() * 0.01).values, np.shape(thS))
particles.to_hdf('./results/P_' + runname + '.h5', 'table')
with h5py.File('./results/S_' + runname + '.h5', 'r+') as f:
dset = f["states"]
dset = [t, len(particles), leftover, len(drained), ix]
else:
TSstore[i, :, :] = rE.part_store(particles, mc)
thetastore[i, :, :] = np.reshape(
(mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] +
(mc.soilmatrix.ts - mc.soilmatrix.tr
)[mc.soilgrid.ravel() - 1] * thS.ravel() * 0.01).values,
np.shape(thS))
with open(
''.join([wdir, '/results/Z', runname, '_Mstat.pick']),
'wb') as handle:
pickle.dump(pickle.dumps([
pickle.dumps(particles),
pickle.dumps([
leftover, drained, t, TSstore, thetastore, npart, i
])
]),
handle,
protocol=2)
runname,
t,
i,
saving=True,
relative=False,
wdir=wdir)
[particles, npart, thS, leftover, drained,
t] = rE.CAOSpy_rundx1(i * output, (i + 1) * output,
mc,
pdyn,
cinf,
precTS,
particles,
leftover,
drained,
6.,
splitfac=4,
prec_2D=False,
maccoat=macscale,
saveDT=saveDT,
clogswitch=clogswitch,
infilt_method=infiltmeth,
exfilt_method=exfiltmeth,
film=film,
infiltscale=infiltscale)
TSstore[i, :, :] = rE.part_store(particles, mc)
if i / 5. == np.round(i / 5.):
with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']),
'wb') as handle:
pickle.dump(pickle.dumps([
pickle.dumps(particles),
def echoRD_job(mcinif='mcini',
mcpick='mc.pickle3',
runname='test',
wdir='./',
pathdir='../echoRD/',
saveDT=True,
aref='Shipitalo',
LTEdef='instant',
infilM='MDA',
exfilM='Ediss',
parallel=False,
largepick=False):
'''
This is a wrapper for running echoRD easily.
Be warned that some definitions are implicit in this wrapper
mcini -- echoRD ini file of run
mcpick -- pickled mode setup
runname -- name of model run
wdir -- working directory path
pathdir -- path to echoRD model
Model parameters
aref -- Advection reference [Shipitalo | Weiler | Zehe | geogene | geogene2 | obs]
infilM -- Infiltration handling [MDA | MED | rand]
LTEdef -- Infiltration assumption [instant | ks | random]
exfilM -- Exfiltration from macropores [Ediss | RWdiff]
saveDT -- optional modified time steps [True | int (factor) | double (static step)]
parallel-- flag for parallel initialisation of particles
largepick- flag for pickling many particles
'''
import numpy as np
import pandas as pd
import scipy as sp
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os, sys
try:
import cPickle as pickle
except:
import pickle
#connect echoRD Tools
lib_path = os.path.abspath(pathdir)
sys.path.append(lib_path)
import vG_conv as vG
from hydro_tools import plotparticles_t, hydroprofile, plotparticles_weier
#connect to echoRD
import run_echoRD as rE
#connect and load project
[dr, mc, mcp, pdyn, cinf, vG] = rE.loadconnect(pathdir='../',
mcinif=mcinif,
experimental=True)
mc = mcp.mcpick_out(mc, mcpick)
mc.advectref = aref
precTS = pd.read_csv(mc.precf, sep=',', skiprows=3)
mc.prects = False
#check for previous runs to hook into
try:
#unpickle:
if largepick:
with open(''.join([wdir, '/results/L', runname, '_Mstat.pick']),
'rb') as handle:
pickle_l = pickle.load(handle)
dummyx = pickle.loads(pickle_l)
particles = pickle.loads(dummyx[0])
[t, ix] = pickle.loads(dummyx[1])
ix += 1
else:
with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']),
'rb') as handle:
pickle_l = pickle.load(handle)
dummyx = pickle.loads(pickle_l)
particles = pickle.loads(dummyx[0])
[leftover, drained, t, TSstore, thetastore, npart,
ix] = pickle.loads(dummyx[1])
ix += 1
print('resuming into stored run at t=' + str(t) + '...')
# define particle size
# WARNING: as in any model so far, we have a volume problem here.
# we consider all parts of the domain as static in volume at this stage.
# however, we will work on a revision of this soon.
mc.gridcellA = mc.mgrid.vertfac * mc.mgrid.latfac
mc.particleA = abs(mc.gridcellA.values) / (
2 * mc.part_sizefac
) #assume average ks at about 0.5 as reference of particle size
mc.particleD = 2. * np.sqrt(mc.particleA / np.pi)
mc.particleV = 3. / 4. * np.pi * (mc.particleD / 2.)**3.
mc.particleD /= np.sqrt(abs(mc.gridcellA.values))
mc.particleV /= np.sqrt(abs(
mc.gridcellA.values)) #assume grid size as 3rd dimension
mc.particlemass = dr.waterdensity(np.array(20), np.array(
-9999)) * mc.particleV #assume 20C as reference for particle mass
#DEBUG: a) we assume 2D=3D; b) change 20C to annual mean T?
#initialise bins and slopes
mc = dr.ini_bins(mc)
mc = dr.mc_diffs(mc, np.max(np.max(mc.mxbin)))
# estimate macropore capacity as relative share of space (as particle volume is not well-defined for the 1D-2D-3D references)
mc.maccap = np.ceil(
(2. * mc.md_area /
(-mc.gridcellA.values * mc.mgrid.latgrid.values)) *
mc.part_sizefac)
mc.mactopfill = np.ceil(
(2. * mc.md_area /
(-mc.gridcellA.values * mc.mgrid.latgrid.values)) *
mc.part_sizefac)[:, 0] * 0. #all empty
# assumption: the pore space is converted into particles through mc.part_sizefac. this is now reprojected to the macropore by using the areal share of the macropores
# DEBUG: there is still some inconcistency about the areas and volumes, but it may be a valid estimate with rather few assumptions
if largepick:
[thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc)
mc.mgrid['cells'] = len(npart.ravel())
except:
#initialise particles
[mc, particles, npart] = dr.particle_setup(mc, paral=parallel)
t = 0.
ix = 0
leftover = 0
dummy = np.floor(mc.t_end / mc.t_out)
TSstore = np.zeros((int(dummy), mc.mgrid.cells[0], 2))
thetastore = np.zeros(
(int(dummy), mc.mgrid.vertgrid[0], mc.mgrid.latgrid[0]))
print('starting new run...')
#define bin assignment mode for infiltration particles
mc.LTEdef = LTEdef #'instant'#'ks' #'instant' #'random'
mc.LTEmemory = mc.soilgrid.ravel() * 0.
#macropore reference
mc.maccon = np.where(
mc.macconnect.ravel() > 0)[0] #index of all connected cells
mc.md_macdepth = np.abs(mc.md_macdepth)
#############
# Run Model #
#############
mc.LTEpercentile = 70 #new parameter
t_end = mc.t_end
infiltmeth = infilM
exfiltmeth = exfilM
film = True
macscale = 1. #scale the macropore coating
clogswitch = False
infiltscale = False
drained = pd.DataFrame(np.array([]))
output = mc.t_out #mind to set also in TXstore.index definition
dummy = np.floor(t_end / output)
#loop through plot cycles
for i in np.arange(dummy.astype(int))[ix:]:
plotparticles_weier(particles,
mc,
pdyn,
vG,
runname,
t,
i,
saving=True,
relative=False,
wdir=wdir)
[particles, npart, thS, leftover, drained,
t] = rE.CAOSpy_rundx1(i * output, (i + 1) * output,
mc,
pdyn,
cinf,
precTS,
particles,
leftover,
drained,
6.,
splitfac=4,
prec_2D=False,
maccoat=macscale,
saveDT=saveDT,
clogswitch=clogswitch,
infilt_method=infiltmeth,
exfilt_method=exfiltmeth,
film=film,
infiltscale=infiltscale)
TSstore[i, :, :] = rE.part_store(particles, mc)
thetastore[i, :, :] = np.reshape(
(mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] +
(mc.soilmatrix.ts - mc.soilmatrix.tr)[mc.soilgrid.ravel() - 1] *
thS.ravel() * 0.01).values, np.shape(thS))
if largepick:
with open(''.join([wdir, '/results/L', runname, '_Mstat.pick']),
'wb') as handle:
pickle.dump(pickle.dumps(
[pickle.dumps(particles),
pickle.dumps([t, i])]),
handle,
protocol=2)
else:
with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']),
'wb') as handle:
pickle.dump(pickle.dumps([
pickle.dumps(particles),
pickle.dumps(
[leftover, drained, t, TSstore, thetastore, npart, i])
]),
handle,
protocol=2)
