except: import pickle #connect echoRD Tools pathdir = '../echoRD/' #path to echoRD lib_path = os.path.abspath(pathdir) sys.path.append(lib_path) import vG_conv as vG from hydro_tools import plotparticles_t, hydroprofile, plotparticles_weier # Prepare echoRD #connect to echoRD import run_echoRD as rE #connect and load project [dr, mc, mcp, pdyn, cinf, vG] = rE.loadconnect(pathdir='../', mcinif='mcini_weierbach_z01', experimental=True) mc = mcp.mcpick_out(mc, 'weierbach_z01.pickle') runname = 'weierbach_z01' mc.advectref = 'Shipitalo' mc.soilmatrix = pd.read_csv(mc.matrixbf, sep=' ') mc.soilmatrix['m'] = np.fmax(1 - 1 / mc.soilmatrix.n, 0.1) mc.md_macdepth = mc.md_depth[np.fmax( 2, np.sum(np.ceil(mc.md_contact), axis=1).astype(int))] mc.md_macdepth[mc.md_macdepth <= 0.] = 0.065 precTS = pd.read_csv(mc.precf, sep=',', skiprows=3)
def echoRD_job(mcinif='mcini', mcpick='mc.pickle3', runname='test', wdir='./', pathdir='../echoRD/', saveDT=True, aref='Shipitalo', LTEdef='instant', infilM='MDA', exfilM='Ediss', model_restart=False, parallel=False, hdf5pick=True, update_mf=False, update_prec=False, update_part=False, update_stochsoil=False, update_md_area=False, update_timenow=False, legacy_pick=False, column=False, colref=False, prec2D=False, macscale=1., fsize=(7, 4), w_rat=[4, 0.6], h_rat=[1, 9]): ''' This is a wrapper for running echoRD easily. Be warned that some definitions are implicit in this wrapper mcini -- echoRD ini file of run mcpick -- pickled mode setup runname -- name of model run wdir -- working directory path pathdir -- path to echoRD model Model parameters aref -- Advection reference [Shipitalo | Weiler | Zehe | geogene | geogene2 | obs] infilM -- Infiltration handling [MDA | MED | rand] LTEdef -- Infiltration assumption [instant | ks | random] exfilM -- Exfiltration from macropores [Ediss | RWdiff] saveDT -- optional modified time steps [True | int (factor) | double (static step)] model_restart- True: setup new model base and start running, 'only': setup new model base and run one time step, False: reload existing model base, 'joint': reload common base setup parallel-- flag for parallel initialisation of particles hdf5pick-- flag using hdf5 for pickling update_mf- change referenced ini moist file update_part- change particle size factor (mc.part_sizefac) update_prec- change total precip amount update_stochsoil- change stochsoil reference file legacy_pick- caompatibility to former runs with different state pickles column -- True if referiing to soil column test cases colref -- reference to circular column instead of 2D pane prec2D -- precipitation definition over area macscale-- scale factor for macropore coating (if other than 1) fsize -- plotting parameter canvas size w_rat -- plotting parameter width ratio plot and marginals h_rat -- plotting parameter height ratio marginals and plot ''' import numpy as np import pandas as pd import scipy as sp import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import os, sys try: import cPickle as pickle except: import pickle try: import h5py except: hdf5pick = False print('h5py not available - falling back to pickle use.') #connect echoRD Tools lib_path = os.path.abspath(pathdir) sys.path.append(lib_path) import vG_conv as vG from hydro_tools import plotparticles_t, hydroprofile, plotparticles_gen #connect to echoRD import run_echoRD as rE #connect and load project [dr, mc, mcp, pdyn, cinf, vG] = rE.loadconnect(pathdir='../', mcinif=mcinif, experimental=True) if (model_restart == True) | (model_restart == 'only'): mc = rE.preproc_echoRD(mc, dr, mcp, mcpick) import macropore_ini as mpo mpo.mac_plot(mc.macP, mc, 'macs_' + runname + '.pdf') else: mc = mcp.mcpick_out(mc, mcpick) mc.advectref = aref if update_stochsoil: mc.stochsoil = update_stochsoil try: if mc.stochsoil == False: hdf5pick = False print( 'h5py not used as mc.stochsoil set False - falling back to pickle use.' ) except: mc.stochsoil = False hdf5pick = False print( 'h5py not used as mc.stochsoil not given - falling back to pickle use.' ) precTS = pd.read_csv(mc.precf, sep=',', skiprows=3) mc.prects = False #check for generic versionings of setup: if update_prec: precTS.total = update_prec precTS.intense = precTS.total / (precTS.tend - precTS.tstart) if update_part: mc.part_sizefac = update_part if update_md_area: mc.md_area *= update_md_area if column: total_volume = np.pi * 0.5**3 mc.particleV = total_volume / (mc.mgrid.vertgrid[0] * mc.mgrid.latgrid[0] * (2 * mc.part_sizefac)) mc.prects = 'column2' mc.cloref = colref if mc.nomac == True: #add macropore into soilmatrix for column test idx = int(np.shape(mc.soilgrid)[1] / 2) mc.soilgrid[:, idx - 1:idx + 1] = 13 #check for previous runs to hook into try: if model_restart == True: raise ValueError('I will overwrite any existing runfile.') #unpickle: if hdf5pick: particles = pd.read_hdf('./results/P_' + runname + '.h5', 'table') with h5py.File('./results/S_' + runname + '.h5', 'r') as f: dset = f['states'][0] [t, len_parts, leftover, len_drain, ix] = dset[:] if ix > 0: ix += 1 ix = int(ix) leftover = int(leftover) else: with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']), 'rb') as handle: pickle_l = pickle.load(handle) dummyx = pickle.loads(pickle_l) particles = pickle.loads(dummyx[0]) if legacy_pick: [leftover, drained, t, TSstore, ix] = pickle.loads(dummyx[1]) dummy = np.floor(mc.t_end / mc.t_out) mc.mgrid['cells'] = np.shape(TSstore)[0] [thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc) thetastore = np.zeros((int(dummy), mc.mgrid.vertgrid[0], mc.mgrid.latgrid[0])) mc.t_end = 24. * 3600. mc.t_out = 60 else: [leftover, drained, t, TSstore, thetastore, npart, ix] = pickle.loads(dummyx[1]) if ix > 0: ix += 1 print('resuming into stored run at t=' + str(t) + '...') if model_restart == 'joint': #rename runfile now runname = runname + str(int(np.round(macscale * 10))).zfill(2) # define particle size # WARNING: as in any model so far, we have a volume problem here. # we consider all parts of the domain as static in volume at this stage. # however, we will work on a revision of this soon. mc.gridcellA = mc.mgrid.vertfac * mc.mgrid.latfac mc.particleA = abs(mc.gridcellA.values) / ( 2 * mc.part_sizefac ) #assume average ks at about 0.5 as reference of particle size mc.particleD = 2. * np.sqrt(mc.particleA / np.pi) mc.particleV = 3. / 4. * np.pi * (mc.particleD / 2.)**3. mc.particleD /= np.sqrt(abs(mc.gridcellA.values)) mc.particleV /= np.sqrt(abs( mc.gridcellA.values)) #assume grid size as 3rd dimension mc.particlemass = dr.waterdensity(np.array(20), np.array( -9999)) * mc.particleV #assume 20C as reference for particle mass #DEBUG: a) we assume 2D=3D; b) change 20C to annual mean T? #initialise bins and slopes mc = dr.ini_bins(mc) mc = dr.mc_diffs(mc, np.max(np.max(mc.mxbin))) # estimate macropore capacity as relative share of space (as particle volume is not well-defined for the 1D-2D-3D references) mc.maccap = np.ceil( (2. * mc.md_area / (-mc.gridcellA.values * mc.mgrid.latgrid.values)) * mc.part_sizefac) mc.mactopfill = np.ceil( (2. * mc.md_area / (-mc.gridcellA.values * mc.mgrid.latgrid.values)) * mc.part_sizefac)[:, 0] * 0. #all empty # assumption: the pore space is converted into particles through mc.part_sizefac. this is now reprojected to the macropore by using the areal share of the macropores # DEBUG: there is still some inconcistency about the areas and volumes, but it may be a valid estimate with rather few assumptions mc.mgrid['cells'] = len(mc.mxbin.ravel()) if hdf5pick: [thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc) except: if update_mf: mc.inimf = update_mf #initialise particles input() [mc, particles, npart] = dr.particle_setup(mc, paral=parallel) t = 0. ix = 0 leftover = 0 try: mc.t_out = mc.t_out except: mc.t_out = 60. dummy = np.floor(mc.t_end / mc.t_out) TSstore = np.zeros((int(dummy), mc.mgrid.cells[0], 2)) thetastore = np.zeros( (int(dummy), mc.mgrid.vertgrid[0], mc.mgrid.latgrid[0])) drained = pd.DataFrame(np.array([])) if hdf5pick: particles.to_hdf('./results/P_' + runname + '.h5', 'table') with h5py.File('./results/S_' + runname + '.h5', 'w') as f: dset = f.create_dataset("states", (1, 5), dtype='f') dset[:] = [t, len(particles), leftover, len(drained), ix] print('starting new run...') #check for NaNs in lookup table for first bins and set to next filled ones: #DEBUG: actually this should not happen. but it did. for i in np.arange(np.shape(mc.D)[1]): idx = np.where(np.isnan(mc.D[:, i]))[0] mc.D[idx, i] = mc.D[np.amax(idx) + 1, i] #define bin assignment mode for infiltration particles mc.LTEdef = LTEdef #'instant'#'ks' #'instant' #'random' mc.LTEmemory = mc.soilgrid.ravel() * 0. #macropore reference mc.maccon = np.where( mc.macconnect.ravel() > 0)[0] #index of all connected cells mc.md_macdepth = np.abs(mc.md_macdepth) ############# # Run Model # ############# mc.LTEpercentile = 70 #new parameter t_end = mc.t_end output = mc.t_out #mind to set also in TXstore.index definition dummy = np.floor(t_end / output) infiltmeth = infilM exfiltmeth = exfilM film = True clogswitch = False infiltscale = False #just store the base files if model_restart == 'only': i = 0 plotparticles_gen(particles, mc, pdyn, vG, runname, t, i, saving=True, relative=False, wdir=wdir, fsize=fsize, w_rat=w_rat, h_rat=h_rat) [thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc) if hdf5pick: with h5py.File(mc.stochsoil, 'r+') as f: dset = f["theta"] dset[:, :, i] = np.reshape( (mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] + (mc.soilmatrix.ts - mc.soilmatrix.tr )[mc.soilgrid.ravel() - 1] * thS.ravel() * 0.01).values, np.shape(thS)) particles.to_hdf('./results/P_' + runname + '.h5', 'table') with h5py.File('./results/S_' + runname + '.h5', 'r+') as f: dset = f["states"] dset = [t, len(particles), leftover, len(drained), ix] print('wrote initial states to P' + runname + 'h5 and S_' + runname + '.h5') else: TSstore[i, :, :] = rE.part_store(particles, mc) thetastore[i, :, :] = np.reshape( (mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] + (mc.soilmatrix.ts - mc.soilmatrix.tr)[mc.soilgrid.ravel() - 1] * thS.ravel() * 0.01).values, np.shape(thS)) with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']), 'wb') as handle: pickle.dump(pickle.dumps([ pickle.dumps(particles), pickle.dumps( [leftover, drained, t, TSstore, thetastore, npart, i]) ]), handle, protocol=2) print('wrote initial states to Z' + runname + '_Mstat.pick') else: if update_timenow: ix = int(update_timenow / mc.t_out) print('Time set manually to ' + str(ix * mc.t_out) + 's') drained = pd.DataFrame(np.array( [])) #new drained dataframe as it is not stored yet # loop through plot cycles for i in np.arange(dummy.astype(int))[ix:]: plotparticles_gen(particles, mc, pdyn, vG, runname, t, i, saving=True, relative=False, wdir=wdir) [particles, npart, thS, leftover, drained, t] = rE.CAOSpy_rundx1(i * output, (i + 1) * output, mc, pdyn, cinf, precTS, particles, leftover, drained, 6., splitfac=4, prec_2D=prec2D, maccoat=macscale, saveDT=saveDT, clogswitch=clogswitch, infilt_method=infiltmeth, exfilt_method=exfiltmeth, film=film, infiltscale=infiltscale) if hdf5pick: with h5py.File(mc.stochsoil, 'r+') as f: dset = f["theta"] dset[:, :, i] = np.reshape( (mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] + (mc.soilmatrix.ts - mc.soilmatrix.tr)[mc.soilgrid.ravel() - 1] * thS.ravel() * 0.01).values, np.shape(thS)) particles.to_hdf('./results/P_' + runname + '.h5', 'table') with h5py.File('./results/S_' + runname + '.h5', 'r+') as f: dset = f["states"] dset = [t, len(particles), leftover, len(drained), ix] else: TSstore[i, :, :] = rE.part_store(particles, mc) thetastore[i, :, :] = np.reshape( (mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] + (mc.soilmatrix.ts - mc.soilmatrix.tr )[mc.soilgrid.ravel() - 1] * thS.ravel() * 0.01).values, np.shape(thS)) with open( ''.join([wdir, '/results/Z', runname, '_Mstat.pick']), 'wb') as handle: pickle.dump(pickle.dumps([ pickle.dumps(particles), pickle.dumps([ leftover, drained, t, TSstore, thetastore, npart, i ]) ]), handle, protocol=2)
pathdir = '../echoRD/' #path to echoRD #pathdir='../' #path to echoRD lib_path = os.path.abspath(pathdir) sys.path.append(lib_path) import vG_conv as vG from hydro_tools import plotparticles_t, hydroprofile, plotparticles_specht # Read Observations obs = pd.read_csv('brspecht.dat', delimiter='\t') obs.index = np.arange(-0.05, -1., -0.1) # Load echoRD setup #connect to echoRD import run_echoRD as rE #connect and load project [dr, mc, mcp, pdyn, cinf, vG] = rE.loadconnect(pathdir=pathdir, mcinif='mcini_specht4y') #[mc,particles,npart,precTS]=mcp.echoRD_pick_out(mc,'weiherbach_testcase.pickle') mcp.mcpick_out(mc, 'specht4y_x2.pickle') mc.advectref = 'Shipitalo' [mc, particles, npart] = dr.particle_setup(mc) precTS = pd.read_csv(mc.precf, sep=',', skiprows=3) mc.prects = False [thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc) #[A,B]=plotparticles_t_obs(particles,obsx,thS/100.,mc,vG,store=True) #shift sprinkling beginning mc.advectref = 'Shipitalo' mc.prects = False precTS.tstart = 60 precTS.tend = 60 + 2.3 * 3600
def echoRD_job(mcinif='mcini', mcpick='mc.pickle3', runname='test', wdir='./', pathdir='../echoRD/', saveDT=True, aref='Shipitalo', LTEdef='instant', infilM='MDA', exfilM='Ediss', parallel=False, largepick=False): ''' This is a wrapper for running echoRD easily. Be warned that some definitions are implicit in this wrapper mcini -- echoRD ini file of run mcpick -- pickled mode setup runname -- name of model run wdir -- working directory path pathdir -- path to echoRD model Model parameters aref -- Advection reference [Shipitalo | Weiler | Zehe | geogene | geogene2 | obs] infilM -- Infiltration handling [MDA | MED | rand] LTEdef -- Infiltration assumption [instant | ks | random] exfilM -- Exfiltration from macropores [Ediss | RWdiff] saveDT -- optional modified time steps [True | int (factor) | double (static step)] parallel-- flag for parallel initialisation of particles largepick- flag for pickling many particles ''' import numpy as np import pandas as pd import scipy as sp import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import os, sys try: import cPickle as pickle except: import pickle #connect echoRD Tools lib_path = os.path.abspath(pathdir) sys.path.append(lib_path) import vG_conv as vG from hydro_tools import plotparticles_t, hydroprofile, plotparticles_weier #connect to echoRD import run_echoRD as rE #connect and load project [dr, mc, mcp, pdyn, cinf, vG] = rE.loadconnect(pathdir='../', mcinif=mcinif, experimental=True) mc = mcp.mcpick_out(mc, mcpick) mc.advectref = aref precTS = pd.read_csv(mc.precf, sep=',', skiprows=3) mc.prects = False #check for previous runs to hook into try: #unpickle: if largepick: with open(''.join([wdir, '/results/L', runname, '_Mstat.pick']), 'rb') as handle: pickle_l = pickle.load(handle) dummyx = pickle.loads(pickle_l) particles = pickle.loads(dummyx[0]) [t, ix] = pickle.loads(dummyx[1]) ix += 1 else: with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']), 'rb') as handle: pickle_l = pickle.load(handle) dummyx = pickle.loads(pickle_l) particles = pickle.loads(dummyx[0]) [leftover, drained, t, TSstore, thetastore, npart, ix] = pickle.loads(dummyx[1]) ix += 1 print('resuming into stored run at t=' + str(t) + '...') # define particle size # WARNING: as in any model so far, we have a volume problem here. # we consider all parts of the domain as static in volume at this stage. # however, we will work on a revision of this soon. mc.gridcellA = mc.mgrid.vertfac * mc.mgrid.latfac mc.particleA = abs(mc.gridcellA.values) / ( 2 * mc.part_sizefac ) #assume average ks at about 0.5 as reference of particle size mc.particleD = 2. * np.sqrt(mc.particleA / np.pi) mc.particleV = 3. / 4. * np.pi * (mc.particleD / 2.)**3. mc.particleD /= np.sqrt(abs(mc.gridcellA.values)) mc.particleV /= np.sqrt(abs( mc.gridcellA.values)) #assume grid size as 3rd dimension mc.particlemass = dr.waterdensity(np.array(20), np.array( -9999)) * mc.particleV #assume 20C as reference for particle mass #DEBUG: a) we assume 2D=3D; b) change 20C to annual mean T? #initialise bins and slopes mc = dr.ini_bins(mc) mc = dr.mc_diffs(mc, np.max(np.max(mc.mxbin))) # estimate macropore capacity as relative share of space (as particle volume is not well-defined for the 1D-2D-3D references) mc.maccap = np.ceil( (2. * mc.md_area / (-mc.gridcellA.values * mc.mgrid.latgrid.values)) * mc.part_sizefac) mc.mactopfill = np.ceil( (2. * mc.md_area / (-mc.gridcellA.values * mc.mgrid.latgrid.values)) * mc.part_sizefac)[:, 0] * 0. #all empty # assumption: the pore space is converted into particles through mc.part_sizefac. this is now reprojected to the macropore by using the areal share of the macropores # DEBUG: there is still some inconcistency about the areas and volumes, but it may be a valid estimate with rather few assumptions if largepick: [thS, npart] = pdyn.gridupdate_thS(particles.lat, particles.z, mc) mc.mgrid['cells'] = len(npart.ravel()) except: #initialise particles [mc, particles, npart] = dr.particle_setup(mc, paral=parallel) t = 0. ix = 0 leftover = 0 dummy = np.floor(mc.t_end / mc.t_out) TSstore = np.zeros((int(dummy), mc.mgrid.cells[0], 2)) thetastore = np.zeros( (int(dummy), mc.mgrid.vertgrid[0], mc.mgrid.latgrid[0])) print('starting new run...') #define bin assignment mode for infiltration particles mc.LTEdef = LTEdef #'instant'#'ks' #'instant' #'random' mc.LTEmemory = mc.soilgrid.ravel() * 0. #macropore reference mc.maccon = np.where( mc.macconnect.ravel() > 0)[0] #index of all connected cells mc.md_macdepth = np.abs(mc.md_macdepth) ############# # Run Model # ############# mc.LTEpercentile = 70 #new parameter t_end = mc.t_end infiltmeth = infilM exfiltmeth = exfilM film = True macscale = 1. #scale the macropore coating clogswitch = False infiltscale = False drained = pd.DataFrame(np.array([])) output = mc.t_out #mind to set also in TXstore.index definition dummy = np.floor(t_end / output) #loop through plot cycles for i in np.arange(dummy.astype(int))[ix:]: plotparticles_weier(particles, mc, pdyn, vG, runname, t, i, saving=True, relative=False, wdir=wdir) [particles, npart, thS, leftover, drained, t] = rE.CAOSpy_rundx1(i * output, (i + 1) * output, mc, pdyn, cinf, precTS, particles, leftover, drained, 6., splitfac=4, prec_2D=False, maccoat=macscale, saveDT=saveDT, clogswitch=clogswitch, infilt_method=infiltmeth, exfilt_method=exfiltmeth, film=film, infiltscale=infiltscale) TSstore[i, :, :] = rE.part_store(particles, mc) thetastore[i, :, :] = np.reshape( (mc.soilmatrix.loc[mc.soilgrid.ravel() - 1, 'tr'] + (mc.soilmatrix.ts - mc.soilmatrix.tr)[mc.soilgrid.ravel() - 1] * thS.ravel() * 0.01).values, np.shape(thS)) if largepick: with open(''.join([wdir, '/results/L', runname, '_Mstat.pick']), 'wb') as handle: pickle.dump(pickle.dumps( [pickle.dumps(particles), pickle.dumps([t, i])]), handle, protocol=2) else: with open(''.join([wdir, '/results/Z', runname, '_Mstat.pick']), 'wb') as handle: pickle.dump(pickle.dumps([ pickle.dumps(particles), pickle.dumps( [leftover, drained, t, TSstore, thetastore, npart, i]) ]), handle, protocol=2)