def test_root_sliced(self):
raw_data = self.raw_data
# compute variables directly from the original data
landpoint_slice = slice(0, 10)
kwargs = {
'cable_out_path': self.cable_out_path,
'zarr_cache_path': self.zarr_cache_path,
'landpoint_slice': landpoint_slice
}
raw_data_sliced = raw_data[..., landpoint_slice, :]
res_A = cacheWrapper(A_from_raw, **kwargs)
self.assertTrue((res_A == 2 * raw_data_sliced).all().compute())
# compute variables directly from the original data
# with batchsize specified
kwargs = {
'cable_out_path': self.cable_out_path,
'zarr_cache_path': self.zarr_cache_path,
'batch_size': 32,
'landpoint_slice': landpoint_slice,
}
res_B = cacheWrapper(B_from_raw, **kwargs)
self.assertTrue((res_B == 2 * raw_data_sliced).all().compute())
# check if a zarr dir has been created
zarr_dirs = [f.name for f in self.zarr_cache_path.iterdir()]
for name in ['A_from_raw', 'B_from_raw']:
self.assertTrue(name in zarr_dirs)
def test_indirectly_dependent_rm(self):
zarr_cache_path = self.zarr_cache_path
cable_out_path = self.cable_out_path
raw_data = self.raw_data
kwargs = {
'cable_out_path': cable_out_path,
'zarr_cache_path': zarr_cache_path
}
# compute something that only
res_D = cacheWrapper(D_from_cached_args, **kwargs)
self.assertTrue((res_D == 20 * raw_data).all().compute())
# check the times the cache files have been touched
# first show that they are untouched by the recent computation
td_before = time_dict(zarr_cache_path)
# now with rm flag
kwargs.update({'rm': True})
res_D = cacheWrapper(D_from_cached_args, **kwargs)
td_after_rm = time_dict(zarr_cache_path)
# check that the newly created result is newer
key = 'D_from_cached_args'
self.assertTrue(td_before[key] < td_after_rm[key])
# check that the dependencies are untouched
rest = frozenset.difference(frozenset(td_after_rm.keys()),
frozenset([key]))
for k in rest:
self.assertEqual(td_before[k], td_after_rm[k])
def test_indirectly_dependent(self):
zarr_cache_path = self.zarr_cache_path
cable_out_path = self.cable_out_path
raw_data = self.raw_data
kwargs = {
'cable_out_path': cable_out_path,
'zarr_cache_path': zarr_cache_path
}
res_D = cacheWrapper(D_from_cached_args, **kwargs)
self.assertTrue((res_D == 20 * raw_data).all().compute())
zarr_dirs = [f.name for f in zarr_cache_path.iterdir()]
for name in [
'A_from_raw', 'B_from_raw', 'C_from_cached_args',
'D_from_cached_args'
]:
self.assertTrue(name in zarr_dirs)
# check the times the cache files have been touched
# first show that they are untouched by the recent computation
td_before = time_dict(zarr_cache_path)
res_D = cacheWrapper(D_from_cached_args, **kwargs)
td_after = time_dict(zarr_cache_path)
self.assertEqual(td_before, td_after)
def test_indirectly_dependent_rec_rm(self):
zarr_cache_path = self.zarr_cache_path
cable_out_path = self.cable_out_path
raw_data = self.raw_data
kwargs = {
'cable_out_path': cable_out_path,
'zarr_cache_path': zarr_cache_path
}
# compute something that only
res_D = cacheWrapper(D_from_cached_args, **kwargs)
self.assertTrue((res_D == 20 * raw_data).all().compute())
# check the times the cache files have been touched
# first show that they are untouched by the recent computation
td_before = time_dict(zarr_cache_path)
# now with rec_rm flag
kwargs.update({'rec_rm': True})
res_D = cacheWrapper(D_from_cached_args, **kwargs)
# check that all dependencies have been recreated
td_after_rec_rm = time_dict(zarr_cache_path)
self.assertTrue(
all([
td_before[key] < td_after_rec_rm[key]
for key in td_before.keys()
]))
def test_root(self):
# compute variables directly from the original data
funcs = [
cC.time,
cC.iveg,
cC.Clitter,
cC.Cplant,
cC.Csoil,
cC.NPP,
cC.fracCalloc,
cC.fromCWDtoS,
cC.fromLeaftoL,
cC.fromMettoS,
cC.fromRoottoL,
cC.fromSOMtoSOM,
cC.fromStrtoS,
cC.fromWoodtoL,
cC.kplant,
cC.xkNlimiting,
cC.xktemp,
cC.xkwater,
]
results = map(lambda func: cacheWrapper(func, **self.kwargs), funcs)
[r for r in results]
def test_indirectly_dependent_sol_val(self):
func = cC.sol_val
res_sol_val = cacheWrapper(func, **self.kwargs)
#self.assertTrue((res_D == 20 * raw_data).all().compute())
self.check_presence([
'NPP',
'fracCalloc',
'u_org',
'u_val',
'Clitter0',
'Cplant0',
'Csoil0',
'x0_org',
'nz',
'x0_val',
"iveg", # for finding valid patch landpoint combies
"Cplant", # for finding points with vegetation
"kplant",
"fromLeaftoL",
"fromRoottoL",
"fromWoodtoL",
"fromMettoS",
"fromStrtoS",
"fromCWDtoS",
"fromSOMtoSOM",
"xktemp",
"xkwater",
"xkNlimiting"
])
def test_indirectly_dependent_u_val(self):
func = cC.u_val
res_u = cacheWrapper(func, **self.kwargs)
#self.assertTrue((res_D == 20 * raw_data).all().compute())
self.check_presence(
['iveg', 'NPP', 'fracCalloc', 'u_org', 'nz', 'u_val'])
def test_indirectly_dependent_B_org(self):
res_B = cacheWrapper(cC.B_org, **self.kwargs)
self.check_presence([
"kplant", "fromLeaftoL", "fromRoottoL", "fromWoodtoL",
"fromMettoS", "fromStrtoS", "fromCWDtoS", "fromSOMtoSOM", "xktemp",
"xkwater", "xkNlimiting"
])
def test_indirectly_dependent_x0_val(self):
func = cC.x0_val
res_x = cacheWrapper(func, **self.kwargs)
self.check_presence([
'Clitter0',
'Cplant0',
'Csoil0',
'x0_org',
'nz',
'x0_val',
])
def test_root(self):
zarr_cache_path = self.zarr_cache_path
cable_out_path = self.cable_out_path
raw_data = self.raw_data
# compute variables directly from the original data
kwargs = {
'cable_out_path': cable_out_path,
'zarr_cache_path': zarr_cache_path,
'batch_size': 32
}
res_A = cacheWrapper(A_from_raw, **kwargs)
self.assertTrue((res_A == 2 * raw_data).all().compute())
# compute variables directly from the original data
# with batchsize specified
res_B = cacheWrapper(B_from_raw, **kwargs)
self.assertTrue((res_B == 2 * raw_data).all().compute())
# check if a zarr dir has been created
zarr_dirs = [f.name for f in self.zarr_cache_path.iterdir()]
for name in ['A_from_raw', 'B_from_raw']:
self.assertTrue(name in zarr_dirs)
def test_root_rm(self):
zarr_cache_path = self.zarr_cache_path
cable_out_path = self.cable_out_path
raw_data = self.raw_data
kwargs = {
'cable_out_path': cable_out_path,
'zarr_cache_path': zarr_cache_path,
'batch_size': 32
}
# compute variables directly from the original data
res_B = cacheWrapper(B_from_raw, **kwargs)
# check the times the cache files have been touched
# first to make sure that they are not touched
td_before = time_dict(zarr_cache_path)
#now with rm flag
kwargs.update({'rm': True})
res_B = cacheWrapper(B_from_raw, **kwargs)
td_after_rm = time_dict(zarr_cache_path)
key = 'B_from_raw'
self.assertTrue(td_before[key] < td_after_rm[key])
def test_root_rec_rm(self):
zarr_cache_path = self.zarr_cache_path
cable_out_path = self.cable_out_path
raw_data = self.raw_data
kwargs = {
'cable_out_path': cable_out_path,
'zarr_cache_path': zarr_cache_path
}
# compute variables directly from the original data
res_B = cacheWrapper(B_from_raw, **kwargs)
# check the times the cache files have been touched
td_before = time_dict(zarr_cache_path)
# now with rec_rm flag which in this case just implies rm
# but has no other impact since there is no recursion
kwargs.update({'rec_rm': True})
res_B = cacheWrapper(B_from_raw, **kwargs)
td_after_rm = time_dict(zarr_cache_path)
key = 'B_from_raw'
# for directly computable vars rec_rm has the same
# effect as rm and only affects B
self.assertTrue(td_before[key] < td_after_rm[key])
def test_indirectly_dependent_B_val(self):
func = cC.B_val
res_B = cacheWrapper(func, **self.kwargs)
self.check_presence([
"iveg", # for finding valid patch landpoint combies
"Cplant", # for finding points with vegetation
"kplant",
"fromLeaftoL",
"fromRoottoL",
"fromWoodtoL",
"fromMettoS",
"fromStrtoS",
"fromCWDtoS",
"fromSOMtoSOM",
"xktemp",
"xkwater",
"xkNlimiting"
])
landpoint_slice=landpoint_slice,
)
if "cable_data_set" not in dir():
cable_data_set = cH.cable_ds(cable_out_path)
args = {
"cable_data_set": cable_data_set,
"zarr_cache_path": zarr_cache_path,
"landpoint_slice": landpoint_slice,
"time_slice": time_slice,
#'batch_size': 128,
"batch_size": 12,
#'rm': True
}
#iveg_mask = cH.cacheWrapper(cC.iveg_mask, **args)
#x_org= cH.cacheWrapper(cC.x_org, **args)
x_org_iveg = cH.cacheWrapper(cC.x_org_iveg, **args)
sol_org_iveg = cH.cacheWrapper(cC.sol_org_iveg, **args)
time = cH.cacheWrapper(cC.time, **args)
patches, landpoints = cC.all_pools_vary_cond_nz(**args)
pcs = patches.compute()
lpcs = landpoints.compute()
pcs, lpcs
p = Path('plots')
p.mkdir(exist_ok=True)
ind = 0 # first pair that has a nonconstant solution for
lp = lpcs[ind]
patch = lpcs[ind]
print(x_org_iveg[0, :, patch, lp])
n = 9
fig = plt.figure(figsize=(15, 25))
my_time_sl = slice(0, 2000)
time_slice=time_slice,
landpoint_slice=landpoint_slice,
)
if "cable_data_set" not in dir():
cable_data_set = cH.cable_ds(cable_out_path)
args = {
"cable_data_set": cable_data_set,
"zarr_cache_path": zarr_cache_path,
"landpoint_slice": landpoint_slice,
"time_slice": time_slice,
#'batch_size': 128,
"batch_size": 8,
#'rm': True
}
x_org_iveg = cH.cacheWrapper(cC.x_org_iveg, **args)
times = cH.cacheWrapper(cC.time, **args)
patches, landpoints = cC.all_pools_vary_cond_nz(**args)
#lets find the first combinations that fulfill our requirements
my_sl= slice(0,96)
pcs = patches[my_sl].compute()
lpcs = landpoints[my_sl].compute()
pcs,lpcs
# p = Path('plots')
# p.mkdir(exist_ok=True)
# for lp in range(landpoint_slice.start, landpoint_slice.stop):
# for patch in range(10):
# print(x_org_iveg[0,:,patch,lp])
# fig=plt.figure()
# for pool in range(n):
# ax = fig.add_subplot(n+1, 1, 2+pool)
def test_root0(self):
funcs = [cC.Clitter0, cC.Cplant0, cC.Csoil0, cC.x0_org]
results = map(lambda func: cacheWrapper(func, **self.kwargs), funcs)
[r for r in results]
def D_from_cached_args(**kwargs):
C = cacheWrapper(C_from_cached_args, **kwargs)
return 5 * C
def C_from_cached_args(**kwargs):
# example for a dependent function
A = cacheWrapper(A_from_raw, **kwargs)
B = cacheWrapper(B_from_raw, **kwargs)
return A + B
def test_numeric_input_tuple(self):
# setup the paths to the testdata
cable_out_path = Path(
'/home/data/cable-data/example_runs/parallel_1901_2004_with_spinup/output/new4'
)
# cable_data_set = cH.cable_ds(cable_out_path)
time_slice = slice(0, None)
landpoint_slice = slice(1590, 1637) # cheated
# landpoint_slice = slice(None,None)
# time_slice=slice(None,None,None)
zarr_cache_path = cP.slice_dir_path(
cable_out_path,
sub_dir_trunk="zarr_mm11",
time_slice=time_slice,
landpoint_slice=landpoint_slice,
)
if "cable_data_set" not in dir():
cable_data_set = cH.cable_ds(cable_out_path)
args = {
"cable_data_set": cable_data_set,
"zarr_cache_path": zarr_cache_path,
"landpoint_slice": landpoint_slice,
"time_slice": time_slice,
#'batch_size': 128,
"batch_size": 12,
#'rm': True
}
x_org_iveg = cH.cacheWrapper(cC.x_org_iveg, **args)
time = cH.cacheWrapper(cC.time, **args)
patches, landpoints = cC.all_pools_vary_cond_nz(**args)
pcs = patches.compute()
lpcs = landpoints.compute()
pcs, lpcs
p = Path('plots')
p.mkdir(exist_ok=True)
ind = 0 # first pair that has a nonconstant solution for
lp = lpcs[ind]
patch = lpcs[ind]
# get the symbolic represantation from the database
mvs = self.mvs
# define some stuff to extend it with
def default(t):
return 1
leaf = Symbol('leaf')
fine_root = Symbol('fine_root')
Npp = Function("Npp")
bvec_leaf = Function("bvec_leaf")
bvec_fine_root = Function("bvec_fine_root")
xk_leaf_cold = Function("xk_leaf_cold")
xk_leaf_dry = Function("xk_leaf_dry")
kleaf = Function("kleaf")
kfroot = Function("kfroot")
# bvec_wood = Function("bvec_wood")
np1 = NumericParameterization(
par_dict={},
func_dict=frozendict({
Npp: default,
bvec_fine_root: default,
bvec_leaf: default,
xk_leaf_cold: default,
kleaf: default,
kfroot: default,
xk_leaf_dry: default,
}),
)
nsv1 = NumericStartValueDict({leaf: 0.3, fine_root: 3.96})
ntimes1 = NumericSimulationTimes(np.linspace(0, 1, 11))
# extend the symbolice version with the new stuff
pvs = mvs.provided_mvar_values
#from IPython import embed; embed()
pvs1 = pvs.union(frozenset({np1, nsv1, ntimes1}))
mvs1 = MVarSet(pvs1)
x = mvs1.get_StateVariableTuple()
Input = mvs1.get_InputTuple()
#B = mvs1.get_CompartmentalMatrix()
sym_times = mvs1.get_NumericSimulationTimes()
sol_smooth = mvs1.get_NumericSolutionArray()
comp_slice = slice(0, 100)
n = sol_smooth.shape[1]
fig = plt.figure()
for pool in range(n):
ax = fig.add_subplot(n + 1, 1, 2 + pool)
title = "\$" + latex(x[pool]) + "\$"
#ax.plot(
# sym_times[comp_slice],
# sol_smooth[comp_slice, pool],
# color='r'
#)
ax.plot(time[comp_slice],
x_org_iveg[comp_slice, pool, patch, lp],
color='b')
fontsize = 10
ax.set_title(title, fontsize=fontsize)
fig.savefig('solution.pdf')
def test_indirectly_dependent_x0_org(self):
func = cC.x0_org
res_x = cacheWrapper(func, **self.kwargs)
zarr_dirs = [f.name for f in zarr_cache_path.iterdir()]
self.check_presence(['Clitter0', 'Cplant0', 'Csoil0' 'x0_org'])
# sub_dir_trunk='zarr_mm',
time_slice=time_slice,
landpoint_slice=landpoint_slice,
)
if "cable_data_set" not in dir():
cable_data_set = cH.cable_ds(cable_out_path)
args = {
"cable_data_set": cable_data_set,
"zarr_cache_path": zarr_cache_path,
"landpoint_slice": landpoint_slice,
"time_slice": time_slice,
"batch_size": 64,
#'batch_size': 8,
#'rm': True
}
sol_org_iveg = cH.cacheWrapper(cC.sol_org_iveg, **args)
sol_org_iveg
# x_val = cH.cacheWrapper(
# cC.x_val,
# **args
# )
# # check that the solution is non negative
# assert(((x_val>=0).all()).compute())
#
# # now check that the solution from the reconstructed Bs is non negative
# sol_val = cH.cacheWrapper(
# cC.sol_val,
# **args
# )
def test_indirectly_dependent_x_org(self):
func = cC.x_org
res_x = cacheWrapper(func, **self.kwargs)
self.check_presence(['Clitter', 'Cplant', 'Csoil', 'x_org'])
