def test_root_sliced(self): raw_data = self.raw_data # compute variables directly from the original data landpoint_slice = slice(0, 10) kwargs = { 'cable_out_path': self.cable_out_path, 'zarr_cache_path': self.zarr_cache_path, 'landpoint_slice': landpoint_slice } raw_data_sliced = raw_data[..., landpoint_slice, :] res_A = cacheWrapper(A_from_raw, **kwargs) self.assertTrue((res_A == 2 * raw_data_sliced).all().compute()) # compute variables directly from the original data # with batchsize specified kwargs = { 'cable_out_path': self.cable_out_path, 'zarr_cache_path': self.zarr_cache_path, 'batch_size': 32, 'landpoint_slice': landpoint_slice, } res_B = cacheWrapper(B_from_raw, **kwargs) self.assertTrue((res_B == 2 * raw_data_sliced).all().compute()) # check if a zarr dir has been created zarr_dirs = [f.name for f in self.zarr_cache_path.iterdir()] for name in ['A_from_raw', 'B_from_raw']: self.assertTrue(name in zarr_dirs)
def test_indirectly_dependent_rm(self): zarr_cache_path = self.zarr_cache_path cable_out_path = self.cable_out_path raw_data = self.raw_data kwargs = { 'cable_out_path': cable_out_path, 'zarr_cache_path': zarr_cache_path } # compute something that only res_D = cacheWrapper(D_from_cached_args, **kwargs) self.assertTrue((res_D == 20 * raw_data).all().compute()) # check the times the cache files have been touched # first show that they are untouched by the recent computation td_before = time_dict(zarr_cache_path) # now with rm flag kwargs.update({'rm': True}) res_D = cacheWrapper(D_from_cached_args, **kwargs) td_after_rm = time_dict(zarr_cache_path) # check that the newly created result is newer key = 'D_from_cached_args' self.assertTrue(td_before[key] < td_after_rm[key]) # check that the dependencies are untouched rest = frozenset.difference(frozenset(td_after_rm.keys()), frozenset([key])) for k in rest: self.assertEqual(td_before[k], td_after_rm[k])
def test_indirectly_dependent(self): zarr_cache_path = self.zarr_cache_path cable_out_path = self.cable_out_path raw_data = self.raw_data kwargs = { 'cable_out_path': cable_out_path, 'zarr_cache_path': zarr_cache_path } res_D = cacheWrapper(D_from_cached_args, **kwargs) self.assertTrue((res_D == 20 * raw_data).all().compute()) zarr_dirs = [f.name for f in zarr_cache_path.iterdir()] for name in [ 'A_from_raw', 'B_from_raw', 'C_from_cached_args', 'D_from_cached_args' ]: self.assertTrue(name in zarr_dirs) # check the times the cache files have been touched # first show that they are untouched by the recent computation td_before = time_dict(zarr_cache_path) res_D = cacheWrapper(D_from_cached_args, **kwargs) td_after = time_dict(zarr_cache_path) self.assertEqual(td_before, td_after)
def test_indirectly_dependent_rec_rm(self): zarr_cache_path = self.zarr_cache_path cable_out_path = self.cable_out_path raw_data = self.raw_data kwargs = { 'cable_out_path': cable_out_path, 'zarr_cache_path': zarr_cache_path } # compute something that only res_D = cacheWrapper(D_from_cached_args, **kwargs) self.assertTrue((res_D == 20 * raw_data).all().compute()) # check the times the cache files have been touched # first show that they are untouched by the recent computation td_before = time_dict(zarr_cache_path) # now with rec_rm flag kwargs.update({'rec_rm': True}) res_D = cacheWrapper(D_from_cached_args, **kwargs) # check that all dependencies have been recreated td_after_rec_rm = time_dict(zarr_cache_path) self.assertTrue( all([ td_before[key] < td_after_rec_rm[key] for key in td_before.keys() ]))
def test_root(self): # compute variables directly from the original data funcs = [ cC.time, cC.iveg, cC.Clitter, cC.Cplant, cC.Csoil, cC.NPP, cC.fracCalloc, cC.fromCWDtoS, cC.fromLeaftoL, cC.fromMettoS, cC.fromRoottoL, cC.fromSOMtoSOM, cC.fromStrtoS, cC.fromWoodtoL, cC.kplant, cC.xkNlimiting, cC.xktemp, cC.xkwater, ] results = map(lambda func: cacheWrapper(func, **self.kwargs), funcs) [r for r in results]
def test_indirectly_dependent_sol_val(self): func = cC.sol_val res_sol_val = cacheWrapper(func, **self.kwargs) #self.assertTrue((res_D == 20 * raw_data).all().compute()) self.check_presence([ 'NPP', 'fracCalloc', 'u_org', 'u_val', 'Clitter0', 'Cplant0', 'Csoil0', 'x0_org', 'nz', 'x0_val', "iveg", # for finding valid patch landpoint combies "Cplant", # for finding points with vegetation "kplant", "fromLeaftoL", "fromRoottoL", "fromWoodtoL", "fromMettoS", "fromStrtoS", "fromCWDtoS", "fromSOMtoSOM", "xktemp", "xkwater", "xkNlimiting" ])
def test_indirectly_dependent_u_val(self): func = cC.u_val res_u = cacheWrapper(func, **self.kwargs) #self.assertTrue((res_D == 20 * raw_data).all().compute()) self.check_presence( ['iveg', 'NPP', 'fracCalloc', 'u_org', 'nz', 'u_val'])
def test_indirectly_dependent_B_org(self): res_B = cacheWrapper(cC.B_org, **self.kwargs) self.check_presence([ "kplant", "fromLeaftoL", "fromRoottoL", "fromWoodtoL", "fromMettoS", "fromStrtoS", "fromCWDtoS", "fromSOMtoSOM", "xktemp", "xkwater", "xkNlimiting" ])
def test_indirectly_dependent_x0_val(self): func = cC.x0_val res_x = cacheWrapper(func, **self.kwargs) self.check_presence([ 'Clitter0', 'Cplant0', 'Csoil0', 'x0_org', 'nz', 'x0_val', ])
def test_root(self): zarr_cache_path = self.zarr_cache_path cable_out_path = self.cable_out_path raw_data = self.raw_data # compute variables directly from the original data kwargs = { 'cable_out_path': cable_out_path, 'zarr_cache_path': zarr_cache_path, 'batch_size': 32 } res_A = cacheWrapper(A_from_raw, **kwargs) self.assertTrue((res_A == 2 * raw_data).all().compute()) # compute variables directly from the original data # with batchsize specified res_B = cacheWrapper(B_from_raw, **kwargs) self.assertTrue((res_B == 2 * raw_data).all().compute()) # check if a zarr dir has been created zarr_dirs = [f.name for f in self.zarr_cache_path.iterdir()] for name in ['A_from_raw', 'B_from_raw']: self.assertTrue(name in zarr_dirs)
def test_root_rm(self): zarr_cache_path = self.zarr_cache_path cable_out_path = self.cable_out_path raw_data = self.raw_data kwargs = { 'cable_out_path': cable_out_path, 'zarr_cache_path': zarr_cache_path, 'batch_size': 32 } # compute variables directly from the original data res_B = cacheWrapper(B_from_raw, **kwargs) # check the times the cache files have been touched # first to make sure that they are not touched td_before = time_dict(zarr_cache_path) #now with rm flag kwargs.update({'rm': True}) res_B = cacheWrapper(B_from_raw, **kwargs) td_after_rm = time_dict(zarr_cache_path) key = 'B_from_raw' self.assertTrue(td_before[key] < td_after_rm[key])
def test_root_rec_rm(self): zarr_cache_path = self.zarr_cache_path cable_out_path = self.cable_out_path raw_data = self.raw_data kwargs = { 'cable_out_path': cable_out_path, 'zarr_cache_path': zarr_cache_path } # compute variables directly from the original data res_B = cacheWrapper(B_from_raw, **kwargs) # check the times the cache files have been touched td_before = time_dict(zarr_cache_path) # now with rec_rm flag which in this case just implies rm # but has no other impact since there is no recursion kwargs.update({'rec_rm': True}) res_B = cacheWrapper(B_from_raw, **kwargs) td_after_rm = time_dict(zarr_cache_path) key = 'B_from_raw' # for directly computable vars rec_rm has the same # effect as rm and only affects B self.assertTrue(td_before[key] < td_after_rm[key])
def test_indirectly_dependent_B_val(self): func = cC.B_val res_B = cacheWrapper(func, **self.kwargs) self.check_presence([ "iveg", # for finding valid patch landpoint combies "Cplant", # for finding points with vegetation "kplant", "fromLeaftoL", "fromRoottoL", "fromWoodtoL", "fromMettoS", "fromStrtoS", "fromCWDtoS", "fromSOMtoSOM", "xktemp", "xkwater", "xkNlimiting" ])
landpoint_slice=landpoint_slice, ) if "cable_data_set" not in dir(): cable_data_set = cH.cable_ds(cable_out_path) args = { "cable_data_set": cable_data_set, "zarr_cache_path": zarr_cache_path, "landpoint_slice": landpoint_slice, "time_slice": time_slice, #'batch_size': 128, "batch_size": 12, #'rm': True } #iveg_mask = cH.cacheWrapper(cC.iveg_mask, **args) #x_org= cH.cacheWrapper(cC.x_org, **args) x_org_iveg = cH.cacheWrapper(cC.x_org_iveg, **args) sol_org_iveg = cH.cacheWrapper(cC.sol_org_iveg, **args) time = cH.cacheWrapper(cC.time, **args) patches, landpoints = cC.all_pools_vary_cond_nz(**args) pcs = patches.compute() lpcs = landpoints.compute() pcs, lpcs p = Path('plots') p.mkdir(exist_ok=True) ind = 0 # first pair that has a nonconstant solution for lp = lpcs[ind] patch = lpcs[ind] print(x_org_iveg[0, :, patch, lp]) n = 9 fig = plt.figure(figsize=(15, 25)) my_time_sl = slice(0, 2000)
time_slice=time_slice, landpoint_slice=landpoint_slice, ) if "cable_data_set" not in dir(): cable_data_set = cH.cable_ds(cable_out_path) args = { "cable_data_set": cable_data_set, "zarr_cache_path": zarr_cache_path, "landpoint_slice": landpoint_slice, "time_slice": time_slice, #'batch_size': 128, "batch_size": 8, #'rm': True } x_org_iveg = cH.cacheWrapper(cC.x_org_iveg, **args) times = cH.cacheWrapper(cC.time, **args) patches, landpoints = cC.all_pools_vary_cond_nz(**args) #lets find the first combinations that fulfill our requirements my_sl= slice(0,96) pcs = patches[my_sl].compute() lpcs = landpoints[my_sl].compute() pcs,lpcs # p = Path('plots') # p.mkdir(exist_ok=True) # for lp in range(landpoint_slice.start, landpoint_slice.stop): # for patch in range(10): # print(x_org_iveg[0,:,patch,lp]) # fig=plt.figure() # for pool in range(n): # ax = fig.add_subplot(n+1, 1, 2+pool)
def test_root0(self): funcs = [cC.Clitter0, cC.Cplant0, cC.Csoil0, cC.x0_org] results = map(lambda func: cacheWrapper(func, **self.kwargs), funcs) [r for r in results]
def D_from_cached_args(**kwargs): C = cacheWrapper(C_from_cached_args, **kwargs) return 5 * C
def C_from_cached_args(**kwargs): # example for a dependent function A = cacheWrapper(A_from_raw, **kwargs) B = cacheWrapper(B_from_raw, **kwargs) return A + B
def test_numeric_input_tuple(self): # setup the paths to the testdata cable_out_path = Path( '/home/data/cable-data/example_runs/parallel_1901_2004_with_spinup/output/new4' ) # cable_data_set = cH.cable_ds(cable_out_path) time_slice = slice(0, None) landpoint_slice = slice(1590, 1637) # cheated # landpoint_slice = slice(None,None) # time_slice=slice(None,None,None) zarr_cache_path = cP.slice_dir_path( cable_out_path, sub_dir_trunk="zarr_mm11", time_slice=time_slice, landpoint_slice=landpoint_slice, ) if "cable_data_set" not in dir(): cable_data_set = cH.cable_ds(cable_out_path) args = { "cable_data_set": cable_data_set, "zarr_cache_path": zarr_cache_path, "landpoint_slice": landpoint_slice, "time_slice": time_slice, #'batch_size': 128, "batch_size": 12, #'rm': True } x_org_iveg = cH.cacheWrapper(cC.x_org_iveg, **args) time = cH.cacheWrapper(cC.time, **args) patches, landpoints = cC.all_pools_vary_cond_nz(**args) pcs = patches.compute() lpcs = landpoints.compute() pcs, lpcs p = Path('plots') p.mkdir(exist_ok=True) ind = 0 # first pair that has a nonconstant solution for lp = lpcs[ind] patch = lpcs[ind] # get the symbolic represantation from the database mvs = self.mvs # define some stuff to extend it with def default(t): return 1 leaf = Symbol('leaf') fine_root = Symbol('fine_root') Npp = Function("Npp") bvec_leaf = Function("bvec_leaf") bvec_fine_root = Function("bvec_fine_root") xk_leaf_cold = Function("xk_leaf_cold") xk_leaf_dry = Function("xk_leaf_dry") kleaf = Function("kleaf") kfroot = Function("kfroot") # bvec_wood = Function("bvec_wood") np1 = NumericParameterization( par_dict={}, func_dict=frozendict({ Npp: default, bvec_fine_root: default, bvec_leaf: default, xk_leaf_cold: default, kleaf: default, kfroot: default, xk_leaf_dry: default, }), ) nsv1 = NumericStartValueDict({leaf: 0.3, fine_root: 3.96}) ntimes1 = NumericSimulationTimes(np.linspace(0, 1, 11)) # extend the symbolice version with the new stuff pvs = mvs.provided_mvar_values #from IPython import embed; embed() pvs1 = pvs.union(frozenset({np1, nsv1, ntimes1})) mvs1 = MVarSet(pvs1) x = mvs1.get_StateVariableTuple() Input = mvs1.get_InputTuple() #B = mvs1.get_CompartmentalMatrix() sym_times = mvs1.get_NumericSimulationTimes() sol_smooth = mvs1.get_NumericSolutionArray() comp_slice = slice(0, 100) n = sol_smooth.shape[1] fig = plt.figure() for pool in range(n): ax = fig.add_subplot(n + 1, 1, 2 + pool) title = "\$" + latex(x[pool]) + "\$" #ax.plot( # sym_times[comp_slice], # sol_smooth[comp_slice, pool], # color='r' #) ax.plot(time[comp_slice], x_org_iveg[comp_slice, pool, patch, lp], color='b') fontsize = 10 ax.set_title(title, fontsize=fontsize) fig.savefig('solution.pdf')
def test_indirectly_dependent_x0_org(self): func = cC.x0_org res_x = cacheWrapper(func, **self.kwargs) zarr_dirs = [f.name for f in zarr_cache_path.iterdir()] self.check_presence(['Clitter0', 'Cplant0', 'Csoil0' 'x0_org'])
# sub_dir_trunk='zarr_mm', time_slice=time_slice, landpoint_slice=landpoint_slice, ) if "cable_data_set" not in dir(): cable_data_set = cH.cable_ds(cable_out_path) args = { "cable_data_set": cable_data_set, "zarr_cache_path": zarr_cache_path, "landpoint_slice": landpoint_slice, "time_slice": time_slice, "batch_size": 64, #'batch_size': 8, #'rm': True } sol_org_iveg = cH.cacheWrapper(cC.sol_org_iveg, **args) sol_org_iveg # x_val = cH.cacheWrapper( # cC.x_val, # **args # ) # # check that the solution is non negative # assert(((x_val>=0).all()).compute()) # # # now check that the solution from the reconstructed Bs is non negative # sol_val = cH.cacheWrapper( # cC.sol_val, # **args # )
def test_indirectly_dependent_x_org(self): func = cC.x_org res_x = cacheWrapper(func, **self.kwargs) self.check_presence(['Clitter', 'Cplant', 'Csoil', 'x_org'])