class FileWriterNode(WriterNode):
def __init__(self,path,nc_var,mode='r+',dtype=np.float64,chunksizes=None,ncparams = None):
self.fm = None #SplitFileManager.open_existing(path,pattern,nc_var)
self.path = path
self.nc_var = nc_var
self._file_mode = mode
self.dtype = dtype
self.schema = AnnualSplitSchema
self._init = True
self._open = True
self.ncparams = ncparams
self.chunksizes = chunksizes
self.dtype = dtype
def close(self):
self.fm.close_all()
def open_files(self): #period,extent):
if self._open:
# self.fm = self.file_manager_type.open_existing(self.path,self.nc_var+'.nc',self.nc_var,mode='r+')
# self.fm = SplitFileManager.open_existing(self.path,self.nc_var+'*.nc',self.nc_var,mode='r+')
self.fm = SplitFileManager.open_existing(self.path,self.nc_var+'.nc',self.nc_var,mode='r+')
self._open = False
def init_files(self,period,extent):
# if self._init:
v = mt.Variable(self.nc_var,'mm')
cs = mt.gen_coordset(period,extent)
mvar = mt.MappedVariable(v,cs,self.dtype) #in_dtype)
self.fm = SplitFileManager(self.path,mvar)
clobber = self._file_mode == 'w'
self.fm.create_files(self.schema,False,clobber,chunksize=self.chunksizes,ncparams=self.ncparams)#,chunksize=(1,16,16)) #None) #(64,32,32)) #(1,16,16))
self._open = True
def write_data(self,coords,data):
data = data.filled(-999.)
self.fm.set_by_coords(coords,data)
def sync_all(self):
for f in self.fm._open_datasets.values():
f.ncd_group.sync()
def process(self,coords,data):
self.open_files()
self.write_data(coords,data)
def get_dataspec(self):
dims = ['time','latitude','longitude']
dtype = self.fm.mapped_var.dtype
return DataSpec('array',dims,dtype)
def init_files(self,period,extent):
# if self._init:
v = mt.Variable(self.nc_var,'mm')
cs = mt.gen_coordset(period,extent)
mvar = mt.MappedVariable(v,cs,self.dtype) #in_dtype)
self.fm = SplitFileManager(self.path,mvar)
clobber = self._file_mode == 'w'
self.fm.create_files(self.schema,False,clobber,chunksize=self.chunksizes,ncparams=self.ncparams)#,chunksize=(1,16,16)) #None) #(64,32,32)) #(1,16,16))
self._open = True
def open_files(self): #period,extent):
if self._open:
self.fm = SplitFileManager.open_existing(self.path,
self.nc_var + '_*.nc',
self.nc_var,
mode='r+')
self._open = False
def __init__(self,path,pattern,nc_var,cache=False,map_func=None):
from awrams.utils.io.data_mapping import SplitFileManager
if map_func is not None:
map_func = funcspec_to_callable(map_func)
self.sfm = SplitFileManager.open_existing(path,pattern,nc_var,map_func=map_func)
self.cache = cache
self.data = None
def test_get_padded_by_coords():
from awrams.utils.io.data_mapping import SplitFileManager
from awrams.utils.mapping_types import gen_coordset
import awrams.utils.datetools as dt
path = os.path.join(os.path.dirname(__file__),'..','..','test_data','simulation')
sfm = SplitFileManager.open_existing(path,'temp_min_day_*.nc','temp_min_day')
# return sfm
extent = sfm.get_extent().ioffset[200:230,200:230]
period = dt.dates('2011')
coords = gen_coordset(period,extent)
data = sfm.get_padded_by_coords(coords)
assert data.shape == coords.shape
def get_initial_states_dict(imap,period,extent):
data_map = {}
data_paths = config_manager.get_system_profile('default').get_settings()['DATA_PATHS']
data_path = join(data_paths['BASE_DATA'],'test_data','simulation')
period = [period[0] - 1]
node_names = {'mleaf_dr': 'init_mleaf_hrudr',
'mleaf_sr': 'init_mleaf_hrusr',
's0_dr': 'init_s0_hrudr',
's0_sr': 'init_s0_hrusr',
'ss_dr': 'init_ss_hrudr',
'ss_sr': 'init_ss_hrusr',
'sd_dr': 'init_sd_hrudr',
'sd_sr': 'init_sd_hrusr',
'sg_bal': 'init_sg',
'sr_bal': 'init_sr'}
for k in 'mleaf_dr','s0_dr','sd_dr','sg_bal','ss_dr','mleaf_sr','s0_sr','sd_sr','sr_bal','ss_sr':
sfm = SplitFileManager.open_existing(data_path,k+'*nc',k)
data_map[node_names[k]] = sfm.get_data(period,extent)[0]
nodes.init_states_from_dict(imap,data_map,extent)
def test_get_padded_by_coords():
from awrams.utils.io.data_mapping import SplitFileManager
from awrams.utils.mapping_types import gen_coordset
import awrams.utils.datetools as dt
data_paths = config_manager.get_system_profile().get_settings(
)['DATA_PATHS']
path = os.path.join(data_paths['BASE_DATA'], 'test_data', 'simulation',
'climate', 'temp_min_day')
sfm = SplitFileManager.open_existing(path, 'temp_min_day_*.nc',
'temp_min_day')
# return sfm
extent = sfm.get_extent().ioffset[200:230, 200:230]
period = dt.dates('2011')
coords = gen_coordset(period, extent)
data = sfm.get_padded_by_coords(coords)
assert data.shape == coords.shape
def get_initial_states_dict(imap, period, extent):
from os.path import join, dirname
from awrams.utils.io.data_mapping import SplitFileManager
from awrams.utils.nodegraph import nodes
data_map = {}
data_path = join(dirname(__file__), '..', '..', 'test_data', 'simulation')
period = [period[0] - 1]
node_names = {
'mleaf_dr': 'init_mleaf_hrudr',
'mleaf_sr': 'init_mleaf_hrusr',
's0_dr': 'init_s0_hrudr',
's0_sr': 'init_s0_hrusr',
'ss_dr': 'init_ss_hrudr',
'ss_sr': 'init_ss_hrusr',
'sd_dr': 'init_sd_hrudr',
'sd_sr': 'init_sd_hrusr',
'sg_bal': 'init_sg',
'sr_bal': 'init_sr'
}
for k in 'mleaf_dr', 's0_dr', 'sd_dr', 'sg_bal', 'ss_dr', 'mleaf_sr', 's0_sr', 'sd_sr', 'sr_bal', 'ss_sr':
sfm = SplitFileManager.open_existing(data_path, k + '*nc', k)
data_map[node_names[k]] = sfm.get_data(period, extent)[0]
nodes.init_states_from_dict(imap, data_map, extent)
def __init__(self,
variable,
search_pattern,
period=None,
day_exist_chn_name=None):
self.sfm = SplitFileManager.open_existing('', search_pattern, variable)
def __init__(self, path, pattern, nc_var, cache=False):
from awrams.utils.io.data_mapping import SplitFileManager
self.sfm = SplitFileManager.open_existing(path, pattern, nc_var)
self.cache = cache
self.data = None
def open_files(self): #period,extent):
if self._open:
# self.fm = self.file_manager_type.open_existing(self.path,self.nc_var+'.nc',self.nc_var,mode='r+')
# self.fm = SplitFileManager.open_existing(self.path,self.nc_var+'*.nc',self.nc_var,mode='r+')
self.fm = SplitFileManager.open_existing(self.path,self.nc_var+'.nc',self.nc_var,mode='r+')
self._open = False
from awrams.utils import extents
import numpy as np
from awrams.utils.nodegraph import nodes, graph
from awrams.simulation.ondemand import OnDemandSimulator
full_extent = extents.get_default_extent()
period = dt.dates('dec 2010 - jan 2011')
extent = full_extent.ioffset[200:250, 200:250]
from awrams.models.awral import model
m = model.AWRALModel()
imap = m.get_default_mapping()
ods = OnDemandSimulator(m, imap)
print("running...")
res = ods.run(period, extent)
from awrams.utils.io.data_mapping import SplitFileManager
print("opening comparison results")
sfm = SplitFileManager.open_existing('./test_sim_outputs', 'qtot*', 'qtot')
qtot = sfm.get_data(period, extent)
max_diff = np.max(np.abs(res['qtot'] - qtot))
print(max_diff)
assert (max_diff < 1e-5)