def schism(tmpdir, kwargs):
grid = pgrid.grid(type='tri2d',
grid_file=(DATA_DIR / 'hgrid.gr3').as_posix())
#update dem
xp = grid.Dataset.SCHISM_hgrid_node_x.values
yp = grid.Dataset.SCHISM_hgrid_node_y.values
kwargs.update({'grid_x': xp, 'grid_y': yp})
#get dem
df = pdem.dem(**kwargs)
grid.Dataset['depth'].loc[:] = -df.Dataset.ival.values
filename_ = str(tmpdir.join('hgrid_.gr3'))
#output to grid file
grid.to_file(filename_)
#read again new grid
grid_ = pgrid.grid(type='tri2d', grid_file=filename_)
#compare
return grid.Dataset.equals(grid_.Dataset)
def test_answer(tmpdir):
rpath = str(tmpdir) + "/"
df = pg.grid(type="tri2d", grid_file=GRID_FILE)
df.to_file(rpath + "test.gr3")
dh = pg.grid(type="tri2d", grid_file=rpath + "test.gr3")
assert df.Dataset.equals(dh.Dataset)
def test_answer(tmpdir):
rpath = str(tmpdir) + '/'
df = pg.grid(type='tri2d', grid_file=GRID_FILE)
df.to_file(rpath + 'test.gr3')
dh = pg.grid(type='tri2d', grid_file=rpath + 'test.gr3')
assert df.Dataset.equals(dh.Dataset)
def create(self, **kwargs):
if not kwargs: kwargs = self.__dict__.copy()
# Grid
self.grid = pgrid.grid(type='r2d', **kwargs)
# set lat/lon from file
if hasattr(self, 'grid_file'):
kwargs.update({'lon_min': self.grid.Dataset.x.values.min()})
kwargs.update({'lon_max': self.grid.Dataset.x.values.max()})
kwargs.update({'lat_min': self.grid.Dataset.y.values.min()})
kwargs.update({'lat_max': self.grid.Dataset.y.values.max()})
nj, ni = self.grid.Dataset.lons.shape
self.nj, self.ni = nj, ni
kwargs.update({'ni': ni, 'nj': nj})
# get bathymetry
self.bath(**kwargs)
# get boundaries
self.bc()
#get meteo
if self.atm: self.force(**kwargs)
#get tide
if self.tide: self.tidebc()
self.config(**kwargs)
def read_folder(self, rfolder, **kwargs):
gfile = glob.glob(rfolder + '/*.grd') # Grid
dfile = glob.glob(rfolder + '/*.dep') # bathymetry
u = glob.glob(rfolder + '/*.amu') # meteo
v = glob.glob(rfolder + '/*.amv')
p = glob.glob(rfolder + '/*.amp')
#config
self.mdf = pd.read_csv(d[0], sep='=')
self.mdf = self.mdf.set_index(self.mdf.columns[0]) # set index
#grid
self.grid = pgrid.grid('r2d', grid_file=gfile[0])
#bath
self.dem.Dataset = d3d.from_dep(dfile[0])
#meteo
mf = []
mf.append(d3d.from_force(u[0], 'u10'))
mf.append(d3d.from_force(v[0], 'v10'))
mf.append(d3d.from_force(p[0], 'msl'))
self.meteo.Dataset = xr.merge(mf)
#---------------------------------------------------------------------
logger.exception('folder incomplete. Abort\n')
sys.exit(1)
def test_d3d_grid(tmpdir, natural_earth, dem_source, window):
grid = pg.grid(type="r2d", geometry=window, resolution=0.1, rpath=str(tmpdir) + "/")
gr = grid.Dataset
xp, yp = gr.lons, gr.lats
df = pdem.dem(**window, grid_x=xp, grid_y=yp, dem_source=dem_source)
df.adjust(natural_earth)
assert np.isnan(df.Dataset.fval.values).sum() == 0
def test_schism_grid(tmpdir, natural_earth, dem_source, window):
grid = pg.grid(type="tri2d", geometry=window, coastlines=natural_earth, rpath=str(tmpdir) + "/")
xg = grid.Dataset.SCHISM_hgrid_node_x.values
yg = grid.Dataset.SCHISM_hgrid_node_y.values
df = pdem.dem(**window, grid_x=xg, grid_y=yg, dem_source=dem_source) # get dem
df.adjust(natural_earth)
assert np.isnan(df.Dataset.fval.values).sum() == 0
def test_answer(tmpdir, coast):
df = pg.grid(type="tri2d", geometry="global", coastlines=coast, rpath=str(tmpdir) + "/")
check = np.isnan(df.Dataset.depth.values).sum() == 0
assert check == True
def read_folder(self, rfolder, **kwargs):
gfile = glob.glob(rfolder + "/*.grd") # Grid
dfile = glob.glob(rfolder + "/*.dep") # bathymetry
u = glob.glob(rfolder + "/*.amu") # meteo
v = glob.glob(rfolder + "/*.amv")
p = glob.glob(rfolder + "/*.amp")
# config
self.mdf = pd.read_csv(d[0], sep="=")
self.mdf = self.mdf.set_index(self.mdf.columns[0]) # set index
# grid
self.grid = pgrid.grid("r2d", grid_file=gfile[0])
# bath
self.dem.Dataset = d3d.from_dep(dfile[0])
# meteo
mf = []
mf.append(d3d.from_force(u[0], "u10"))
mf.append(d3d.from_force(v[0], "v10"))
mf.append(d3d.from_force(p[0], "msl"))
self.meteo.Dataset = xr.merge(mf)
# ---------------------------------------------------------------------
logger.exception("folder incomplete. Abort\n")
sys.exit(1)
def func(tmpdir, name):
filename = DATA_DIR / name
# read grid file
grid = pgrid.grid(type="tri2d", grid_file=filename)
filename_ = str(tmpdir.join("hgrid_.gr3"))
# output to grid file
grid.to_file(filename_)
# read again new grid
grid_ = pgrid.grid(type="tri2d", grid_file=filename_)
# cleanup
os.remove(filename_)
# compare
return grid.Dataset.equals(grid_.Dataset)
def test_answer(tmpdir, window, coast):
df = pg.grid(type='tri2d',
geometry=window,
coastlines=coast,
rpath=str(tmpdir) + '/')
check = np.isnan(df.Dataset.depth.values).sum() == 0
assert check == True
def test_d3d_grid(tmpdir, dic):
grid = pg.grid(type="r2d",
geometry=dic,
resolution=0.1,
rpath=str(tmpdir) + "/")
gr = grid.Dataset
xp, yp = gr.lons, gr.lats
# get dem
df = pdem.dem(**dic, grid_x=xp, grid_y=yp, dem_source=DEM_SOURCE)
df.adjust(natural_earth)
assert np.isnan(df.Dataset.fval.values).sum() == 0
def set(self, **kwargs):
if isinstance(self.model, str):
self.model = pyposeidon.read_model(self.model)
for attr, value in self.model.__dict__.items():
if not hasattr(self, attr):
setattr(self, attr, value)
execute = get_value(self, kwargs, "execute", False)
pwd = os.getcwd()
files = ["bctides.in", "launchSchism.sh", "/sflux/sflux_inputs.txt", "/outputs/flux.out"]
files_sym = [
"hgrid.gr3",
"hgrid.ll",
"manning.gr3",
"vgrid.in",
"drag.gr3",
"rough.gr3",
"station.in",
"windrot_geo2proj.gr3",
]
station_files = [
"/outputs/staout_1",
"/outputs/staout_2",
"/outputs/staout_3",
"/outputs/staout_4",
"/outputs/staout_5",
"/outputs/staout_6",
"/outputs/staout_7",
"/outputs/staout_8",
"/outputs/staout_9",
]
self.origin = self.model.rpath
self.date0 = self.model.date
if not os.path.exists(self.origin):
sys.stdout.write("Initial folder not present {}\n".format(self.origin))
sys.exit(1)
ppath = self.ppath
# create the folder/run path
rpath = self.cpath
if not os.path.exists(rpath):
os.makedirs(rpath)
copy2(ppath + self.tag + "_model.json", rpath) # copy the info file
# load model
with open(rpath + self.tag + "_model.json", "rb") as f:
info = pd.read_json(f, lines=True).T
info[info.isnull().values] = None
info = info.to_dict()[0]
args = set(kwargs.keys()).intersection(info.keys()) # modify dic with kwargs
for attr in list(args):
info[attr] = kwargs[attr]
info["config_file"] = ppath + "param.nml"
# update the properties
info["start_date"] = self.date
info["time_frame"] = self.time_frame
info["end_date"] = self.date + pd.to_timedelta(self.time_frame)
info["meteo_source"] = self.meteo
info["rpath"] = rpath
# info['grid_file'] = ppath + '/hgrid.gr3'
# for attr, value in self.items():
# setattr(info, attr, value)
m = pmodel(**info)
# Grid
m.grid = pgrid.grid(type="tri2d", **info)
# get lat/lon from file
if hasattr(self, "grid_file"):
info.update({"lon_min": m.grid.Dataset.SCHISM_hgrid_node_x.values.min()})
info.update({"lon_max": m.grid.Dataset.SCHISM_hgrid_node_x.values.max()})
info.update({"lat_min": m.grid.Dataset.SCHISM_hgrid_node_y.values.min()})
info.update({"lat_max": m.grid.Dataset.SCHISM_hgrid_node_y.values.max()})
# copy/link necessary files
logger.debug("copy necessary files")
for filename in files:
ipath = glob.glob(ppath + filename)
if ipath:
try:
copy2(ppath + filename, rpath + filename)
except:
dir_name, file_name = os.path.split(filename)
if not os.path.exists(rpath + dir_name):
os.makedirs(rpath + dir_name)
copy2(ppath + filename, rpath + filename)
logger.debug(".. done")
# copy the station files
logger.debug("copy station files")
for filename in station_files:
ipath = glob.glob(ppath + filename)
if ipath:
try:
copy2(ppath + filename, rpath + filename)
except:
dir_name, file_name = os.path.split(filename)
if not os.path.exists(rpath + dir_name):
os.makedirs(rpath + dir_name)
copy2(ppath + filename, rpath + filename)
logger.debug(".. done")
# symlink the station files
# logger.debug('symlink station files')
# for filename in station_files:
# ipath = glob.glob(self.path+self.folders[0] + filename)
# if ipath:
# if not os.path.exists(rpath + '/outputs/'):
# os.makedirs(rpath + '/outputs/')
# try:
# os.symlink(ipath[0],rpath + filename)
# except OSError as e:
# if e.errno == errno.EEXIST:
# logger.warning('Restart link present\n')
# logger.warning('overwriting\n')
# os.remove(rpath + filename)
# os.symlink(ipath[0],rpath + filename)
# logger.debug('.. done')
# symlink the big files
logger.debug("symlink model files")
for filename in files_sym:
ipath = glob.glob(self.origin + filename)
if ipath:
try:
os.symlink(ipath[0], rpath + filename)
except OSError as e:
if e.errno == errno.EEXIST:
logger.warning("Restart link present\n")
logger.warning("overwriting\n")
os.remove(rpath + filename)
os.symlink(ipath[0], rpath + filename)
logger.debug(".. done")
# create restart file
logger.debug("create restart file")
# check for combine hotstart
hotout = int((self.date - self.date0).total_seconds() / info["params"]["core"]["dt"])
logger.debug("hotout_it = {}".format(hotout))
resfile = glob.glob(ppath + "/outputs/hotstart_it={}.nc".format(hotout))
if not resfile:
# load model model from ppath
with open(ppath + self.tag + "_model.json", "rb") as f:
ph = pd.read_json(f, lines=True).T
ph[ph.isnull().values] = None
ph = ph.to_dict()[0]
p = pmodel(**ph)
p.hotstart(it=hotout)
# link restart file
inresfile = "/outputs/hotstart_it={}.nc".format(hotout)
outresfile = "/hotstart.nc"
logger.info("set restart\n")
try:
os.symlink(ppath + inresfile, rpath + outresfile)
except OSError as e:
if e.errno == errno.EEXIST:
logger.warning("Restart link present\n")
logger.warning("overwriting\n")
os.remove(rpath + outresfile)
os.symlink(ppath + inresfile, rpath + outresfile)
else:
raise e
# get new meteo
logger.info("process meteo\n")
flag = get_value(self, kwargs, "update", [])
check = [os.path.exists(rpath + "sflux/" + f) for f in ["sflux_air_1.0001.nc"]]
if (np.any(check) == False) or ("meteo" in flag):
m.force(**info)
if hasattr(self, "meteo_split_by"):
times, datasets = zip(*m.meteo.Dataset.groupby("time.{}".format(self.meteo_split_by)))
mpaths = ["sflux/sflux_air_1.{:04d}.nc".format(t + 1) for t in np.arange(len(times))]
for das, mpath in list(zip(datasets, mpaths)):
m.to_force(das, vars=["msl", "u10", "v10"], rpath=rpath, filename=mpath, date=self.date0)
else:
m.to_force(m.meteo.Dataset, vars=["msl", "u10", "v10"], rpath=rpath, date=self.date0)
else:
logger.warning("meteo files present\n")
# modify param file
rnday_new = (self.date - self.date0).total_seconds() / (3600 * 24.0) + pd.to_timedelta(
self.time_frame
).total_seconds() / (3600 * 24.0)
hotout_write = int(rnday_new * 24 * 3600 / info["params"]["core"]["dt"])
info["parameters"].update(
{
"ihot": 2,
"rnday": rnday_new,
"start_hour": self.date0.hour,
"start_day": self.date0.day,
"start_month": self.date0.month,
"start_year": self.date0.year,
}
)
m.config(output=True, **info)
m.config_file = rpath + "param.nml"
os.chdir(rpath)
# subprocess.call(rpath+'run_flow2d3d.sh',shell=True)
m.save()
if execute:
m.run()
# cleanup
# os.remove(rpath+'hotstart.nc')
# save compiled nc file
# out = data(**{'solver':m.solver,'rpath':rpath,'savenc':True})
logger.info("done for date :" + datetime.datetime.strftime(self.date, "%Y%m%d.%H"))
os.chdir(pwd)
def set(self,**kwargs):
if isinstance(self.model,str):
self.model = pyposeidon.read_model(self.model)
for attr, value in self.model.__dict__.items():
if not hasattr(self, attr): setattr(self, attr, value)
execute = get_value(self,kwargs,'execute', False)
pwd = os.getcwd()
files = [ 'bctides.in', 'launchSchism.sh','/sflux/sflux_inputs.txt', '/outputs/flux.out']
files_sym = ['hgrid.gr3', 'hgrid.ll', 'manning.gr3', 'vgrid.in', 'drag.gr3', 'rough.gr3', 'station.in', 'windrot_geo2proj.gr3']
station_files = ['/outputs/staout_1' , '/outputs/staout_2' , '/outputs/staout_3' , '/outputs/staout_4' , '/outputs/staout_5' , '/outputs/staout_6' , '/outputs/staout_7' , '/outputs/staout_8' , '/outputs/staout_9']
self.origin=self.model.rpath
self.date0 = self.model.date
if not os.path.exists(self.origin):
sys.stdout.write('Initial folder not present {}\n'.format(self.origin))
sys.exit(1)
ppath = self.ppath
# create the folder/run path
rpath=self.cpath
if not os.path.exists(rpath):
os.makedirs(rpath)
copy2(ppath+self.tag+'_model.json',rpath) #copy the info file
# load model
with open(rpath+self.tag+'_model.json', 'rb') as f:
info = pd.read_json(f,lines=True).T
info[info.isnull().values] = None
info = info.to_dict()[0]
args = set(kwargs.keys()).intersection(info.keys()) # modify dic with kwargs
for attr in list(args):
info[attr] = kwargs[attr]
info['config_file'] = ppath + 'param.nml'
#update the properties
info['start_date'] = self.date
info['time_frame'] = self.time_frame
info['end_date'] = self.date + pd.to_timedelta(self.time_frame)
info['meteo_source'] = self.meteo
info['rpath'] = rpath
# info['grid_file'] = ppath + '/hgrid.gr3'
# for attr, value in self.items():
# setattr(info, attr, value)
m=pmodel(**info)
# Grid
m.grid=pgrid.grid(type='tri2d',**info)
# get lat/lon from file
if hasattr(self, 'grid_file'):
info.update({'lon_min' : m.grid.Dataset.SCHISM_hgrid_node_x.values.min()})
info.update({'lon_max' : m.grid.Dataset.SCHISM_hgrid_node_x.values.max()})
info.update({'lat_min' : m.grid.Dataset.SCHISM_hgrid_node_y.values.min()})
info.update({'lat_max' : m.grid.Dataset.SCHISM_hgrid_node_y.values.max()})
# copy/link necessary files
logger.debug('copy necessary files')
for filename in files:
ipath = glob.glob(ppath+filename)
if ipath:
try:
copy2(ppath+filename,rpath+filename)
except:
dir_name ,file_name = os.path.split(filename)
if not os.path.exists(rpath + dir_name):
os.makedirs(rpath + dir_name)
copy2(ppath+filename,rpath+filename)
logger.debug('.. done')
#copy the station files
logger.debug('copy station files')
for filename in station_files:
ipath = glob.glob(ppath+filename)
if ipath:
try:
copy2(ppath+filename,rpath+filename)
except:
dir_name ,file_name = os.path.split(filename)
if not os.path.exists(rpath + dir_name):
os.makedirs(rpath + dir_name)
copy2(ppath+filename,rpath+filename)
logger.debug('.. done')
#symlink the station files
#logger.debug('symlink station files')
#for filename in station_files:
# ipath = glob.glob(self.path+self.folders[0] + filename)
# if ipath:
# if not os.path.exists(rpath + '/outputs/'):
# os.makedirs(rpath + '/outputs/')
# try:
# os.symlink(ipath[0],rpath + filename)
# except OSError as e:
# if e.errno == errno.EEXIST:
# logger.warning('Restart link present\n')
# logger.warning('overwriting\n')
# os.remove(rpath + filename)
# os.symlink(ipath[0],rpath + filename)
#logger.debug('.. done')
#symlink the big files
logger.debug('symlink model files')
for filename in files_sym:
ipath = glob.glob(self.origin+filename)
if ipath:
try:
os.symlink(ipath[0],rpath+filename)
except OSError as e:
if e.errno == errno.EEXIST:
logger.warning('Restart link present\n')
logger.warning('overwriting\n')
os.remove(rpath+filename)
os.symlink(ipath[0],rpath+filename)
logger.debug('.. done')
# create restart file
logger.debug('create restart file')
#check for combine hotstart
hotout=int((self.date - self.date0).total_seconds()/info['params']['core']['dt'])
logger.debug('hotout_it = {}'.format(hotout))
resfile=glob.glob(ppath+'/outputs/hotstart_it={}.nc'.format(hotout))
if not resfile:
# load model model from ppath
with open(ppath+self.tag+'_model.json', 'rb') as f:
ph = pd.read_json(f,lines=True).T
ph[ph.isnull().values] = None
ph = ph.to_dict()[0]
p = pmodel(**ph)
p.hotstart(it=hotout)
# link restart file
inresfile='/outputs/hotstart_it={}.nc'.format(hotout)
outresfile='/hotstart.nc'
logger.info('set restart\n')
try:
os.symlink(ppath+inresfile,rpath+outresfile)
except OSError as e:
if e.errno == errno.EEXIST:
logger.warning('Restart link present\n')
logger.warning('overwriting\n')
os.remove(rpath+outresfile)
os.symlink(ppath+inresfile,rpath+outresfile)
else:
raise e
#get new meteo
logger.info('process meteo\n')
flag = get_value(self,kwargs,'update',[])
check=[os.path.exists(rpath+'sflux/'+ f) for f in ['sflux_air_1.0001.nc']]
if (np.any(check)==False) or ('meteo' in flag):
m.force(**info)
if hasattr(self, 'meteo_split_by'):
times, datasets = zip(*m.meteo.Dataset.groupby('time.{}'.format(self.meteo_split_by)))
mpaths = ['sflux/sflux_air_1.{:04d}.nc'.format(t + 1) for t in np.arange(len(times))]
for das,mpath in list(zip(datasets,mpaths)):
m.to_force(das,vars=['msl','u10','v10'],rpath=rpath, filename=mpath, date=self.date0)
else:
m.to_force(m.meteo.Dataset,vars=['msl','u10','v10'],rpath=rpath,date=self.date0)
else:
logger.warning('meteo files present\n')
# modify param file
rnday_new = (self.date - self.date0).total_seconds()/(3600*24.) + pd.to_timedelta(self.time_frame).total_seconds()/(3600*24.)
hotout_write = int(rnday_new * 24 * 3600 / info['params']['core']['dt'])
info['parameters'].update({'ihot': 2, 'rnday':rnday_new, 'start_hour':self.date0.hour , 'start_day':self.date0.day, 'start_month':self.date0.month, 'start_year':self.date0.year})
m.config(output=True, **info)
m.config_file = rpath + 'param.nml'
os.chdir(rpath)
#subprocess.call(rpath+'run_flow2d3d.sh',shell=True)
m.save()
if execute : m.run()
#cleanup
# os.remove(rpath+'hotstart.nc')
# save compiled nc file
#out = data(**{'solver':m.solver,'rpath':rpath,'savenc':True})
logger.info('done for date :'+datetime.datetime.strftime(self.date,'%Y%m%d.%H'))
os.chdir(pwd)