def test_use_list_options(foo_nc):
nco = Nco(debug=True)
options = []
options.extend(["-a", 'units,time,o,c,"days since 1999-01-01"'])
options.extend(["-a", "long_name,time,o,c,time"])
options.extend(["-a", "calendar,time,o,c,noleap"])
nco.ncatted(input=foo_nc, output="out.nc", options=options)
def test_use_list_options(foo_nc):
nco = Nco(debug=True)
options = []
options.extend(['-a', 'units,time,o,c,"days since 1999-01-01"'])
options.extend(['-a', 'long_name,time,o,c,time'])
options.extend(['-a', 'calendar,time,o,c,noleap'])
nco.ncatted(input=foo_nc, output='out.nc', options=options)
def test_use_list_options(foo_nc):
nco = Nco(debug=True)
options = []
options.extend(['-a', 'units,time,o,c,"days since 1999-01-01"'])
options.extend(['-a', 'long_name,time,o,c,time'])
options.extend(['-a', 'calendar,time,o,c,noleap'])
nco.ncatted(input=foo_nc, output='out.nc', options=options)
def ocean_atlas_enhance(output_dir):
output_file = output_dir.parent / 'ocean_atlas.nc'
output_tmp_file = output_dir.parent / 'ocean_atlas_tmp.nc'
# change value that fits in a packed short var
nco = Nco()
options = ['-O', '-a _FillValue,,o,f,-127']
nco.ncatted(input=str(output_file),
output=str(output_tmp_file),
options=options)
# pack to use bytes
# - requires output file defined with -o option
options = ['-O', '-M flt_byt', f"-o {str(output_file)}"]
nco.ncpdq(input=str(output_tmp_file),
output=str(output_file),
options=options)
def run(self, latidx, lonidx):
try:
inputfile_dir = self.config.get_dict(self.translator_type,
'inputfile_dir',
default=os.path.join(
'..', '..'))
inputfiles = self.config.get_dict(self.translator_type,
'inputfile',
default='1.soil.tile.nc4')
inputfiles = param_to_list(inputfiles)
latdelta, londelta = [
double(d) / 60 for d in self.config.get('delta').split(',')
] # convert to degrees
nco = Nco()
outputfiles = self.config.get_dict(
self.translator_type,
'outputfile',
default=inputs_to_outputs(inputfiles))
outputfiles = param_to_list(outputfiles)
inputfiles = apply_prefix(inputfiles, inputfile_dir)
for i in range(len(inputfiles)):
inputfile = inputfiles[i]
outputfile = outputfiles[i]
with nc(inputfile) as f:
variables = f.variables.keys()
soil_id = f.getncattr('soil_id')
# get latitude, longitude limits
minlat = 90 - latdelta * latidx
maxlat = minlat + latdelta
minlon = -180 + londelta * (lonidx - 1)
maxlon = minlon + londelta
# additional options
options = '-h -a lat,lon -d lat,%f,%f -d lon,%f,%f --rd' % (
minlat, maxlat, minlon, maxlon)
if 'cropland' in variables:
options += ' -w cropland'
# perform aggregation
nco.ncwa(input=inputfile, output=outputfile, options=options)
# add degenerate profile dimension
nco.ncecat(input=outputfile,
output=outputfile,
options='-O -h -u profile')
nco.ncap2(input=outputfile,
output=outputfile,
options='-O -h -s profile[profile]=1')
# add soil_id variable
nco.ncap2(input=outputfile,
output=outputfile,
options='-O -h -s soil_id[profile,lat,lon]=1')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a units,soil_id,c,c,"mapping"')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a long_name,soil_id,c,c,"%s"' %
str(soil_id))
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a soil_id,global,d,c,""')
# change latitude, longitude to simulated point
with nc(outputfile, 'a') as f:
latv = f.variables['lat']
latv[:] = 90 - latdelta * (latidx - 0.5)
lonv = f.variables['lon']
lonv[:] = -180 + londelta * (lonidx - 0.5)
return True
except:
print "[%s]: %s" % (os.path.basename(__file__),
traceback.format_exc())
return False
def combinelon(prefix, inputdir, fill_value='1e20', daily=False, year=None):
if daily:
files = [
inputdir + sep + f
for f in filter(listdir(inputdir), '%s*.%d.psims.nc' %
(prefix, year))
]
else:
files = [
inputdir + sep + f
for f in filter(listdir(inputdir), '%s*.psims.nc' % prefix)
]
# tile latitude and longitude indices
tlatidx = basename(files[0]).split('_')[1]
lonidx = [int(basename(f).split('_')[2][:4]) for f in files]
# get file information
with nc(files[0]) as f:
vars = setdiff1d(f.variables.keys(),
['time', 'scen', 'irr', 'lat', 'lon'])
nscen = f.variables['scen'].size
ldim = f.variables[vars[0]].dimensions[0]
vunits = [0] * len(vars)
vlnames = [0] * len(vars)
for i in range(len(vars)):
var = f.variables[vars[i]]
vunits[i] = var.units if 'units' in var.ncattrs() else ''
vlnames[i] = var.long_name if 'long_name' in var.ncattrs() else ''
# fill longitude gaps
for idx in setdiff1d(fulllonidx, lonidx):
if daily:
lonfile = inputdir + sep + '%s_%s_%04d.%d.psims.nc' % (
prefix, tlatidx, idx, year)
else:
lonfile = inputdir + sep + '%s_%s_%04d.psims.nc' % (prefix,
tlatidx, idx)
copyfile(files[0], lonfile)
lons = arange(-180 + tlond * (idx - 1) + lond / 2., -180 + tlond * idx,
lond)
with nc(lonfile, 'a') as f:
lonvar = f.variables['lon']
lonvar[:] = lons
for i in range(len(vars)):
var = f.variables[vars[i]]
var[:] = masked_array(zeros(var.shape), mask=ones(var.shape))
files.append(lonfile)
# output file
if daily:
outputfile = outputdir + sep + '%s_%s.%d.psims.nc' % (prefix, tlatidx,
year)
else:
outputfile = outputdir + sep + '%s_%s.psims.nc' % (prefix, tlatidx)
nco = Nco()
# make longitude lead dimension
for i in range(len(files)):
nco.ncpdq(input=files[i],
output=files[i],
options='-O -h -a lon,%s' % str(ldim))
# concatenate all files
if daily:
inputfiles = ' '.join([
inputdir + sep + '%s_%s_%04d.%d.psims.nc' %
(prefix, tlatidx, idx, year) for idx in fulllonidx
])
else:
inputfiles = ' '.join([
inputdir + sep + '%s_%s_%04d.psims.nc' % (prefix, tlatidx, idx)
for idx in fulllonidx
])
nco.ncrcat(input=inputfiles, output=outputfile, options='-h')
# make latitude lead dimension
nco.ncpdq(input=outputfile, output=outputfile, options='-O -h -a lat,lon')
# add new scenario dimension
nscennew = nscen / (1 + irrflag)
scen_range = ','.join([str(s) for s in range(1, nscennew + 1)])
scenopt = '-O -h -s \'defdim("scen_new",%d)\' -s "scen_new[scen_new]={%s}"' % (
nscennew, scen_range)
nco.ncap2(input=outputfile, output=outputfile, options=scenopt)
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a units,scen_new,c,c,"no"')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a long_name,scen_new,c,c,"scenario"')
# add irr dimension
nirr = 1 + irrflag
irr_range = ','.join([str(i) for i in range(1, nirr + 1)])
irr_lname = ['ir', 'rf'][:1 + irrflag] if irr1st else ['rf', 'ir'][:1 +
irrflag]
irropt = '-O -h -s \'defdim("irr",%d)\' -s "irr[irr]={%s}"' % (nirr,
irr_range)
nco.ncap2(input=outputfile, output=outputfile, options=irropt)
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a units,irr,c,c,"mapping"')
nco.ncatted(input=outputfile,
output=outputfile,
options='-O -h -a long_name,irr,c,c,"%s"' %
','.join(irr_lname))
# refactor variables
for i in range(len(vars)):
var = str(vars[i])
# create new variable
opt = '-O -h -s "\'%s_new\'[lat,scen_new,irr,lon,time]=0.0f"' % var
nco.ncap2(input=outputfile, output=outputfile, options=opt)
# set attributes
opt = '-O -h -a _FillValue,%s_new,c,f,%s' % (var, fill_value)
nco.ncatted(input=outputfile, output=outputfile, options=opt)
if vunits[i]:
opt = '-O -h -a units,%s_new,c,c,"%s"' % (var, str(vunits[i]))
nco.ncatted(input=outputfile, output=outputfile, options=opt)
if vlnames[i]:
opt = '-O -h -a long_name,%s_new,c,c,"%s"' % (var, str(vlnames[i]))
nco.ncatted(input=outputfile, output=outputfile, options=opt)
# set value
opt = '-O -h -s "\'%s_new\'(:,:,:,:,:)=\'%s\'"' % (var, var)
nco.ncap2(input=outputfile, output=outputfile, options=opt)
# remove old variable
opt = '-O -h -x -v %s' % var
nco.ncks(input=outputfile, output=outputfile, options=opt)
# rename new variable
opt = '-O -h -v %s_new,%s' % (var, var)
nco.ncrename(input=outputfile, output=outputfile, options=opt)
# remove old scenario dimension
nco.ncks(input=outputfile, output=outputfile, options='-O -h -x -v scen')
nco.ncrename(input=outputfile,
output=outputfile,
options='-O -h -v scen_new,scen')
# limit spatial extent to sim grid
nco.ncks(input=outputfile,
output=outputfile,
options='-O -h -d lon,%f,%f' % (lon0, lon1))
dirs.append(dir_ini)
dir_hs = 'processed_hillshade'
dirs.append(dir_hs)
for dir_processed in dirs:
if not os.path.isdir(os.path.join(idir, dir_processed)):
os.mkdir(os.path.join(idir, dir_processed))
pvars = ('thk', 'usurf', 'velsurf_mag', 'velbase_mag')
fill_value = -2e9
v_str = ' '.join('='.join([x, str(fill_value) + ';']) for x in pvars)
m_str = 'sftgif=mask*0; where(thk>10) {sftgif=1;}; where(usurf>300) {sftgif=3;};'
ncap2_str = 'where(thk<10) {{ {} }}; {}'.format(v_str, m_str)
exp_files = glob(os.path.join(idir, 'state', '*.nc'))
for exp_file in exp_files:
exp_basename = os.path.split(exp_file)[-1].split('.nc')[0]
exp_nc_wd = os.path.join(idir, dir_nc, exp_basename + '.nc')
exp_gtiff_wd = os.path.join(idir, dir_gtiff, exp_basename + '.tif')
logger.info('masking variables where ice thickness < 10m')
nco.ncap2(input='-s "{}" {}'.format(ncap2_str, exp_file), output=exp_nc_wd, overwrite=True)
opt = [c.Atted(mode="o", att_name="_FillValue", var_name=myvar, value=fill_value) for myvar in pvars]
nco.ncatted(input=exp_nc_wd, options=opt)
logger.info('extracting ice-noice transition')
exp_ini_wd = os.path.join(idir, dir_ini, exp_basename + '.shp')
cmd = ['extract_interface.py', '-m', 'sftgif', '-t', 'ice_noice', '--epsg', '26710', '-o', exp_ini_wd, exp_nc_wd]
sub.call(cmd)
for mvar in pvars:
m_exp_nc_wd = 'NETCDF:{}:{}'.format(exp_nc_wd, mvar)
m_exp_gtiff_wd = os.path.join(idir, dir_gtiff, mvar + '_' + exp_basename + '.tif')
logger.info('Converting variable {} to GTiff and save as {}'.format(mvar, m_exp_gtiff_wd))
gdal.Translate(m_exp_gtiff_wd, m_exp_nc_wd, options=gdal_gtiff_options)
gdal.Translate(ofile_merged_wp, ifile_wp, options=gdal_nc_options)
rDict = {'Band1': 'velsurf_mag'}
opt = [c.Rename('variable', rDict)]
nco.ncrename(input=ofile_merged_wp, options=opt)
opt = [
c.Atted(mode="o",
att_name="proj4",
var_name='global',
value="+init=epsg:3413",
stype='c'),
c.Atted(mode="o",
att_name="units",
var_name='velsurf_mag',
value="m year-1")
]
nco.ncatted(input=ofile_merged_wp, options=opt)
for mvar in ('vx', 'vy', 'ex', 'ey'):
ifile = '.'.join(['_'.join([basename, mvar, version]), 'tif'])
ofile = '.'.join(['_'.join([basename, mvar, version]), 'nc'])
ofile_wp = os.path.join(basedir, ofile)
ifile_wp = os.path.join(basedir, ifile)
gdal.Translate(ofile_wp, ifile_wp, options=gdal_nc_options)
rDict = {'Band1': var_dict[mvar]}
opt = [c.Rename('variable', rDict)]
nco.ncrename(input=ofile_wp, options=opt)
opt = [
c.Atted(mode="o",
att_name="units",
var_name=var_dict[mvar],
value="m year-1")
]
options=[c.Rename("variable", rDict),
c.Rename("dimension", dDict)])
logger.info("add topo file {} to {}".format(topo_file, merged_file_time_mean))
nco.ncks(input=topo_file_tmp_2,
output=merged_file_time_mean,
append=True,
variable="usurf")
opt = [
c.Atted(mode="o", att_name="units", var_name="usurf", value="m"),
c.Atted(mode="o",
att_name="standard_name",
var_name="usurf",
value="surface_altitude"),
]
nco.ncatted(input=merged_file_time_mean, options=opt)
for grid_spacing in (18000, 9000, 4500, 3600, 3000, 2400, 1800, 1500, 1200,
900, 600):
grid_file = "epsg3413_griddes_{}m.nc".format(grid_spacing)
logger.info("generating grid description {}".format(grid_file))
create_epsg3413_grid(grid_file, grid_spacing)
epsg3413_merged_file_time_mean = "DMI-HIRHAM5_GL2_ERAI_2001_2014_{}_EPSG3413_{}m.nc".format(
time_mean, grid_spacing)
logger.info("remapping {} to {}".format(merged_file_time_mean,
epsg3413_merged_file_time_mean))
tmpfile = cdo.remapycon("{} -setgrid,{}".format(grid_file,
rotated_grid_file),
input=merged_file_time_mean,
options="-f nc4")
cdo.setmisstoc(0, input=tmpfile, output=epsg3413_merged_file_time_mean)
daily_mean = "DM"
time_mean = "TM"
pr_merged_file_daily_mean = "DMI-HIRHAM5_GL2_ERAI_1980_2014_PR_{}.nc".format(daily_mean)
# logger.info('merge files')
# tmpfile = cdo.merge(input=' '.join(pr_files))
# logger.info('removing height dimension')
# nco.ncwa(input=tmpfile, output=pr_merged_file_daily_mean, average='height')
# logger.info('adjusting time axis')
# adjust_time_axis(pr_merged_file_daily_mean, '1980-1-1', '2015-1-1', 'days', '1980-1-1', 'daily')
opt = [c.Atted(mode="o", att_name="units", var_name="time", value="days since 1980-01-01 03:00:00")]
nco.ncatted(input=pr_merged_file_daily_mean, options=opt)
pr_merged_file_time_mean = "DMI-HIRHAM5_GL2_ERAI_1980_2014_PR_{}.nc".format(time_mean)
logger.info("calculate time mean")
cdo.timmean(input=pr_merged_file_daily_mean, output=pr_merged_file_time_mean)
# # add topo file
# logger.info('add topo file {} to {}'.format(topo_file, pr_merged_file_time_mean))
# nco.ncks(input=topo_file, output=pr_merged_file_time_mean, append=True)
for grid_spacing in (18000, 9000, 4500, 3600, 3000, 2400, 1800, 1500, 1200, 900, 600, 450, 300):
grid_file = "epsg3413_griddes_{}m.nc".format(grid_spacing)
logger.info("generating grid description {}".format(grid_file))
create_epsg3413_grid(grid_file, grid_spacing)
epsg3414_pr_merged_file_time_mean = "DMI-HIRHAM5_GL2_ERAI_1980_2014_PR_{}_EPSG3413_{}m.nc".format(
time_mean, grid_spacing
)
logger.info("Remapping {} to {}".format(pr_merged_file_time_mean, epsg3414_pr_merged_file_time_mean))
#sub.call(cmd)
logger.info('masking variables where ice thickness < 10m')
nco.ncks(input=exp_file,
output=exp_nc_wd,
variable=','.join([x for x in pvars]),
overwrite=True)
nco.ncap2(input='-6 -s "{}" {}'.format(ncap2_str, exp_nc_wd),
output=exp_nc_wd,
overwrite=True)
opt = [
c.Atted(mode="o",
att_name="_FillValue",
var_name=myvar,
value=fill_value) for myvar in ppvars
]
nco.ncatted(input=exp_nc_wd, options=opt)
for mvar in pvars:
m_exp_nc_wd = 'NETCDF:{}:{}'.format(exp_nc_wd, mvar)
m_exp_gtiff_wd = os.path.join(idir, dir_gtiff,
mvar + '_' + exp_basename + '.tif')
logger.info('Converting variable {} to GTiff and save as {}'.format(
mvar, m_exp_gtiff_wd))
gdal.Translate(m_exp_gtiff_wd, m_exp_nc_wd, options=gdal_gtiff_options)
if mvar == 'usurf':
m_exp_hs_wd = os.path.join(idir, dir_hs,
mvar + '_' + exp_basename + '_hs.tif')
logger.info('Generating hillshade {}'.format(m_exp_hs_wd))
gdal.DEMProcessing(m_exp_hs_wd,
m_exp_nc_wd,
'hillshade',
options=gdal_options)
