def test_use_list_options(foo_nc): nco = Nco(debug=True) options = [] options.extend(["-a", 'units,time,o,c,"days since 1999-01-01"']) options.extend(["-a", "long_name,time,o,c,time"]) options.extend(["-a", "calendar,time,o,c,noleap"]) nco.ncatted(input=foo_nc, output="out.nc", options=options)
def test_use_list_options(foo_nc): nco = Nco(debug=True) options = [] options.extend(['-a', 'units,time,o,c,"days since 1999-01-01"']) options.extend(['-a', 'long_name,time,o,c,time']) options.extend(['-a', 'calendar,time,o,c,noleap']) nco.ncatted(input=foo_nc, output='out.nc', options=options)
def ocean_atlas_enhance(output_dir): output_file = output_dir.parent / 'ocean_atlas.nc' output_tmp_file = output_dir.parent / 'ocean_atlas_tmp.nc' # change value that fits in a packed short var nco = Nco() options = ['-O', '-a _FillValue,,o,f,-127'] nco.ncatted(input=str(output_file), output=str(output_tmp_file), options=options) # pack to use bytes # - requires output file defined with -o option options = ['-O', '-M flt_byt', f"-o {str(output_file)}"] nco.ncpdq(input=str(output_tmp_file), output=str(output_file), options=options)
def run(self, latidx, lonidx): try: inputfile_dir = self.config.get_dict(self.translator_type, 'inputfile_dir', default=os.path.join( '..', '..')) inputfiles = self.config.get_dict(self.translator_type, 'inputfile', default='1.soil.tile.nc4') inputfiles = param_to_list(inputfiles) latdelta, londelta = [ double(d) / 60 for d in self.config.get('delta').split(',') ] # convert to degrees nco = Nco() outputfiles = self.config.get_dict( self.translator_type, 'outputfile', default=inputs_to_outputs(inputfiles)) outputfiles = param_to_list(outputfiles) inputfiles = apply_prefix(inputfiles, inputfile_dir) for i in range(len(inputfiles)): inputfile = inputfiles[i] outputfile = outputfiles[i] with nc(inputfile) as f: variables = f.variables.keys() soil_id = f.getncattr('soil_id') # get latitude, longitude limits minlat = 90 - latdelta * latidx maxlat = minlat + latdelta minlon = -180 + londelta * (lonidx - 1) maxlon = minlon + londelta # additional options options = '-h -a lat,lon -d lat,%f,%f -d lon,%f,%f --rd' % ( minlat, maxlat, minlon, maxlon) if 'cropland' in variables: options += ' -w cropland' # perform aggregation nco.ncwa(input=inputfile, output=outputfile, options=options) # add degenerate profile dimension nco.ncecat(input=outputfile, output=outputfile, options='-O -h -u profile') nco.ncap2(input=outputfile, output=outputfile, options='-O -h -s profile[profile]=1') # add soil_id variable nco.ncap2(input=outputfile, output=outputfile, options='-O -h -s soil_id[profile,lat,lon]=1') nco.ncatted(input=outputfile, output=outputfile, options='-O -h -a units,soil_id,c,c,"mapping"') nco.ncatted(input=outputfile, output=outputfile, options='-O -h -a long_name,soil_id,c,c,"%s"' % str(soil_id)) nco.ncatted(input=outputfile, output=outputfile, options='-O -h -a soil_id,global,d,c,""') # change latitude, longitude to simulated point with nc(outputfile, 'a') as f: latv = f.variables['lat'] latv[:] = 90 - latdelta * (latidx - 0.5) lonv = f.variables['lon'] lonv[:] = -180 + londelta * (lonidx - 0.5) return True except: print "[%s]: %s" % (os.path.basename(__file__), traceback.format_exc()) return False
def combinelon(prefix, inputdir, fill_value='1e20', daily=False, year=None): if daily: files = [ inputdir + sep + f for f in filter(listdir(inputdir), '%s*.%d.psims.nc' % (prefix, year)) ] else: files = [ inputdir + sep + f for f in filter(listdir(inputdir), '%s*.psims.nc' % prefix) ] # tile latitude and longitude indices tlatidx = basename(files[0]).split('_')[1] lonidx = [int(basename(f).split('_')[2][:4]) for f in files] # get file information with nc(files[0]) as f: vars = setdiff1d(f.variables.keys(), ['time', 'scen', 'irr', 'lat', 'lon']) nscen = f.variables['scen'].size ldim = f.variables[vars[0]].dimensions[0] vunits = [0] * len(vars) vlnames = [0] * len(vars) for i in range(len(vars)): var = f.variables[vars[i]] vunits[i] = var.units if 'units' in var.ncattrs() else '' vlnames[i] = var.long_name if 'long_name' in var.ncattrs() else '' # fill longitude gaps for idx in setdiff1d(fulllonidx, lonidx): if daily: lonfile = inputdir + sep + '%s_%s_%04d.%d.psims.nc' % ( prefix, tlatidx, idx, year) else: lonfile = inputdir + sep + '%s_%s_%04d.psims.nc' % (prefix, tlatidx, idx) copyfile(files[0], lonfile) lons = arange(-180 + tlond * (idx - 1) + lond / 2., -180 + tlond * idx, lond) with nc(lonfile, 'a') as f: lonvar = f.variables['lon'] lonvar[:] = lons for i in range(len(vars)): var = f.variables[vars[i]] var[:] = masked_array(zeros(var.shape), mask=ones(var.shape)) files.append(lonfile) # output file if daily: outputfile = outputdir + sep + '%s_%s.%d.psims.nc' % (prefix, tlatidx, year) else: outputfile = outputdir + sep + '%s_%s.psims.nc' % (prefix, tlatidx) nco = Nco() # make longitude lead dimension for i in range(len(files)): nco.ncpdq(input=files[i], output=files[i], options='-O -h -a lon,%s' % str(ldim)) # concatenate all files if daily: inputfiles = ' '.join([ inputdir + sep + '%s_%s_%04d.%d.psims.nc' % (prefix, tlatidx, idx, year) for idx in fulllonidx ]) else: inputfiles = ' '.join([ inputdir + sep + '%s_%s_%04d.psims.nc' % (prefix, tlatidx, idx) for idx in fulllonidx ]) nco.ncrcat(input=inputfiles, output=outputfile, options='-h') # make latitude lead dimension nco.ncpdq(input=outputfile, output=outputfile, options='-O -h -a lat,lon') # add new scenario dimension nscennew = nscen / (1 + irrflag) scen_range = ','.join([str(s) for s in range(1, nscennew + 1)]) scenopt = '-O -h -s \'defdim("scen_new",%d)\' -s "scen_new[scen_new]={%s}"' % ( nscennew, scen_range) nco.ncap2(input=outputfile, output=outputfile, options=scenopt) nco.ncatted(input=outputfile, output=outputfile, options='-O -h -a units,scen_new,c,c,"no"') nco.ncatted(input=outputfile, output=outputfile, options='-O -h -a long_name,scen_new,c,c,"scenario"') # add irr dimension nirr = 1 + irrflag irr_range = ','.join([str(i) for i in range(1, nirr + 1)]) irr_lname = ['ir', 'rf'][:1 + irrflag] if irr1st else ['rf', 'ir'][:1 + irrflag] irropt = '-O -h -s \'defdim("irr",%d)\' -s "irr[irr]={%s}"' % (nirr, irr_range) nco.ncap2(input=outputfile, output=outputfile, options=irropt) nco.ncatted(input=outputfile, output=outputfile, options='-O -h -a units,irr,c,c,"mapping"') nco.ncatted(input=outputfile, output=outputfile, options='-O -h -a long_name,irr,c,c,"%s"' % ','.join(irr_lname)) # refactor variables for i in range(len(vars)): var = str(vars[i]) # create new variable opt = '-O -h -s "\'%s_new\'[lat,scen_new,irr,lon,time]=0.0f"' % var nco.ncap2(input=outputfile, output=outputfile, options=opt) # set attributes opt = '-O -h -a _FillValue,%s_new,c,f,%s' % (var, fill_value) nco.ncatted(input=outputfile, output=outputfile, options=opt) if vunits[i]: opt = '-O -h -a units,%s_new,c,c,"%s"' % (var, str(vunits[i])) nco.ncatted(input=outputfile, output=outputfile, options=opt) if vlnames[i]: opt = '-O -h -a long_name,%s_new,c,c,"%s"' % (var, str(vlnames[i])) nco.ncatted(input=outputfile, output=outputfile, options=opt) # set value opt = '-O -h -s "\'%s_new\'(:,:,:,:,:)=\'%s\'"' % (var, var) nco.ncap2(input=outputfile, output=outputfile, options=opt) # remove old variable opt = '-O -h -x -v %s' % var nco.ncks(input=outputfile, output=outputfile, options=opt) # rename new variable opt = '-O -h -v %s_new,%s' % (var, var) nco.ncrename(input=outputfile, output=outputfile, options=opt) # remove old scenario dimension nco.ncks(input=outputfile, output=outputfile, options='-O -h -x -v scen') nco.ncrename(input=outputfile, output=outputfile, options='-O -h -v scen_new,scen') # limit spatial extent to sim grid nco.ncks(input=outputfile, output=outputfile, options='-O -h -d lon,%f,%f' % (lon0, lon1))
dirs.append(dir_ini) dir_hs = 'processed_hillshade' dirs.append(dir_hs) for dir_processed in dirs: if not os.path.isdir(os.path.join(idir, dir_processed)): os.mkdir(os.path.join(idir, dir_processed)) pvars = ('thk', 'usurf', 'velsurf_mag', 'velbase_mag') fill_value = -2e9 v_str = ' '.join('='.join([x, str(fill_value) + ';']) for x in pvars) m_str = 'sftgif=mask*0; where(thk>10) {sftgif=1;}; where(usurf>300) {sftgif=3;};' ncap2_str = 'where(thk<10) {{ {} }}; {}'.format(v_str, m_str) exp_files = glob(os.path.join(idir, 'state', '*.nc')) for exp_file in exp_files: exp_basename = os.path.split(exp_file)[-1].split('.nc')[0] exp_nc_wd = os.path.join(idir, dir_nc, exp_basename + '.nc') exp_gtiff_wd = os.path.join(idir, dir_gtiff, exp_basename + '.tif') logger.info('masking variables where ice thickness < 10m') nco.ncap2(input='-s "{}" {}'.format(ncap2_str, exp_file), output=exp_nc_wd, overwrite=True) opt = [c.Atted(mode="o", att_name="_FillValue", var_name=myvar, value=fill_value) for myvar in pvars] nco.ncatted(input=exp_nc_wd, options=opt) logger.info('extracting ice-noice transition') exp_ini_wd = os.path.join(idir, dir_ini, exp_basename + '.shp') cmd = ['extract_interface.py', '-m', 'sftgif', '-t', 'ice_noice', '--epsg', '26710', '-o', exp_ini_wd, exp_nc_wd] sub.call(cmd) for mvar in pvars: m_exp_nc_wd = 'NETCDF:{}:{}'.format(exp_nc_wd, mvar) m_exp_gtiff_wd = os.path.join(idir, dir_gtiff, mvar + '_' + exp_basename + '.tif') logger.info('Converting variable {} to GTiff and save as {}'.format(mvar, m_exp_gtiff_wd)) gdal.Translate(m_exp_gtiff_wd, m_exp_nc_wd, options=gdal_gtiff_options)
gdal.Translate(ofile_merged_wp, ifile_wp, options=gdal_nc_options) rDict = {'Band1': 'velsurf_mag'} opt = [c.Rename('variable', rDict)] nco.ncrename(input=ofile_merged_wp, options=opt) opt = [ c.Atted(mode="o", att_name="proj4", var_name='global', value="+init=epsg:3413", stype='c'), c.Atted(mode="o", att_name="units", var_name='velsurf_mag', value="m year-1") ] nco.ncatted(input=ofile_merged_wp, options=opt) for mvar in ('vx', 'vy', 'ex', 'ey'): ifile = '.'.join(['_'.join([basename, mvar, version]), 'tif']) ofile = '.'.join(['_'.join([basename, mvar, version]), 'nc']) ofile_wp = os.path.join(basedir, ofile) ifile_wp = os.path.join(basedir, ifile) gdal.Translate(ofile_wp, ifile_wp, options=gdal_nc_options) rDict = {'Band1': var_dict[mvar]} opt = [c.Rename('variable', rDict)] nco.ncrename(input=ofile_wp, options=opt) opt = [ c.Atted(mode="o", att_name="units", var_name=var_dict[mvar], value="m year-1") ]
options=[c.Rename("variable", rDict), c.Rename("dimension", dDict)]) logger.info("add topo file {} to {}".format(topo_file, merged_file_time_mean)) nco.ncks(input=topo_file_tmp_2, output=merged_file_time_mean, append=True, variable="usurf") opt = [ c.Atted(mode="o", att_name="units", var_name="usurf", value="m"), c.Atted(mode="o", att_name="standard_name", var_name="usurf", value="surface_altitude"), ] nco.ncatted(input=merged_file_time_mean, options=opt) for grid_spacing in (18000, 9000, 4500, 3600, 3000, 2400, 1800, 1500, 1200, 900, 600): grid_file = "epsg3413_griddes_{}m.nc".format(grid_spacing) logger.info("generating grid description {}".format(grid_file)) create_epsg3413_grid(grid_file, grid_spacing) epsg3413_merged_file_time_mean = "DMI-HIRHAM5_GL2_ERAI_2001_2014_{}_EPSG3413_{}m.nc".format( time_mean, grid_spacing) logger.info("remapping {} to {}".format(merged_file_time_mean, epsg3413_merged_file_time_mean)) tmpfile = cdo.remapycon("{} -setgrid,{}".format(grid_file, rotated_grid_file), input=merged_file_time_mean, options="-f nc4") cdo.setmisstoc(0, input=tmpfile, output=epsg3413_merged_file_time_mean)
daily_mean = "DM" time_mean = "TM" pr_merged_file_daily_mean = "DMI-HIRHAM5_GL2_ERAI_1980_2014_PR_{}.nc".format(daily_mean) # logger.info('merge files') # tmpfile = cdo.merge(input=' '.join(pr_files)) # logger.info('removing height dimension') # nco.ncwa(input=tmpfile, output=pr_merged_file_daily_mean, average='height') # logger.info('adjusting time axis') # adjust_time_axis(pr_merged_file_daily_mean, '1980-1-1', '2015-1-1', 'days', '1980-1-1', 'daily') opt = [c.Atted(mode="o", att_name="units", var_name="time", value="days since 1980-01-01 03:00:00")] nco.ncatted(input=pr_merged_file_daily_mean, options=opt) pr_merged_file_time_mean = "DMI-HIRHAM5_GL2_ERAI_1980_2014_PR_{}.nc".format(time_mean) logger.info("calculate time mean") cdo.timmean(input=pr_merged_file_daily_mean, output=pr_merged_file_time_mean) # # add topo file # logger.info('add topo file {} to {}'.format(topo_file, pr_merged_file_time_mean)) # nco.ncks(input=topo_file, output=pr_merged_file_time_mean, append=True) for grid_spacing in (18000, 9000, 4500, 3600, 3000, 2400, 1800, 1500, 1200, 900, 600, 450, 300): grid_file = "epsg3413_griddes_{}m.nc".format(grid_spacing) logger.info("generating grid description {}".format(grid_file)) create_epsg3413_grid(grid_file, grid_spacing) epsg3414_pr_merged_file_time_mean = "DMI-HIRHAM5_GL2_ERAI_1980_2014_PR_{}_EPSG3413_{}m.nc".format( time_mean, grid_spacing ) logger.info("Remapping {} to {}".format(pr_merged_file_time_mean, epsg3414_pr_merged_file_time_mean))
#sub.call(cmd) logger.info('masking variables where ice thickness < 10m') nco.ncks(input=exp_file, output=exp_nc_wd, variable=','.join([x for x in pvars]), overwrite=True) nco.ncap2(input='-6 -s "{}" {}'.format(ncap2_str, exp_nc_wd), output=exp_nc_wd, overwrite=True) opt = [ c.Atted(mode="o", att_name="_FillValue", var_name=myvar, value=fill_value) for myvar in ppvars ] nco.ncatted(input=exp_nc_wd, options=opt) for mvar in pvars: m_exp_nc_wd = 'NETCDF:{}:{}'.format(exp_nc_wd, mvar) m_exp_gtiff_wd = os.path.join(idir, dir_gtiff, mvar + '_' + exp_basename + '.tif') logger.info('Converting variable {} to GTiff and save as {}'.format( mvar, m_exp_gtiff_wd)) gdal.Translate(m_exp_gtiff_wd, m_exp_nc_wd, options=gdal_gtiff_options) if mvar == 'usurf': m_exp_hs_wd = os.path.join(idir, dir_hs, mvar + '_' + exp_basename + '_hs.tif') logger.info('Generating hillshade {}'.format(m_exp_hs_wd)) gdal.DEMProcessing(m_exp_hs_wd, m_exp_nc_wd, 'hillshade', options=gdal_options)