def normalize_time(netcdf_file): epoch_units = 'seconds since 1970-01-01T00:00:00Z' millisecond_units = 'milliseconds since 1858-11-17T00:00:00Z' with nc4.Dataset(netcdf_file, 'a') as nc: # Signell said this works, any problems and we can all blame him! time_data = nc4.num2date( ( np.int64(nc.variables['time'][:]) - 2400001 ) * 3600 * 24 * 1000 + nc.variables['time2'][:].__array__(), units=millisecond_units ) nc.renameVariable("time", "old_time") nc.sync() time = nc.createVariable('time', 'f8', ('time')) time.units = epoch_units time.standard_name = "time" time.long_name = "time of measurement" time.calendar = "gregorian" time.axis = "T" time[:] = nc4.date2num(time_data, units=epoch_units).round() o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-x', '-v', 'time2,old_time' ] )
def __check_weather(self, filename, latidx, lonidx, latdelta, londelta): with Dataset(filename) as f: lats = f.variables['lat'][:] lons = f.variables['lon'][:] glatdelta = abs(diff(lats)[0]) glondelta = abs(diff(lons)[0]) minlat, maxlat, minlon, maxlon = self.__get_range( latidx, lonidx, latdelta, londelta, glatdelta, glondelta) # select first time nco = Nco() options = '-h -d time,0 -d lat,%f,%f -d lon,%f,%f' % (minlat, maxlat, minlon, maxlon) tmpfile = "checker.wth.nc4" nco.ncks(input=filename, output=tmpfile, options=options) with Dataset(tmpfile) as f: varnames = f.variables.keys() prvars = ['pr', 'prcp', 'pre', 'precip'] found = False for i in range(len(prvars)): patt = '%s$|%s.*' % (prvars[i], prvars[i]) for j in range(len(varnames)): if compile(patt).match(varnames[j]): pr = f.variables[varnames[j]][:] found = True break if found: break if not found: pr = array([]) hasweather = pr.size and (not isMaskedArray(pr) or not pr.mask.all()) remove(tmpfile) return hasweather
def normalize_ctd_depths(netcdf_file): with CFDataset(netcdf_file, 'a') as nc: depths = nc.variables['depth'][:][0] z_atts = nc.vatts('depth') # Remove the old depth variable so we can add a new one with no dimensions o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-x', '-v', 'depth' ] ) # Add the new Z variable with nc4.Dataset(netcdf_file, 'a') as nc: z = nc.createVariable('depth', 'f4') z[:] = depths z_atts.update({ 'standard_name': 'depth', 'axis': 'Z', 'positive': 'down' }) z.setncatts(z_atts)
def test_ncks_hdf2nc(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively, it is now easy to convert HDF4 files to netCDF files directly, e.g., ncks fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+) ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9) """ nco = Nco(debug=True) nco.ncks(input=hdf_file, output='foo.nc') nco.ncks(input=hdf_file, output='foo.nc', hdf4=True)
def test_command_line_options(foo_nc): """ 3.4 Command Line Options ncks -D 3 in.nc # Short option ncks --dbg_lvl=3 in.nc # Long option, preferred form ncks --dbg_lvl 3 in.nc # Long option, alternate form """ nco = Nco(debug=True) nco.ncks(input=foo_nc, options=["-O -D 3"]) nco.ncks(input=foo_nc, options=["-O --dbg_lvl=3"]) nco.ncks(input=foo_nc, options=["-O --dbg_lvl 3"])
def normalize_netcdf4(netcdf_file): o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-4', '-L3' ] )
def test_ncks_hdf2nc(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively, it is now easy to convert HDF4 files to netCDF files directly, e.g., ncks fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+) ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9) """ if hdf_file is None: pytest.skip("Skipped because h5py is not installed") nco = Nco(debug=True) nco.ncks(input=hdf_file, output="foo.nc") nco.ncks(input=hdf_file, output="foo.nc", hdf4=True)
def convert_dl_to_nc4c(self, fns): self.log.debug('Converting to NETCDF4_CLASSIC ...') if not _NCO: self.log.error( 'Cannot convert to NETCDF4_CLASSIC: nco is not installed') raise ImportError('cannot import nco library') nco = Nco() fns_new = [] for fn in fns: fn_new = tempfile.mktemp(suffix='.nc', dir=self.cfg['temppath']) nco.ncks(input=fn, output=fn_new, options=['-O', '-7']) fns_new.append(fn_new) self.cfg['dldriver'].clean_tempfiles([fn]) self.log.debug('... done.') return fns_new
def test_temp_output_files(foo_nc): """ 2.3 Temporary Output Files ncks in.nc out.nc # Default: create out.pid.tmp.nc then move to out.nc ncks --wrt_tmp_fl in.nc out.nc # Same as default ncks --no_tmp_fl in.nc out.nc # Create out.nc directly on disk ncks --no_tmp_fl in.nc in.nc # ERROR-prone! Overwrite in.nc with itself ncks --create_ram --no_tmp_fl in.nc in.nc # Create in RAM, write to disk ncks --open_ram --no_tmp_fl in.nc in.nc # Read into RAM, write to disk """ nco = Nco(debug=True) nco.ncks(input=foo_nc, output='bar.nc') nco.ncks(input=foo_nc, output='bar.nc', wrt_tmp_fl=True) nco.ncks(input=foo_nc, output='bar.nc', no_tmp_fl=True) nco.ncks(input=foo_nc, output=foo_nc, no_tmp_fl=True, create_ram=True) nco.ncks(input=foo_nc, output=foo_nc, no_tmp_fl=True, open_ram=True)
def run(self, ds, vars, bbox, start, end, chunk=None): # Split large data sets into chunks if chunk: dates = pd.date_range(start, end, freq=chunk) else: dates = [start, end] print(self.locs['chunks'].filename) self.locs['chunks'] = DatetimeLoc(datetimes=dates, template=self.locs['chunks']) self.locs['chunks'].configure(self.cfg) # Download nco = Nco() for i, (start_date, end_date) in enumerate(zip(dates[:-1], dates[1:])): info = {'year': start_date.year} logging.debug(ds.loc.url.format(**info)) logging.debug(self.locs['chunks'].locs[i].filename) nco.ncks( input=ds.loc.url.format(**info), output=self.locs['chunks'].locs[i].path, options=[ '--mk_rec_dmn time', '-v ' + ','.join(vars), '-d ' + ','.join( ['time', start_date.isoformat(), end_date.isoformat()]), '-d ' + ','.join( ['lon', str(bbox.min.lon), str(bbox.max.lon)]), '-d ' + ','.join( ['lat', str(bbox.min.lat), str(bbox.max.lat)]) ]) # Merge chunks cat_args = [ 'SKIP_SAME_TIME=1', 'cdo', 'mergetime', os.path.join(self.locs['chunks'].dirname, '*'), self.locs['data'].path ] logging.info('Calling process %s', ' '.join(cat_args)) cat_process = subprocess.Popen(cat_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) cat_output, _ = cat_process.communicate()
def preprocess_emep_perturbation(**kwargs): varlist = ['D3_ug_CO', 'PS', 'hyam', 'hybm', 'P0'] species = kwargs['species'] options_string = ['-7', '-O'] # for NETCDF4_CLASSIC output nco = Nco() city = kwargs.get('city') fn_city = os.path.join( kwargs.get('path'), DictFormatter().format(kwargs.get('fnpattern'), city_id='{}_ALL_P15_'.format(city))) if not os.path.isfile(fn_city): raise DataNotAvailable if species == 'co': fn_orig = os.path.join( kwargs.get('path'), DictFormatter().format(kwargs.get('fnpattern'), city_id='')) if not os.path.isfile(fn_orig): raise DataNotAvailable fn_tmp_orig = os.path.join(kwargs.get('path_out'), '{}_orig.nc'.format(city)) fn_tmp_city = os.path.join(kwargs.get('path_out'), '{}_pert.nc'.format(city)) options_string += ['-v' + ','.join(varlist)] nco.ncks(input=fn_orig, output=fn_tmp_orig, options=options_string) nco.ncks(input=fn_city, output=fn_tmp_city, options=options_string) fn_out_tracer = os.path.join(kwargs.get('path_out'), 'emep_pert_{}_co.nc'.format(city)) check_call([ 'ncdiff', '-7', '-O', '-vD3_ug_CO', fn_tmp_orig, fn_tmp_city, fn_out_tracer ]) os.remove(fn_tmp_city) os.remove(fn_tmp_orig) elif species == 'pressure': options_string += ['-v' + ','.join(varlist[1:])] fn_out_pressure = os.path.join(kwargs.get('path_out'), 'emep_pert_{}_pressure.nc'.format(city)) nco.ncks(input=fn_city, output=fn_out_pressure, options=options_string) else: raise ValueError
def test_ncks_append_variables(foo_nc, bar_nc): """ 2.4 Appending Variables The simplest way to create the union of two files is ncks -A fl_1.nc fl_2.nc """ nco = Nco(debug=True) nco.ncks(input=foo_nc, output=bar_nc, options=['-A']) nco.ncks(input=foo_nc, output=bar_nc, append=True) nco.ncks(input=foo_nc, output=bar_nc, apn=True) nco.ncks(input=foo_nc, output=bar_nc)
def __check_soil(self, filename, latidx, lonidx, latdelta, londelta): with nc(filename) as f: lats, lons = f.variables['lat'][:], f.variables['lon'][:] glatdelta, glondelta = abs(diff(lats)[0]), abs(diff(lons)[0]) minlat, maxlat, minlon, maxlon = self.__get_range( latidx, lonidx, latdelta, londelta, glatdelta, glondelta) nco = Nco() options = '-h -d lat,%f,%f -d lon,%f,%f' % (minlat, maxlat, minlon, maxlon) nco.ncks(input=filename, output='tmp.nc4', options=options) with nc('tmp.nc4') as f: slsi = f.variables['slsi'][:] hassoil = slsi.size and (not isMaskedArray(slsi) or not slsi.mask.all()) remove('tmp.nc4') return hassoil
def test_determining_file_format(foo3c, foo364, foo4c, hdf_file): """ 3.9.2 Determining File Format ncks -M foo_3c.nc ncks -M foo_364.nc ncks -M foo_4c.nc ncks -M foo_4.nc ncks -D 2 -M hdf.hdf # ncks -D 2 -M http://thredds-test.ucar.edu/thredds/dodsC/testdods/in.nc ncks -D 2 -M foo_4.nc """ nco = Nco(debug=True) nco.ncks(input=foo3c, options='-M') nco.ncks(input=foo364, options='-M') nco.ncks(input=foo4c, options='-M') assert os.path.isfile(hdf_file) nco.ncks(input=hdf_file, options='-D 2 -M') # nco.ncks(input='http://thredds-test.ucar.edu/thredds/dodsC/testdods/in.nc', # options='-D 2 -M') # does not work from command line either nco.ncks(input=foo4c, options='-D 2 -M')
def test_determining_file_format(foo3c, foo364, foo4c, hdf_file): """ 3.9.2 Determining File Format ncks -M foo_3c.nc ncks -M foo_364.nc ncks -M foo_4c.nc ncks -M foo_4.nc ncks -D 2 -M hdf.hdf # ncks -D 2 -M http://thredds-test.ucar.edu/thredds/dodsC/testdods/in.nc ncks -D 2 -M foo_4.nc """ nco = Nco(debug=True) nco.ncks(input=foo3c, options=['-M']) nco.ncks(input=foo364, options=['-M']) nco.ncks(input=foo4c, options=['-M']) nco.ncks(input=foo4c, options=['-D 2 -M']) if hdf_file is not None: assert os.path.isfile(hdf_file) nco.ncks(input=hdf_file, options=['-D 2 -M'])
def normalize_locations(netcdf_file): with CFDataset(netcdf_file) as nc: y = nc.variables.get("lat") y_data = y[:] y_atts = nc.vatts('lat') x = nc.variables.get("lon") x_data = x[:] x_atts = nc.vatts('lon') # Remove the old x/y variable so we can add new ones with no dimensions o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-x', '-v', 'lat,lon' ] ) # Add the new X/Y variables with nc4.Dataset(netcdf_file, 'a') as nc: lat = nc.createVariable('lat', y_data.dtype) lat[:] = y_data y_atts.update({ 'standard_name': 'latitude', 'axis': 'Y' }) lat.setncatts(y_atts) lon = nc.createVariable('lon', x_data.dtype) lon[:] = x_data x_atts.update({ 'standard_name': 'longitude', 'axis': 'X' }) lon.setncatts(x_atts)
def test_ncks_hdf2nc3(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Obtaining a netCDF3 file from an HDF4 is now easy, even though the HDF4 file may contain netCDF4 atomic types (e.g., unsigned bytes, 64-bit integers): ncks -3 fl.hdf fl.nc # HDF4->netCDF3 (NCO 4.4.0+, netCDF 4.3.1+) ncks -7 -L 1 fl.hdf fl.nc # HDF4->netCDF4 (NCO 4.4.0+, netCDF 4.3.1+) ncks --hdf4 -3 fl.hdf fl.nc # HDF4->netCDF3 (netCDF 4.3.0-) ncks --hdf4 -7 fl.hdf fl.nc # HDF4->netCDF4 classic (netCDF 4.3.0-) """ nco = Nco(debug=True) nco.ncks(input=hdf_file, output='foo.nc', options='-3') nco.ncks(input=hdf_file, output='foo.nc', options='-7 -L 1') nco.ncks(input=hdf_file, output='foo.nc', options='-3', hdf4=True) nco.ncks(input=hdf_file, output='foo.nc', options='-7', hdf4=True)
def test_ncks_hdf2nc3(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Obtaining a netCDF3 file from an HDF4 is now easy, even though the HDF4 file may contain netCDF4 atomic types (e.g., unsigned bytes, 64-bit integers): ncks -3 fl.hdf fl.nc # HDF4->netCDF3 (NCO 4.4.0+, netCDF 4.3.1+) ncks -7 -L 1 fl.hdf fl.nc # HDF4->netCDF4 (NCO 4.4.0+, netCDF 4.3.1+) ncks --hdf4 -3 fl.hdf fl.nc # HDF4->netCDF3 (netCDF 4.3.0-) ncks --hdf4 -7 fl.hdf fl.nc # HDF4->netCDF4 classic (netCDF 4.3.0-) """ if hdf_file is None: pytest.skip("Skipped because h5py is not installed") nco = Nco(debug=True) nco.ncks(input=hdf_file, output="foo.nc", options=["-3"]) nco.ncks(input=hdf_file, output="foo.nc", options=["-7 -L 1"]) nco.ncks(input=hdf_file, output="foo.nc", options=["-3"], hdf4=True) nco.ncks(input=hdf_file, output="foo.nc", options=["-7"], hdf4=True)
def test_hyperslabs(testfileglobal): """ 3.15 Hyperslabs # First and second indices of lon dimension ncks -F -d lon,1,2 in.nc out.nc # Second and third indices of lon dimension ncks -d lon,1,2 in.nc out.nc # All longitude values between 1 and 2 degrees ncks -d lon,1.0,2.0 in.nc out.nc # All longitude values between 1 and 2 degrees ncks -F -d lon,1.0,2.0 in.nc out.nc # Every other longitude value between 0 and 90 degrees ncks -F -d lon,0.0,90.0,2 in.nc out.nc # Last two indices of lon dimension ncks -F -d lon,1,-2 in.nc out.nc # Third-to-last to last index of lon dimension ncks -F -d lon,-3,-1 in.nc out.nc # Third-to-last to last index of lon dimension ncks -F -d lon,-3, in.nc out.nc """ nco = Nco(debug=True) nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,1,2') nco.ncks(input=testfileglobal, output='out.nc', dimension='lon,1,2') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,1.0,2.0') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,0.0,90.0,2') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,1,-2') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,-3,-1') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,-3')
def test_file_conversion(foo3c, foo4c): """ 3.9.2 Determining File Format ncks --fl_fmt=classic in.nc foo_3c.nc # netCDF3 classic ncks --fl_fmt=64bit in.nc foo_364.nc # netCDF3 64bit ncks --fl_fmt=netcdf4_classic in.nc foo_4c.nc # netCDF4 classic ncks --fl_fmt=netcdf4 in.nc foo_4.nc # netCDF4 ncks -3 in.nc foo_3c.nc # netCDF3 classic ncks --3 in.nc foo_3c.nc # netCDF3 classic ncks -6 in.nc foo_364.nc # netCDF3 64bit ncks --64 in.nc foo_364.nc # netCDF3 64bit ncks -4 in.nc foo_4.nc # netCDF4 ncks --4 in.nc foo_4.nc # netCDF4 ncks -7 in.nc foo_4c.nc # netCDF4 classic ncks --7 in.nc foo_4c.nc # netCDF4 classic """ nco = Nco(debug=True) nco.ncks(input=foo4c, output='foo_3c.nc', fl_fmt='classic') nco.ncks(input=foo4c, output='foo_364.nc', fl_fmt='64bit') nco.ncks(input=foo3c, output='foo_4c.nc', fl_fmt='netcdf4_classic') nco.ncks(input=foo3c, output='foo_4.nc', fl_fmt='netcdf4') nco.ncks(input=foo4c, output='foo_3c.nc', options=['-3']) nco.ncks(input=foo4c, output='foo_3c.nc', options=['--3']) nco.ncks(input=foo3c, output='foo_364c.nc', options=['-6']) nco.ncks(input=foo3c, output='foo_364c.nc', options=['--64bit_offset']) nco.ncks(input=foo3c, output='foo_4.nc', options=['-4']) nco.ncks(input=foo3c, output='foo_4.nc', options=['--4']) nco.ncks(input=foo3c, output='foo_4c.nc', options=['-7']) nco.ncks(input=foo3c, output='foo_4c.nc', options=['--7'])
def test_operator_prints_out(bar_nc): nco = Nco() dump = nco.ncks(input=bar_nc, options=["--help"]) print(dump)
def test_error_exception(): nco = Nco() assert hasattr(nco, "nonExistingMethod") is False with pytest.raises(NCOException): nco.ncks(input="", output="")
def test_ncks_extract(foo_nc): nco = Nco(debug=True) infile = foo_nc nco.ncks(input=infile, output="out.nc", variable="random")
def normalize_depth_locations(netcdf_file): redimension = [] with CFDataset(netcdf_file, 'a') as nc: y = nc.variables.get("lat") y_data = y[:] y_atts = nc.vatts('lat') x = nc.variables.get("lon") x_data = x[:] x_atts = nc.vatts('lon') # Get list of variables to re-write with different dimensions for vname, ov in nc.variables.items(): if 'lat' in ov.dimensions and 'lon' in ov.dimensions: redimension.append(vname) for vname in redimension: ov = nc.variables[vname] if 'depth' in ov.dimensions: vdata = ov[:, :, 0, 0] dims = ('time', 'depth') else: vdata = ov[:, 0, 0] dims = ('time',) vatts = nc.vatts(vname) nc.renameVariable(vname, '{}_old'.format(vname)) nc.sync() v = nc.createVariable(vname, vdata.dtype, dims) v.setncatts(vatts) v[:] = vdata # Remove the old variables remove_vars = [ '{}_old'.format(vname) for vname in redimension ] remove_vars += ['lat', 'lon'] o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-x', '-v', ','.join(remove_vars) ] ) # Add the new X/Y variables with nc4.Dataset(netcdf_file, 'a') as nc: lat = nc.createVariable('lat', y_data.dtype) lat[:] = y_data y_atts.update({ 'standard_name': 'latitude', 'axis': 'Y' }) lat.setncatts(y_atts) lon = nc.createVariable('lon', x_data.dtype) lon[:] = x_data x_atts.update({ 'standard_name': 'longitude', 'axis': 'X' }) lon.setncatts(x_atts)
def test_file_conversion(foo3c, foo4c): """ 3.9.2 Determining File Format ncks --fl_fmt=classic in.nc foo_3c.nc # netCDF3 classic ncks --fl_fmt=64bit in.nc foo_364.nc # netCDF3 64bit ncks --fl_fmt=netcdf4_classic in.nc foo_4c.nc # netCDF4 classic ncks --fl_fmt=netcdf4 in.nc foo_4.nc # netCDF4 ncks -3 in.nc foo_3c.nc # netCDF3 classic ncks --3 in.nc foo_3c.nc # netCDF3 classic ncks -6 in.nc foo_364.nc # netCDF3 64bit ncks --64 in.nc foo_364.nc # netCDF3 64bit ncks -4 in.nc foo_4.nc # netCDF4 ncks --4 in.nc foo_4.nc # netCDF4 ncks -7 in.nc foo_4c.nc # netCDF4 classic ncks --7 in.nc foo_4c.nc # netCDF4 classic """ nco = Nco(debug=True) nco.ncks(input=foo4c, output="foo_3c.nc", fl_fmt="classic") nco.ncks(input=foo4c, output="foo_364.nc", fl_fmt="64bit") nco.ncks(input=foo3c, output="foo_4c.nc", fl_fmt="netcdf4_classic") nco.ncks(input=foo3c, output="foo_4.nc", fl_fmt="netcdf4") nco.ncks(input=foo4c, output="foo_3c.nc", options=["-3"]) nco.ncks(input=foo4c, output="foo_3c.nc", options=["--3"]) nco.ncks(input=foo3c, output="foo_364c.nc", options=["-6"]) nco.ncks(input=foo3c, output="foo_364c.nc", options=["--64bit_offset"]) nco.ncks(input=foo3c, output="foo_4.nc", options=["-4"]) nco.ncks(input=foo3c, output="foo_4.nc", options=["--4"]) nco.ncks(input=foo3c, output="foo_4c.nc", options=["-7"]) nco.ncks(input=foo3c, output="foo_4c.nc", options=["--7"])
def test_ncks_extract(foo_nc): nco = Nco(debug=True) infile = foo_nc nco.ncks(input=infile, output='out.nc', variable='random')