def test_returnMaArray(bar_mask_nc, random_masked_field): nco = Nco() field = nco.ncea(input=bar_mask_nc, output="tmp.nc", returnMaArray='random', options=['-O']) assert type(field) == np.ma.core.MaskedArray
def test_returnArray(foo_nc): nco = Nco(cdfMod='netcdf4') random1 = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=['-O']).variables['random'][:] assert type(random1) == np.ndarray random2 = nco.ncea(input=foo_nc, output="tmp.nc",returnArray='random' ,options=['-O']) assert type(random2) == np.ndarray np.testing.assert_equal(random1, random2)
def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None): """ Combine many files into a single file on disk. Defaults to using the 'time' dimension. """ nco = None try: nco = Nco() except BaseException: # This is not necessarily an import error (could be wrong PATH) raise ImportError("NCO not found. The NCO python bindings are required to use 'Collection.combine'.") if len(members) > 0 and hasattr(members[0], 'path'): # A member DotDoct was passed in, we only need the paths members = [ m.path for m in members ] options = ['-4'] # NetCDF4 options += ['-L', '3'] # Level 3 compression options += ['-h'] # Don't append to the history global attribute if dimension is not None: if start_index is None: start_index = 0 if stop_index is None: stop_index = '' if stride is None: stride = 1 options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)] nco.ncrcat(input=members, output=output_file, options=options)
def normalize_time(netcdf_file): epoch_units = 'seconds since 1970-01-01T00:00:00Z' millisecond_units = 'milliseconds since 1858-11-17T00:00:00Z' with nc4.Dataset(netcdf_file, 'a') as nc: # Signell said this works, any problems and we can all blame him! time_data = nc4.num2date( ( np.int64(nc.variables['time'][:]) - 2400001 ) * 3600 * 24 * 1000 + nc.variables['time2'][:].__array__(), units=millisecond_units ) nc.renameVariable("time", "old_time") nc.sync() time = nc.createVariable('time', 'f8', ('time')) time.units = epoch_units time.standard_name = "time" time.long_name = "time of measurement" time.calendar = "gregorian" time.axis = "T" time[:] = nc4.date2num(time_data, units=epoch_units).round() o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-x', '-v', 'time2,old_time' ] )
def test_use_list_options(foo_nc): nco = Nco(debug=True) options = [] options.extend(['-a', 'units,time,o,c,"days since 1999-01-01"']) options.extend(['-a', 'long_name,time,o,c,time']) options.extend(['-a', 'calendar,time,o,c,noleap']) nco.ncatted(input=foo_nc, output='out.nc', options=options)
def test_use_list_options(foo_nc): nco = Nco(debug=True) options = [] options.extend(["-a", 'units,time,o,c,"days since 1999-01-01"']) options.extend(["-a", "long_name,time,o,c,time"]) options.extend(["-a", "calendar,time,o,c,noleap"]) nco.ncatted(input=foo_nc, output="out.nc", options=options)
def __check_weather(self, filename, latidx, lonidx, latdelta, londelta): with Dataset(filename) as f: lats = f.variables['lat'][:] lons = f.variables['lon'][:] glatdelta = abs(diff(lats)[0]) glondelta = abs(diff(lons)[0]) minlat, maxlat, minlon, maxlon = self.__get_range( latidx, lonidx, latdelta, londelta, glatdelta, glondelta) # select first time nco = Nco() options = '-h -d time,0 -d lat,%f,%f -d lon,%f,%f' % (minlat, maxlat, minlon, maxlon) tmpfile = "checker.wth.nc4" nco.ncks(input=filename, output=tmpfile, options=options) with Dataset(tmpfile) as f: varnames = f.variables.keys() prvars = ['pr', 'prcp', 'pre', 'precip'] found = False for i in range(len(prvars)): patt = '%s$|%s.*' % (prvars[i], prvars[i]) for j in range(len(varnames)): if compile(patt).match(varnames[j]): pr = f.variables[varnames[j]][:] found = True break if found: break if not found: pr = array([]) hasweather = pr.size and (not isMaskedArray(pr) or not pr.mask.all()) remove(tmpfile) return hasweather
def normalize_locations(netcdf_file): with CFDataset(netcdf_file) as nc: y = nc.variables.get("lat") y_data = y[:] y_atts = nc.vatts('lat') x = nc.variables.get("lon") x_data = x[:] x_atts = nc.vatts('lon') # Remove the old x/y variable so we can add new ones with no dimensions o = Nco() o.ncks(input=netcdf_file, output=netcdf_file, options=['-O', '-h', '-x', '-v', 'lat,lon']) # Add the new X/Y variables with nc4.Dataset(netcdf_file, 'a') as nc: lat = nc.createVariable('lat', y_data.dtype) lat[:] = y_data y_atts.update({'standard_name': 'latitude', 'axis': 'Y'}) lat.setncatts(y_atts) lon = nc.createVariable('lon', x_data.dtype) lon[:] = x_data x_atts.update({'standard_name': 'longitude', 'axis': 'X'}) lon.setncatts(x_atts)
def test_use_list_options(foo_nc): nco = Nco(debug=True) options = [] options.extend(['-a', 'units,time,o,c,days since 1999-01-01']) options.extend(['-a', 'long_name,time,o,c,time']) options.extend(['-a', 'calendar,time,o,c,noleap']) nco.ncrcat(input=foo_nc, output='out.nc', options=options)
def normalize_time(netcdf_file): epoch_units = 'seconds since 1970-01-01T00:00:00Z' millisecond_units = 'milliseconds since 1858-11-17T00:00:00Z' with nc4.Dataset(netcdf_file, 'a') as nc: # Signell said this works, any problems and we can all blame him! time_data = nc4.num2date( (np.int64(nc.variables['time'][:]) - 2400001) * 3600 * 24 * 1000 + nc.variables['time2'][:].__array__(), units=millisecond_units) nc.renameVariable("time", "old_time") nc.sync() time = nc.createVariable('time', 'f8', ('time')) time.units = epoch_units time.standard_name = "time" time.long_name = "time of measurement" time.calendar = "gregorian" time.axis = "T" time[:] = nc4.date2num(time_data, units=epoch_units).round() o = Nco() o.ncks(input=netcdf_file, output=netcdf_file, options=['-O', '-h', '-x', '-v', 'time2,old_time'])
def normalize_ctd_depths(netcdf_file): with CFDataset(netcdf_file, 'a') as nc: depths = nc.variables['depth'][:][0] z_atts = nc.vatts('depth') # Remove the old depth variable so we can add a new one with no dimensions o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-x', '-v', 'depth' ] ) # Add the new Z variable with nc4.Dataset(netcdf_file, 'a') as nc: z = nc.createVariable('depth', 'f4') z[:] = depths z_atts.update({ 'standard_name': 'depth', 'axis': 'Z', 'positive': 'down' }) z.setncatts(z_atts)
def netCDF_merge(self, directory): """ To combine mutiple downloaded erai netCDF files into a large file with specified chunk_size(e.g. 500), -- give the full name of merged file to the output = outfile -- pass all data from the first input netfile to the merged file name -- loop over the files_list, append file one by one into the merge file -- pass the mergae netcdf file to interpolation module to process (to use nc.MFDataset by reading it) Args: ncfile_in: the full name of downloaded files (file directory + files names) e.g.: '/home/xquan/src/globsim/examples/erai/era_sa_*.nc' '/home/xquan/src/globsim/examples/erai/era_pl_*.nc' '/home/xquan/src/globsim/examples/erai/era_sf_*.nc' Output: merged netCDF files era_all_0.nc, era_all_1.nc, ..., """ # set up nco operator nco = Nco() # loop over filetypes, read, report file_type = ['erai_sa_*.nc', 'erai_sf_*.nc', 'erai_pl_*.nc'] for ft in file_type: ncfile_in = path.join(directory, ft) # get the file list files_list = glob.glob(ncfile_in) files_list.sort() num = len(files_list) # set up the name of merged file if ncfile_in[-7:-5] == 'sa': merged_file = path.join( ncfile_in[:-11], 'erai_sa_all_' + files_list[0][-23:-15] + "_" + files_list[num - 1][-11:-3] + '.nc') elif ncfile_in[-7:-5] == 'sf': merged_file = path.join( ncfile_in[:-11], 'erai_sf_all_' + files_list[0][-23:-15] + '_' + files_list[num - 1][-11:-3] + '.nc') elif ncfile_in[-7:-5] == 'pl': merged_file = path.join( ncfile_in[:-11], 'erai_pl_all_' + files_list[0][-23:-15] + '_' + files_list[num - 1][-11:-3] + '.nc') else: print('There is not such type of file') # combined files into merged files nco.ncrcat(input=files_list, output=merged_file, append=True) print('The Merged File below is saved:') print(merged_file) # clear up the data for fl in files_list: remove(fl)
def convert_to_netcdf3(input_file, output_file=None): nco = Nco() output_file = output_file or input_file tmp_file = input_file[:-3] + "-tmp.nc" nco.ncks(input=input_file, output=tmp_file, options=['-3']) shutil.move(tmp_file, output_file) LOGGER.info("Converted to NetCDF3 file: {}".format(output_file)) return output_file
def test_init_options(): nco = Nco(debug=True) assert nco.debug nco = Nco(force_output=False) assert nco.force_output is False nco = Nco(returnCdf=True, return_none_on_error=True) assert nco.return_cdf assert nco.return_none_on_error
def test_initOptions(): nco = Nco(debug=True) assert nco.debug nco = Nco(forceOutput=False) assert nco.forceOutput is False nco = Nco(returnCdf=True, returnNoneOnError=True) assert nco.returnCdf assert nco.returnNoneOnError
def normalize_depth_locations(netcdf_file): redimension = [] with CFDataset(netcdf_file, 'a') as nc: y = nc.variables.get("lat") y_data = y[:] y_atts = nc.vatts('lat') x = nc.variables.get("lon") x_data = x[:] x_atts = nc.vatts('lon') # Get list of variables to re-write with different dimensions for vname, ov in nc.variables.items(): if 'lat' in ov.dimensions and 'lon' in ov.dimensions: redimension.append(vname) for vname in redimension: ov = nc.variables[vname] if 'depth' in ov.dimensions: vdata = ov[:, :, 0, 0] dims = ('time', 'depth') else: vdata = ov[:, 0, 0] dims = ('time', ) vatts = nc.vatts(vname) nc.renameVariable(vname, '{}_old'.format(vname)) nc.sync() v = nc.createVariable(vname, vdata.dtype, dims) v.setncatts(vatts) v[:] = vdata # Remove the old variables remove_vars = ['{}_old'.format(vname) for vname in redimension] remove_vars += ['lat', 'lon'] o = Nco() o.ncks(input=netcdf_file, output=netcdf_file, options=['-O', '-h', '-x', '-v', ','.join(remove_vars)]) # Add the new X/Y variables with nc4.Dataset(netcdf_file, 'a') as nc: lat = nc.createVariable('lat', y_data.dtype) lat[:] = y_data y_atts.update({'standard_name': 'latitude', 'axis': 'Y'}) lat.setncatts(y_atts) lon = nc.createVariable('lon', x_data.dtype) lon[:] = x_data x_atts.update({'standard_name': 'longitude', 'axis': 'X'}) lon.setncatts(x_atts)
def ocean_atlas_merge_variables(output_dir, month): ocean_atlas_files = output_dir.glob(f'ocean_atlas_*_{month.month:02}.nc') ocean_atlas_files = list(ocean_atlas_files) ocean_atlas_files.sort() outfile = output_dir / f'ocean_atlas_{month.month:02}.nc' nco = Nco() options = ['-A', '-4', '-L 5'] outfile = shutil.copy(ocean_atlas_files[0], outfile) for f in ocean_atlas_files[1::]: nco.ncks(input=str(f), output=str(outfile), options=options)
def test_return_array(foo_nc): nco = Nco(cdf_module="netcdf4") random1 = nco.ncea( input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"] ).variables["random"][:] assert isinstance(random1, np.ndarray) random2 = nco.ncea( input=foo_nc, output="tmp.nc", returnArray="random", options=["-O"] ) assert isinstance(random2, np.ndarray) np.testing.assert_equal(random1, random2)
def narr_merge_variables(output_dir): narr_files = output_dir.glob('narr_*.nc') narr_files = list(narr_files) narr_files.sort() outfile = output_dir.parent / 'narr.nc' nco = Nco() options = ['-A', '-4', '-L 5'] outfile = shutil.copy(narr_files[0], outfile) for f in narr_files[1::]: nco.ncks(input=str(f), output=str(outfile), options=options)
def normalize_netcdf4(netcdf_file): o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-4', '-L3' ] )
def test_ncks_hdf2nc(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively, it is now easy to convert HDF4 files to netCDF files directly, e.g., ncks fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+) ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9) """ nco = Nco(debug=True) nco.ncks(input=hdf_file, output='foo.nc') nco.ncks(input=hdf_file, output='foo.nc', hdf4=True)
def test_ncks_append_variables(foo_nc, bar_nc): """ 2.4 Appending Variables The simplest way to create the union of two files is ncks -A fl_1.nc fl_2.nc """ nco = Nco(debug=True) nco.ncks(input=foo_nc, output=bar_nc, options=["-A"]) nco.ncks(input=foo_nc, output=bar_nc, append=True) nco.ncks(input=foo_nc, output=bar_nc, apn=True) nco.ncks(input=foo_nc, output=bar_nc)
def test_command_line_options(foo_nc): """ 3.4 Command Line Options ncks -D 3 in.nc # Short option ncks --dbg_lvl=3 in.nc # Long option, preferred form ncks --dbg_lvl 3 in.nc # Long option, alternate form """ nco = Nco(debug=True) nco.ncks(input=foo_nc, options=["-O -D 3"]) nco.ncks(input=foo_nc, options=["-O --dbg_lvl=3"]) nco.ncks(input=foo_nc, options=["-O --dbg_lvl 3"])
def test_command_line_options(foo_nc): """ 3.4 Command Line Options ncks -D 3 in.nc # Short option ncks --dbg_lvl=3 in.nc # Long option, preferred form ncks --dbg_lvl 3 in.nc # Long option, alternate form """ nco = Nco(debug=True) test = nco.ncks(input=foo_nc, options='-D 3') test = nco.ncks(input=foo_nc, options='--dbg_lvl=3') test = nco.ncks(input=foo_nc, options='--dbg_lvl 3') test = nco.ncks(input=foo_nc, dbg_lvl=3)
def test_return_cdf(foo_nc): nco = Nco(cdf_module="scipy") test_cdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"]) assert type(test_cdf) == scipy.io.netcdf.netcdf_file expected_vars = ["time", "random"] for var in expected_vars: assert var in list(test_cdf.variables.keys()) nco = Nco(cdf_module="netcdf4") test_cdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"]) assert type(test_cdf) == netCDF4.Dataset for var in expected_vars: assert var in list(test_cdf.variables.keys())
def test_returnCdf(foo_nc): nco = Nco(cdfMod='scipy') testCdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True,options=['-O']) assert type(testCdf) == scipy.io.netcdf.netcdf_file expected_vars = ['time', 'random'] for var in expected_vars: assert var in list(testCdf.variables.keys()) nco = Nco(cdfMod='netcdf4') testCdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=['-O']) assert type(testCdf) == netCDF4.Dataset for var in expected_vars: assert var in list(testCdf.variables.keys())
def ocean_atlas_merge_time(output_dir): variable_merged_files = output_dir.glob('ocean_atlas_??.nc') variable_merged_files = [ str(merged_file) for merged_file in list(variable_merged_files) ] variable_merged_files.sort() output_file = output_dir.parent / 'ocean_atlas.nc' nco = Nco() options = ['-A'] nco.ncrcat(input=variable_merged_files, output=str(output_file), options=options)
def test_returnArray(foo_nc): nco = Nco(cdfMod='netcdf4') random1 = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=['-O']).variables['random'][:] assert type(random1) == np.ndarray random2 = nco.ncea(input=foo_nc, output="tmp.nc", returnArray='random', options=['-O']) assert type(random2) == np.ndarray np.testing.assert_equal(random1, random2)
def test_ncks_hdf2nc(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively, it is now easy to convert HDF4 files to netCDF files directly, e.g., ncks fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+) ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9) """ if hdf_file is None: pytest.skip("Skipped because h5py is not installed") nco = Nco(debug=True) nco.ncks(input=hdf_file, output="foo.nc") nco.ncks(input=hdf_file, output="foo.nc", hdf4=True)
def narr_enhance(output_dir): outfile = output_dir.parent / 'narr.nc' outtmp = output_dir.parent / 'narr_tmp.nc' # remove unecessary vars nco = Nco() options = ['-O', '-C', '-x', '-v climatology_bounds,valid_yr_count'] nco.ncks(input=str(outfile), output=str(outfile), options=options) # change lon from [0, 360) to [-180, 180) options = [ '-O', '--msa', '-d lon,181.,360.', '-d lon,0.,180.0', ] nco.ncks(input=str(outfile), output=str(outtmp), options=options) options = ['-O', "-s 'where(lon > 180) lon=lon-360'"] nco.ncap2(input=str(outtmp), output=str(outfile), options=options) # times don't work with xarray, so the year is changed to the middle of climatology # climatology: 1981-01-01 - 2010-12-31 # midyear is 1996 time_range = xr.cftime_range(start='1996', end='1996-12-31', freq='MS') new_units = 'days since 1970-01-01 00:00:00' new_times = nc.date2num(time_range, units=new_units) with nc.Dataset(outfile, 'a') as ncd: time = ncd.variables['time'] time[:] = new_times time.units = new_units
def convert_dl_to_nc4c(self, fns): self.log.debug('Converting to NETCDF4_CLASSIC ...') if not _NCO: self.log.error( 'Cannot convert to NETCDF4_CLASSIC: nco is not installed') raise ImportError('cannot import nco library') nco = Nco() fns_new = [] for fn in fns: fn_new = tempfile.mktemp(suffix='.nc', dir=self.cfg['temppath']) nco.ncks(input=fn, output=fn_new, options=['-O', '-7']) fns_new.append(fn_new) self.cfg['dldriver'].clean_tempfiles([fn]) self.log.debug('... done.') return fns_new
def test_ncks_append_variables(foo_nc, bar_nc): """ 2.4 Appending Variables The simplest way to create the union of two files is ncks -A fl_1.nc fl_2.nc """ nco = Nco(debug=True) nco.ncks(input=foo_nc, output=bar_nc, options=['-A']) nco.ncks(input=foo_nc, output=bar_nc, append=True) nco.ncks(input=foo_nc, output=bar_nc, apn=True) nco.ncks(input=foo_nc, output=bar_nc)
def test_specifying_input_files(testfiles8589): """ 3.5 Specifying Input Files ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra 8[56789].nc 8589.nc ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra -n 5,2,1 85.nc 8589.nc """ inputs = ['85.nc', '86.nc', '87.nc', '88.nc', '89.nc'] nco = Nco(debug=True) nco.ncra(input=inputs, ouptut='8589.nc') nco.ncra(input='8[56789].nc', ouptut='8589.nc') srcdir = os.path.split(inputs[0]) nco.ncra(input=inputs, ouptut='8589.nc', path=srcdir) nco.ncra(input=inputs, ouptut='8589.nc', nintap='5,2,1')
def xgeo_multifile_load(nc_wildcard, nc_dir=None): """ Read data from multiple (e.g. hourly xgeo data) netcdf files :param nc_wildcard: common section of the filenams, e.g. mf_files*.nc :return: """ if dir: nc_path = os.path.join(nc_dir, nc_wildcard) else: nc_path = nc_wildcard nco = Nco() nc_temp = nco.ncrcat(input=nc_path) nc = netCDF4.Dataset(nc_temp) #add function to restrict dates and times return nc
def run(self, ds, vars, bbox, start, end, chunk=None): # Split large data sets into chunks if chunk: dates = pd.date_range(start, end, freq=chunk) else: dates = [start, end] print(self.locs['chunks'].filename) self.locs['chunks'] = DatetimeLoc(datetimes=dates, template=self.locs['chunks']) self.locs['chunks'].configure(self.cfg) # Download nco = Nco() for i, (start_date, end_date) in enumerate(zip(dates[:-1], dates[1:])): info = {'year': start_date.year} logging.debug(ds.loc.url.format(**info)) logging.debug(self.locs['chunks'].locs[i].filename) nco.ncks( input=ds.loc.url.format(**info), output=self.locs['chunks'].locs[i].path, options=[ '--mk_rec_dmn time', '-v ' + ','.join(vars), '-d ' + ','.join( ['time', start_date.isoformat(), end_date.isoformat()]), '-d ' + ','.join( ['lon', str(bbox.min.lon), str(bbox.max.lon)]), '-d ' + ','.join( ['lat', str(bbox.min.lat), str(bbox.max.lat)]) ]) # Merge chunks cat_args = [ 'SKIP_SAME_TIME=1', 'cdo', 'mergetime', os.path.join(self.locs['chunks'].dirname, '*'), self.locs['data'].path ] logging.info('Calling process %s', ' '.join(cat_args)) cat_process = subprocess.Popen(cat_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) cat_output, _ = cat_process.communicate()
def remove_variables(self): ########################################################################### if len(self._rvars) > 0: Nco().ncks(str(self._ofile), output=str(self._ofile), exclude=True, force=True, variable=",".join(self._rvars))
def test_specifying_input_files(testfiles8589): """ 3.5 Specifying Input Files ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra 8[56789].nc 8589.nc ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra -n 5,2,1 85.nc 8589.nc """ nco = Nco(debug=True) nco.ncra(input=testfiles8589, output='8589.nc') nco.ncra(input=testfiles8589, output='8589.nc', nintap='5,2,1') srcdir = os.path.dirname(testfiles8589[0]) basenames = [os.path.basename(x) for x in testfiles8589] nco.ncra(input=basenames, output='8589.nc', path=srcdir) regpath = os.path.join(srcdir, '8[56789].nc') nco.ncra(input=regpath, output='8589.nc')
def normalize_summary_depths(netcdf_file): redimension = [] with CFDataset(netcdf_file, 'a') as nc: nc.renameDimension('depth', 'z') z_atts = nc.vatts('depth') # Get list of variables to re-write with different dimensions for vname, ov in nc.variables.items(): if 'depth' in ov.dimensions: redimension.append(vname) for vname in redimension: ov = nc.variables[vname] vatts = nc.vatts(vname) nc.renameVariable(vname, '{}_old'.format(vname)) nc.sync() v = nc.createVariable(vname, ov.dtype, ('time', )) v.setncatts(vatts) v[:] = ov[:, 0] # Remove the old variables remove_vars = ['{}_old'.format(vname) for vname in redimension] remove_vars += ['depth'] o = Nco() o.ncks(input=netcdf_file, output=netcdf_file, options=['-O', '-h', '-x', '-v', ','.join(remove_vars)]) # Add the new Z variable with nc4.Dataset(netcdf_file, 'a') as nc: z = nc.createVariable('depth', 'f4') z[:] = None z_atts.update({ 'standard_name': 'depth', 'axis': 'Z', 'positive': 'down' }) z.setncatts(z_atts)
def __check_soil(self, filename, latidx, lonidx, latdelta, londelta): with nc(filename) as f: lats, lons = f.variables['lat'][:], f.variables['lon'][:] glatdelta, glondelta = abs(diff(lats)[0]), abs(diff(lons)[0]) minlat, maxlat, minlon, maxlon = self.__get_range( latidx, lonidx, latdelta, londelta, glatdelta, glondelta) nco = Nco() options = '-h -d lat,%f,%f -d lon,%f,%f' % (minlat, maxlat, minlon, maxlon) nco.ncks(input=filename, output='tmp.nc4', options=options) with nc('tmp.nc4') as f: slsi = f.variables['slsi'][:] hassoil = slsi.size and (not isMaskedArray(slsi) or not slsi.mask.all()) remove('tmp.nc4') return hassoil
def test_specifying_input_files(testfiles8589): """ 3.5 Specifying Input Files ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra 8[56789].nc 8589.nc ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra -n 5,2,1 85.nc 8589.nc """ nco = Nco(debug=True) nco.ncra(input=testfiles8589, output="8589.nc") nco.ncra(input=testfiles8589, output="8589.nc", nintap="5,2,1") srcdir = os.path.dirname(testfiles8589[0]) basenames = [os.path.basename(x) for x in testfiles8589] nco.ncra(input=basenames, output="8589.nc", path=srcdir) # unable to use brackets, perhaps because we're no longer using shell=True when calling subprocess()? regpath = os.path.join(srcdir, "8[56789].nc") nco.ncra(input=regpath, output="8589.nc", use_shell=True)
def normalize_ctd_depths(netcdf_file): with CFDataset(netcdf_file, 'a') as nc: depths = nc.variables['depth'][:][0] z_atts = nc.vatts('depth') # Remove the old depth variable so we can add a new one with no dimensions o = Nco() o.ncks(input=netcdf_file, output=netcdf_file, options=['-O', '-h', '-x', '-v', 'depth']) # Add the new Z variable with nc4.Dataset(netcdf_file, 'a') as nc: z = nc.createVariable('depth', 'f4') z[:] = depths z_atts.update({ 'standard_name': 'depth', 'axis': 'Z', 'positive': 'down' }) z.setncatts(z_atts)
def normalize_locations(netcdf_file): with CFDataset(netcdf_file) as nc: y = nc.variables.get("lat") y_data = y[:] y_atts = nc.vatts('lat') x = nc.variables.get("lon") x_data = x[:] x_atts = nc.vatts('lon') # Remove the old x/y variable so we can add new ones with no dimensions o = Nco() o.ncks( input=netcdf_file, output=netcdf_file, options=[ '-O', '-h', '-x', '-v', 'lat,lon' ] ) # Add the new X/Y variables with nc4.Dataset(netcdf_file, 'a') as nc: lat = nc.createVariable('lat', y_data.dtype) lat[:] = y_data y_atts.update({ 'standard_name': 'latitude', 'axis': 'Y' }) lat.setncatts(y_atts) lon = nc.createVariable('lon', x_data.dtype) lon[:] = x_data x_atts.update({ 'standard_name': 'longitude', 'axis': 'X' }) lon.setncatts(x_atts)
def test_specifying_input_files(testfiles8589): """ 3.5 Specifying Input Files ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra 8[56789].nc 8589.nc ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra -n 5,2,1 85.nc 8589.nc """ nco = Nco(debug=True) nco.ncra(input=testfiles8589, output='8589.nc') nco.ncra(input=testfiles8589, output='8589.nc', nintap='5,2,1') srcdir = os.path.dirname(testfiles8589[0]) basenames = [ os.path.basename(x) for x in testfiles8589 ] nco.ncra(input=basenames, output='8589.nc', path=srcdir) regpath = os.path.join(srcdir, '8[56789].nc') nco.ncra(input=regpath, output='8589.nc')
def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None): """ Combine many files into a single file on disk. Defaults to using the 'time' dimension. """ nco = None try: nco = Nco() except BaseException: raise ImportError("NCO not found. The NCO python bindings are required to use 'Collection.combine'.") if len(members) > 0 and hasattr(members[0], 'path'): # A member DotDoct was passed in, we only need the paths members = [ m.path for m in members ] options = ['-4'] # NetCDF4 options += ['-L', '3'] # Level 3 compression options += ['-h'] # Don't append to the history global attribute if dimension is not None: if start_index is None: start_index = 0 if stop_index is None: stop_index = '' if stride is None: stride = 1 options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)] nco.ncrcat(input=members, output=output_file, options=options)
def dailyAve(): from nco import Nco import datetime nco = Nco() for d in range(365): dp = datetime.date(startY,1,1)+datetime.timedelta(d) print "Averaging TRMM 3B42 for day "+dp.strftime('%j')+"..." ifile = ' '.join("3B42_daily."+str(year)+"."+dp.strftime('%m')+"."+dp.strftime('%d')+".7.nc" for year in range(startY,endY)) ofile = "3B42_aver."+dp.strftime('%j')+".nc" if not os.path.isfile(ofile): nco.ncra(input=ifile, output=ofile) nco.ncrcat(input="3B42_aver.*.nc", output="3B42_cat.nc", options="-d time,1,365") nco.ncwa(input="3B42_cat.nc", output="3B42_MAP.nc", options='-N -a time') return None
def test_ncks_hdf2nc3(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Obtaining a netCDF3 file from an HDF4 is now easy, even though the HDF4 file may contain netCDF4 atomic types (e.g., unsigned bytes, 64-bit integers): ncks -3 fl.hdf fl.nc # HDF4->netCDF3 (NCO 4.4.0+, netCDF 4.3.1+) ncks -7 -L 1 fl.hdf fl.nc # HDF4->netCDF4 (NCO 4.4.0+, netCDF 4.3.1+) ncks --hdf4 -3 fl.hdf fl.nc # HDF4->netCDF3 (netCDF 4.3.0-) ncks --hdf4 -7 fl.hdf fl.nc # HDF4->netCDF4 classic (netCDF 4.3.0-) """ nco = Nco(debug=True) nco.ncks(input=hdf_file, output='foo.nc', options='-3') nco.ncks(input=hdf_file, output='foo.nc', options='-7 -L 1') nco.ncks(input=hdf_file, output='foo.nc', options='-3', hdf4=True) nco.ncks(input=hdf_file, output='foo.nc', options='-7', hdf4=True)
def test_ncks_hdf2nc3(hdf_file): """ 1.6 netCDF2/3/4 and HDF4/5 Support Obtaining a netCDF3 file from an HDF4 is now easy, even though the HDF4 file may contain netCDF4 atomic types (e.g., unsigned bytes, 64-bit integers): ncks -3 fl.hdf fl.nc # HDF4->netCDF3 (NCO 4.4.0+, netCDF 4.3.1+) ncks -7 -L 1 fl.hdf fl.nc # HDF4->netCDF4 (NCO 4.4.0+, netCDF 4.3.1+) ncks --hdf4 -3 fl.hdf fl.nc # HDF4->netCDF3 (netCDF 4.3.0-) ncks --hdf4 -7 fl.hdf fl.nc # HDF4->netCDF4 classic (netCDF 4.3.0-) """ if hdf_file is None: pytest.skip("Skipped because h5py is not installed") nco = Nco(debug=True) nco.ncks(input=hdf_file, output="foo.nc", options=["-3"]) nco.ncks(input=hdf_file, output="foo.nc", options=["-7 -L 1"]) nco.ncks(input=hdf_file, output="foo.nc", options=["-3"], hdf4=True) nco.ncks(input=hdf_file, output="foo.nc", options=["-7"], hdf4=True)
def test_temp_output_files(foo_nc): """ 2.3 Temporary Output Files ncks in.nc out.nc # Default: create out.pid.tmp.nc then move to out.nc ncks --wrt_tmp_fl in.nc out.nc # Same as default ncks --no_tmp_fl in.nc out.nc # Create out.nc directly on disk ncks --no_tmp_fl in.nc in.nc # ERROR-prone! Overwrite in.nc with itself ncks --create_ram --no_tmp_fl in.nc in.nc # Create in RAM, write to disk ncks --open_ram --no_tmp_fl in.nc in.nc # Read into RAM, write to disk """ nco = Nco(debug=True) nco.ncks(input=foo_nc, output='bar.nc') nco.ncks(input=foo_nc, output='bar.nc', wrt_tmp_fl=True) nco.ncks(input=foo_nc, output='bar.nc', no_tmp_fl=True) nco.ncks(input=foo_nc, output=foo_nc, no_tmp_fl=True, create_ram=True) nco.ncks(input=foo_nc, output=foo_nc, no_tmp_fl=True, open_ram=True)
def test_hyperslabs(testfileglobal): """ 3.15 Hyperslabs # First and second indices of lon dimension ncks -F -d lon,1,2 in.nc out.nc # Second and third indices of lon dimension ncks -d lon,1,2 in.nc out.nc # All longitude values between 1 and 2 degrees ncks -d lon,1.0,2.0 in.nc out.nc # All longitude values between 1 and 2 degrees ncks -F -d lon,1.0,2.0 in.nc out.nc # Every other longitude value between 0 and 90 degrees ncks -F -d lon,0.0,90.0,2 in.nc out.nc # Last two indices of lon dimension ncks -F -d lon,1,-2 in.nc out.nc # Third-to-last to last index of lon dimension ncks -F -d lon,-3,-1 in.nc out.nc # Third-to-last to last index of lon dimension ncks -F -d lon,-3, in.nc out.nc """ nco = Nco(debug=True) nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,1,2') nco.ncks(input=testfileglobal, output='out.nc', dimension='lon,1,2') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,1.0,2.0') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,0.0,90.0,2') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,1,-2') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,-3,-1') nco.ncks(input=testfileglobal, output='out.nc', fortran=True, dimension='lon,-3')
def test_file_conversion(foo3c, foo4c): """ 3.9.2 Determining File Format ncks --fl_fmt=classic in.nc foo_3c.nc # netCDF3 classic ncks --fl_fmt=64bit in.nc foo_364.nc # netCDF3 64bit ncks --fl_fmt=netcdf4_classic in.nc foo_4c.nc # netCDF4 classic ncks --fl_fmt=netcdf4 in.nc foo_4.nc # netCDF4 ncks -3 in.nc foo_3c.nc # netCDF3 classic ncks --3 in.nc foo_3c.nc # netCDF3 classic ncks -6 in.nc foo_364.nc # netCDF3 64bit ncks --64 in.nc foo_364.nc # netCDF3 64bit ncks -4 in.nc foo_4.nc # netCDF4 ncks --4 in.nc foo_4.nc # netCDF4 ncks -7 in.nc foo_4c.nc # netCDF4 classic ncks --7 in.nc foo_4c.nc # netCDF4 classic """ nco = Nco(debug=True) nco.ncks(input=foo4c, output='foo_3c.nc', fl_fmt='classic') nco.ncks(input=foo4c, output='foo_364.nc', fl_fmt='64bit') nco.ncks(input=foo3c, output='foo_4c.nc', fl_fmt='netcdf4_classic') nco.ncks(input=foo3c, output='foo_4.nc', fl_fmt='netcdf4') nco.ncks(input=foo4c, output='foo_3c.nc', options=['-3']) nco.ncks(input=foo4c, output='foo_3c.nc', options=['--3']) nco.ncks(input=foo3c, output='foo_364c.nc', options=['-6']) nco.ncks(input=foo3c, output='foo_364c.nc', options=['--64bit_offset']) nco.ncks(input=foo3c, output='foo_4.nc', options=['-4']) nco.ncks(input=foo3c, output='foo_4.nc', options=['--4']) nco.ncks(input=foo3c, output='foo_4c.nc', options=['-7']) nco.ncks(input=foo3c, output='foo_4c.nc', options=['--7'])
def test_determining_file_format(foo3c, foo364, foo4c, hdf_file): """ 3.9.2 Determining File Format ncks -M foo_3c.nc ncks -M foo_364.nc ncks -M foo_4c.nc ncks -M foo_4.nc ncks -D 2 -M hdf.hdf # ncks -D 2 -M http://thredds-test.ucar.edu/thredds/dodsC/testdods/in.nc ncks -D 2 -M foo_4.nc """ nco = Nco(debug=True) nco.ncks(input=foo3c, options=['-M']) nco.ncks(input=foo364, options=['-M']) nco.ncks(input=foo4c, options=['-M']) nco.ncks(input=foo4c, options=['-D 2 -M']) if hdf_file is not None: assert os.path.isfile(hdf_file) nco.ncks(input=hdf_file, options=['-D 2 -M'])
def test_use_list_inputs(foo_nc, bar_nc): nco = Nco(debug=True) infiles = [foo_nc, bar_nc] nco.ncrcat(input=infiles, output="out.nc")
def test_ncra_single_file(foo_nc): nco = Nco(debug=True) infile = foo_nc nco.ncra(input=infile, output="out.nc")
def goesr_nc_concat(fn_list, fn_concat=None, read_only=True, global_attrs=None, debug=False): """ Returns a NetCDF object representing a concatenation of multiple files using NCO bindings. Args: fn_list (list): List of files. e.g.: ['OR_SEIS-L1b-MPSL_G16_20032991650000_20032991650290_20151252148107.nc', 'OR_SEIS-L1b-MPSL_G16_20032991650300_20032991650590_20151252148107.nc'] Keyword Args: fn_concat (string): Optionally save new NetCDF file permanently to this path (relative is ok). e.g.: '/tmp/test_goesr_nc_concat.nc' read_only (True): Only effective if 'fn_concat' supplied. True: returned NC object is writable. False: returned NC object is read only. global_attrs (list): Returns list of global attributes for each file concatenated. This is a temporary step toward figuring out what to do with time varying global attributes. debug (True): True: print debug on STDOUT. False: no STDOUT. Returns: NetCDF object reference to aggregated result. None if an exception occurs. Examples (also, see main() below): 1) Get MPS-LO: >>> import glob, numpy as np, goesr_nc_concat >>> fn_list = sorted( glob.glob( ... 'OR_SEIS-L1b-MPSL_G??_??????????????_??????????????_??????????????.nc') ) >>> nc_all = goesr_nc_concat.goesr_nc_concat( fn_list, debug=True ) 2) Print Global Attributes: >>> print( 'Global Attributes:' ) >>> nc_all_dict = nc_all.__dict__ >>> for attr in nc_all_dict: print( '\t%s: %s' % (attr, nc_all_dict[ attr ]) ) 3) Save concatenation to file system permanently: >>> fn_concat = '/tmp/test_goesr_nc_concat.nc' >>> nc_all = goesr_nc_concat.goesr_nc_concat( fn_list, fn_concat=fn_concat, debug=True ) 4) Use data: >>> print( 'Mean Diff-e-flux: %f, %s' % ... ( np.mean( nc_all.variables['DiffElectronFluxes'] ), ... nc_all.variables['DiffElectronFluxes'].units ) ) Authors: R. Redmon (NOAA/NCEI), """ # TODO: Handle time varying global attributes. As sidecar dictionary? As new variables in returned object? # TODO: Replace STDOUT debug with logger. try: ' CONFIG ' my_name = 'goesr_nc_concat' file_rw_access = 'r' if read_only else 'r+' # Read Only or Writable? if 0 == len(fn_list): return None ' Imports ' import shutil, traceback from netCDF4 import Dataset as NCDataset from nco import Nco ' Concatenate to a Temporary File ' nco = Nco() fn_tmp = nco.ncrcat(input=fn_list, options='--create_ram') if debug: print(my_name + ': Aggregated %d files to temporary file: %s' % (len(fn_list), fn_tmp)) ' Save Temporary File Permanently (User Choice) ' if fn_concat != None: shutil.copy2(fn_tmp, fn_concat) if debug: print(my_name + ': Saved aggregated file permanently as: %s' % fn_concat) fn_tmp = fn_concat ' Open Concatenate as netcdf4-python::Dataset ' nc_all = NCDataset(fn_tmp, file_rw_access) ' Keep Global Attributes as Ordered List (User Choice) ' if global_attrs != None: for fn in fn_list: global_attrs.append(NCDataset(fn).__dict__) ' Return NC Object Reference ' return nc_all except Exception as e: # TODO: Exception prints stack trace. print(my_name + ': EXCEPTION: %s' % str(e)) print(repr(traceback.format_stack())) return None
def test_ncra_mult_files(foo_nc, bar_nc): nco = Nco(debug=True) infiles = [foo_nc, bar_nc] nco.ncra(input=infiles, output="out.nc")