def test_returnMaArray(bar_mask_nc, random_masked_field):
nco = Nco()
field = nco.ncea(input=bar_mask_nc,
output="tmp.nc",
returnMaArray='random',
options=['-O'])
assert type(field) == np.ma.core.MaskedArray
def test_returnArray(foo_nc):
nco = Nco(cdfMod='netcdf4')
random1 = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=['-O']).variables['random'][:]
assert type(random1) == np.ndarray
random2 = nco.ncea(input=foo_nc, output="tmp.nc",returnArray='random' ,options=['-O'])
assert type(random2) == np.ndarray
np.testing.assert_equal(random1, random2)
def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None):
""" Combine many files into a single file on disk. Defaults to using the 'time' dimension. """
nco = None
try:
nco = Nco()
except BaseException:
# This is not necessarily an import error (could be wrong PATH)
raise ImportError("NCO not found. The NCO python bindings are required to use 'Collection.combine'.")
if len(members) > 0 and hasattr(members[0], 'path'):
# A member DotDoct was passed in, we only need the paths
members = [ m.path for m in members ]
options = ['-4'] # NetCDF4
options += ['-L', '3'] # Level 3 compression
options += ['-h'] # Don't append to the history global attribute
if dimension is not None:
if start_index is None:
start_index = 0
if stop_index is None:
stop_index = ''
if stride is None:
stride = 1
options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)]
nco.ncrcat(input=members, output=output_file, options=options)
def normalize_time(netcdf_file):
epoch_units = 'seconds since 1970-01-01T00:00:00Z'
millisecond_units = 'milliseconds since 1858-11-17T00:00:00Z'
with nc4.Dataset(netcdf_file, 'a') as nc:
# Signell said this works, any problems and we can all blame him!
time_data = nc4.num2date(
(
np.int64(nc.variables['time'][:]) - 2400001
) * 3600 * 24 * 1000 + nc.variables['time2'][:].__array__(),
units=millisecond_units
)
nc.renameVariable("time", "old_time")
nc.sync()
time = nc.createVariable('time', 'f8', ('time'))
time.units = epoch_units
time.standard_name = "time"
time.long_name = "time of measurement"
time.calendar = "gregorian"
time.axis = "T"
time[:] = nc4.date2num(time_data, units=epoch_units).round()
o = Nco()
o.ncks(
input=netcdf_file,
output=netcdf_file,
options=[
'-O',
'-h',
'-x',
'-v', 'time2,old_time'
]
)
def test_use_list_options(foo_nc):
nco = Nco(debug=True)
options = []
options.extend(['-a', 'units,time,o,c,"days since 1999-01-01"'])
options.extend(['-a', 'long_name,time,o,c,time'])
options.extend(['-a', 'calendar,time,o,c,noleap'])
nco.ncatted(input=foo_nc, output='out.nc', options=options)
def test_use_list_options(foo_nc):
nco = Nco(debug=True)
options = []
options.extend(["-a", 'units,time,o,c,"days since 1999-01-01"'])
options.extend(["-a", "long_name,time,o,c,time"])
options.extend(["-a", "calendar,time,o,c,noleap"])
nco.ncatted(input=foo_nc, output="out.nc", options=options)
def __check_weather(self, filename, latidx, lonidx, latdelta, londelta):
with Dataset(filename) as f:
lats = f.variables['lat'][:]
lons = f.variables['lon'][:]
glatdelta = abs(diff(lats)[0])
glondelta = abs(diff(lons)[0])
minlat, maxlat, minlon, maxlon = self.__get_range(
latidx, lonidx, latdelta, londelta, glatdelta, glondelta)
# select first time
nco = Nco()
options = '-h -d time,0 -d lat,%f,%f -d lon,%f,%f' % (minlat, maxlat,
minlon, maxlon)
tmpfile = "checker.wth.nc4"
nco.ncks(input=filename, output=tmpfile, options=options)
with Dataset(tmpfile) as f:
varnames = f.variables.keys()
prvars = ['pr', 'prcp', 'pre', 'precip']
found = False
for i in range(len(prvars)):
patt = '%s$|%s.*' % (prvars[i], prvars[i])
for j in range(len(varnames)):
if compile(patt).match(varnames[j]):
pr = f.variables[varnames[j]][:]
found = True
break
if found:
break
if not found:
pr = array([])
hasweather = pr.size and (not isMaskedArray(pr)
or not pr.mask.all())
remove(tmpfile)
return hasweather
def normalize_locations(netcdf_file):
with CFDataset(netcdf_file) as nc:
y = nc.variables.get("lat")
y_data = y[:]
y_atts = nc.vatts('lat')
x = nc.variables.get("lon")
x_data = x[:]
x_atts = nc.vatts('lon')
# Remove the old x/y variable so we can add new ones with no dimensions
o = Nco()
o.ncks(input=netcdf_file,
output=netcdf_file,
options=['-O', '-h', '-x', '-v', 'lat,lon'])
# Add the new X/Y variables
with nc4.Dataset(netcdf_file, 'a') as nc:
lat = nc.createVariable('lat', y_data.dtype)
lat[:] = y_data
y_atts.update({'standard_name': 'latitude', 'axis': 'Y'})
lat.setncatts(y_atts)
lon = nc.createVariable('lon', x_data.dtype)
lon[:] = x_data
x_atts.update({'standard_name': 'longitude', 'axis': 'X'})
lon.setncatts(x_atts)
def test_use_list_options(foo_nc):
nco = Nco(debug=True)
options = []
options.extend(['-a', 'units,time,o,c,days since 1999-01-01'])
options.extend(['-a', 'long_name,time,o,c,time'])
options.extend(['-a', 'calendar,time,o,c,noleap'])
nco.ncrcat(input=foo_nc, output='out.nc', options=options)
def normalize_time(netcdf_file):
epoch_units = 'seconds since 1970-01-01T00:00:00Z'
millisecond_units = 'milliseconds since 1858-11-17T00:00:00Z'
with nc4.Dataset(netcdf_file, 'a') as nc:
# Signell said this works, any problems and we can all blame him!
time_data = nc4.num2date(
(np.int64(nc.variables['time'][:]) - 2400001) * 3600 * 24 * 1000 +
nc.variables['time2'][:].__array__(),
units=millisecond_units)
nc.renameVariable("time", "old_time")
nc.sync()
time = nc.createVariable('time', 'f8', ('time'))
time.units = epoch_units
time.standard_name = "time"
time.long_name = "time of measurement"
time.calendar = "gregorian"
time.axis = "T"
time[:] = nc4.date2num(time_data, units=epoch_units).round()
o = Nco()
o.ncks(input=netcdf_file,
output=netcdf_file,
options=['-O', '-h', '-x', '-v', 'time2,old_time'])
def normalize_ctd_depths(netcdf_file):
with CFDataset(netcdf_file, 'a') as nc:
depths = nc.variables['depth'][:][0]
z_atts = nc.vatts('depth')
# Remove the old depth variable so we can add a new one with no dimensions
o = Nco()
o.ncks(
input=netcdf_file,
output=netcdf_file,
options=[
'-O',
'-h',
'-x',
'-v', 'depth'
]
)
# Add the new Z variable
with nc4.Dataset(netcdf_file, 'a') as nc:
z = nc.createVariable('depth', 'f4')
z[:] = depths
z_atts.update({
'standard_name': 'depth',
'axis': 'Z',
'positive': 'down'
})
z.setncatts(z_atts)
def netCDF_merge(self, directory):
"""
To combine mutiple downloaded erai netCDF files into a large file with
specified chunk_size(e.g. 500),
-- give the full name of merged file to the output = outfile
-- pass all data from the first input netfile to the merged file name
-- loop over the files_list, append file one by one into the merge file
-- pass the mergae netcdf file to interpolation module to process
(to use nc.MFDataset by reading it)
Args:
ncfile_in: the full name of downloaded files (file directory + files names)
e.g.:
'/home/xquan/src/globsim/examples/erai/era_sa_*.nc'
'/home/xquan/src/globsim/examples/erai/era_pl_*.nc'
'/home/xquan/src/globsim/examples/erai/era_sf_*.nc'
Output: merged netCDF files
era_all_0.nc, era_all_1.nc, ...,
"""
# set up nco operator
nco = Nco()
# loop over filetypes, read, report
file_type = ['erai_sa_*.nc', 'erai_sf_*.nc', 'erai_pl_*.nc']
for ft in file_type:
ncfile_in = path.join(directory, ft)
# get the file list
files_list = glob.glob(ncfile_in)
files_list.sort()
num = len(files_list)
# set up the name of merged file
if ncfile_in[-7:-5] == 'sa':
merged_file = path.join(
ncfile_in[:-11], 'erai_sa_all_' + files_list[0][-23:-15] +
"_" + files_list[num - 1][-11:-3] + '.nc')
elif ncfile_in[-7:-5] == 'sf':
merged_file = path.join(
ncfile_in[:-11], 'erai_sf_all_' + files_list[0][-23:-15] +
'_' + files_list[num - 1][-11:-3] + '.nc')
elif ncfile_in[-7:-5] == 'pl':
merged_file = path.join(
ncfile_in[:-11], 'erai_pl_all_' + files_list[0][-23:-15] +
'_' + files_list[num - 1][-11:-3] + '.nc')
else:
print('There is not such type of file')
# combined files into merged files
nco.ncrcat(input=files_list, output=merged_file, append=True)
print('The Merged File below is saved:')
print(merged_file)
# clear up the data
for fl in files_list:
remove(fl)
def convert_to_netcdf3(input_file, output_file=None):
nco = Nco()
output_file = output_file or input_file
tmp_file = input_file[:-3] + "-tmp.nc"
nco.ncks(input=input_file, output=tmp_file, options=['-3'])
shutil.move(tmp_file, output_file)
LOGGER.info("Converted to NetCDF3 file: {}".format(output_file))
return output_file
def test_init_options():
nco = Nco(debug=True)
assert nco.debug
nco = Nco(force_output=False)
assert nco.force_output is False
nco = Nco(returnCdf=True, return_none_on_error=True)
assert nco.return_cdf
assert nco.return_none_on_error
def test_initOptions():
nco = Nco(debug=True)
assert nco.debug
nco = Nco(forceOutput=False)
assert nco.forceOutput is False
nco = Nco(returnCdf=True, returnNoneOnError=True)
assert nco.returnCdf
assert nco.returnNoneOnError
def normalize_depth_locations(netcdf_file):
redimension = []
with CFDataset(netcdf_file, 'a') as nc:
y = nc.variables.get("lat")
y_data = y[:]
y_atts = nc.vatts('lat')
x = nc.variables.get("lon")
x_data = x[:]
x_atts = nc.vatts('lon')
# Get list of variables to re-write with different dimensions
for vname, ov in nc.variables.items():
if 'lat' in ov.dimensions and 'lon' in ov.dimensions:
redimension.append(vname)
for vname in redimension:
ov = nc.variables[vname]
if 'depth' in ov.dimensions:
vdata = ov[:, :, 0, 0]
dims = ('time', 'depth')
else:
vdata = ov[:, 0, 0]
dims = ('time', )
vatts = nc.vatts(vname)
nc.renameVariable(vname, '{}_old'.format(vname))
nc.sync()
v = nc.createVariable(vname, vdata.dtype, dims)
v.setncatts(vatts)
v[:] = vdata
# Remove the old variables
remove_vars = ['{}_old'.format(vname) for vname in redimension]
remove_vars += ['lat', 'lon']
o = Nco()
o.ncks(input=netcdf_file,
output=netcdf_file,
options=['-O', '-h', '-x', '-v', ','.join(remove_vars)])
# Add the new X/Y variables
with nc4.Dataset(netcdf_file, 'a') as nc:
lat = nc.createVariable('lat', y_data.dtype)
lat[:] = y_data
y_atts.update({'standard_name': 'latitude', 'axis': 'Y'})
lat.setncatts(y_atts)
lon = nc.createVariable('lon', x_data.dtype)
lon[:] = x_data
x_atts.update({'standard_name': 'longitude', 'axis': 'X'})
lon.setncatts(x_atts)
def ocean_atlas_merge_variables(output_dir, month):
ocean_atlas_files = output_dir.glob(f'ocean_atlas_*_{month.month:02}.nc')
ocean_atlas_files = list(ocean_atlas_files)
ocean_atlas_files.sort()
outfile = output_dir / f'ocean_atlas_{month.month:02}.nc'
nco = Nco()
options = ['-A', '-4', '-L 5']
outfile = shutil.copy(ocean_atlas_files[0], outfile)
for f in ocean_atlas_files[1::]:
nco.ncks(input=str(f), output=str(outfile), options=options)
def test_return_array(foo_nc):
nco = Nco(cdf_module="netcdf4")
random1 = nco.ncea(
input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"]
).variables["random"][:]
assert isinstance(random1, np.ndarray)
random2 = nco.ncea(
input=foo_nc, output="tmp.nc", returnArray="random", options=["-O"]
)
assert isinstance(random2, np.ndarray)
np.testing.assert_equal(random1, random2)
def narr_merge_variables(output_dir):
narr_files = output_dir.glob('narr_*.nc')
narr_files = list(narr_files)
narr_files.sort()
outfile = output_dir.parent / 'narr.nc'
nco = Nco()
options = ['-A', '-4', '-L 5']
outfile = shutil.copy(narr_files[0], outfile)
for f in narr_files[1::]:
nco.ncks(input=str(f), output=str(outfile), options=options)
def normalize_netcdf4(netcdf_file):
o = Nco()
o.ncks(
input=netcdf_file,
output=netcdf_file,
options=[
'-O',
'-h',
'-4',
'-L3'
]
)
def test_ncks_hdf2nc(hdf_file):
"""
1.6 netCDF2/3/4 and HDF4/5 Support
Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively,
it is now easy to convert HDF4 files to netCDF files directly, e.g.,
ncks fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+)
ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9)
"""
nco = Nco(debug=True)
nco.ncks(input=hdf_file, output='foo.nc')
nco.ncks(input=hdf_file, output='foo.nc', hdf4=True)
def test_ncks_append_variables(foo_nc, bar_nc):
"""
2.4 Appending Variables
The simplest way to create the union of two files is
ncks -A fl_1.nc fl_2.nc
"""
nco = Nco(debug=True)
nco.ncks(input=foo_nc, output=bar_nc, options=["-A"])
nco.ncks(input=foo_nc, output=bar_nc, append=True)
nco.ncks(input=foo_nc, output=bar_nc, apn=True)
nco.ncks(input=foo_nc, output=bar_nc)
def test_command_line_options(foo_nc):
"""
3.4 Command Line Options
ncks -D 3 in.nc # Short option
ncks --dbg_lvl=3 in.nc # Long option, preferred form
ncks --dbg_lvl 3 in.nc # Long option, alternate form
"""
nco = Nco(debug=True)
nco.ncks(input=foo_nc, options=["-O -D 3"])
nco.ncks(input=foo_nc, options=["-O --dbg_lvl=3"])
nco.ncks(input=foo_nc, options=["-O --dbg_lvl 3"])
def test_command_line_options(foo_nc):
"""
3.4 Command Line Options
ncks -D 3 in.nc # Short option
ncks --dbg_lvl=3 in.nc # Long option, preferred form
ncks --dbg_lvl 3 in.nc # Long option, alternate form
"""
nco = Nco(debug=True)
test = nco.ncks(input=foo_nc, options='-D 3')
test = nco.ncks(input=foo_nc, options='--dbg_lvl=3')
test = nco.ncks(input=foo_nc, options='--dbg_lvl 3')
test = nco.ncks(input=foo_nc, dbg_lvl=3)
def test_return_cdf(foo_nc):
nco = Nco(cdf_module="scipy")
test_cdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"])
assert type(test_cdf) == scipy.io.netcdf.netcdf_file
expected_vars = ["time", "random"]
for var in expected_vars:
assert var in list(test_cdf.variables.keys())
nco = Nco(cdf_module="netcdf4")
test_cdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=["-O"])
assert type(test_cdf) == netCDF4.Dataset
for var in expected_vars:
assert var in list(test_cdf.variables.keys())
def test_returnCdf(foo_nc):
nco = Nco(cdfMod='scipy')
testCdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True,options=['-O'])
assert type(testCdf) == scipy.io.netcdf.netcdf_file
expected_vars = ['time', 'random']
for var in expected_vars:
assert var in list(testCdf.variables.keys())
nco = Nco(cdfMod='netcdf4')
testCdf = nco.ncea(input=foo_nc, output="tmp.nc", returnCdf=True, options=['-O'])
assert type(testCdf) == netCDF4.Dataset
for var in expected_vars:
assert var in list(testCdf.variables.keys())
def ocean_atlas_merge_time(output_dir):
variable_merged_files = output_dir.glob('ocean_atlas_??.nc')
variable_merged_files = [
str(merged_file) for merged_file in list(variable_merged_files)
]
variable_merged_files.sort()
output_file = output_dir.parent / 'ocean_atlas.nc'
nco = Nco()
options = ['-A']
nco.ncrcat(input=variable_merged_files,
output=str(output_file),
options=options)
def test_returnArray(foo_nc):
nco = Nco(cdfMod='netcdf4')
random1 = nco.ncea(input=foo_nc,
output="tmp.nc",
returnCdf=True,
options=['-O']).variables['random'][:]
assert type(random1) == np.ndarray
random2 = nco.ncea(input=foo_nc,
output="tmp.nc",
returnArray='random',
options=['-O'])
assert type(random2) == np.ndarray
np.testing.assert_equal(random1, random2)
def test_ncks_hdf2nc(hdf_file):
"""
1.6 netCDF2/3/4 and HDF4/5 Support
Converting HDF4 files to netCDF: Since NCO reads HDF4 files natively,
it is now easy to convert HDF4 files to netCDF files directly, e.g.,
ncks fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+)
ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9)
"""
if hdf_file is None:
pytest.skip("Skipped because h5py is not installed")
nco = Nco(debug=True)
nco.ncks(input=hdf_file, output="foo.nc")
nco.ncks(input=hdf_file, output="foo.nc", hdf4=True)
def narr_enhance(output_dir):
outfile = output_dir.parent / 'narr.nc'
outtmp = output_dir.parent / 'narr_tmp.nc'
# remove unecessary vars
nco = Nco()
options = ['-O', '-C', '-x', '-v climatology_bounds,valid_yr_count']
nco.ncks(input=str(outfile), output=str(outfile), options=options)
# change lon from [0, 360) to [-180, 180)
options = [
'-O',
'--msa',
'-d lon,181.,360.',
'-d lon,0.,180.0',
]
nco.ncks(input=str(outfile), output=str(outtmp), options=options)
options = ['-O', "-s 'where(lon > 180) lon=lon-360'"]
nco.ncap2(input=str(outtmp), output=str(outfile), options=options)
# times don't work with xarray, so the year is changed to the middle of climatology
# climatology: 1981-01-01 - 2010-12-31
# midyear is 1996
time_range = xr.cftime_range(start='1996', end='1996-12-31', freq='MS')
new_units = 'days since 1970-01-01 00:00:00'
new_times = nc.date2num(time_range, units=new_units)
with nc.Dataset(outfile, 'a') as ncd:
time = ncd.variables['time']
time[:] = new_times
time.units = new_units
def convert_dl_to_nc4c(self, fns):
self.log.debug('Converting to NETCDF4_CLASSIC ...')
if not _NCO:
self.log.error(
'Cannot convert to NETCDF4_CLASSIC: nco is not installed')
raise ImportError('cannot import nco library')
nco = Nco()
fns_new = []
for fn in fns:
fn_new = tempfile.mktemp(suffix='.nc', dir=self.cfg['temppath'])
nco.ncks(input=fn, output=fn_new, options=['-O', '-7'])
fns_new.append(fn_new)
self.cfg['dldriver'].clean_tempfiles([fn])
self.log.debug('... done.')
return fns_new
def test_ncks_append_variables(foo_nc, bar_nc):
"""
2.4 Appending Variables
The simplest way to create the union of two files is
ncks -A fl_1.nc fl_2.nc
"""
nco = Nco(debug=True)
nco.ncks(input=foo_nc, output=bar_nc, options=['-A'])
nco.ncks(input=foo_nc, output=bar_nc, append=True)
nco.ncks(input=foo_nc, output=bar_nc, apn=True)
nco.ncks(input=foo_nc, output=bar_nc)
def test_specifying_input_files(testfiles8589):
"""
3.5 Specifying Input Files
ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra 8[56789].nc 8589.nc
ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra -n 5,2,1 85.nc 8589.nc
"""
inputs = ['85.nc', '86.nc', '87.nc', '88.nc', '89.nc']
nco = Nco(debug=True)
nco.ncra(input=inputs, ouptut='8589.nc')
nco.ncra(input='8[56789].nc', ouptut='8589.nc')
srcdir = os.path.split(inputs[0])
nco.ncra(input=inputs, ouptut='8589.nc', path=srcdir)
nco.ncra(input=inputs, ouptut='8589.nc', nintap='5,2,1')
def xgeo_multifile_load(nc_wildcard, nc_dir=None):
"""
Read data from multiple (e.g. hourly xgeo data) netcdf files
:param nc_wildcard: common section of the filenams, e.g. mf_files*.nc
:return:
"""
if dir:
nc_path = os.path.join(nc_dir, nc_wildcard)
else:
nc_path = nc_wildcard
nco = Nco()
nc_temp = nco.ncrcat(input=nc_path)
nc = netCDF4.Dataset(nc_temp)
#add function to restrict dates and times
return nc
def run(self, ds, vars, bbox, start, end, chunk=None):
# Split large data sets into chunks
if chunk:
dates = pd.date_range(start, end, freq=chunk)
else:
dates = [start, end]
print(self.locs['chunks'].filename)
self.locs['chunks'] = DatetimeLoc(datetimes=dates,
template=self.locs['chunks'])
self.locs['chunks'].configure(self.cfg)
# Download
nco = Nco()
for i, (start_date, end_date) in enumerate(zip(dates[:-1], dates[1:])):
info = {'year': start_date.year}
logging.debug(ds.loc.url.format(**info))
logging.debug(self.locs['chunks'].locs[i].filename)
nco.ncks(
input=ds.loc.url.format(**info),
output=self.locs['chunks'].locs[i].path,
options=[
'--mk_rec_dmn time', '-v ' + ','.join(vars),
'-d ' + ','.join(
['time',
start_date.isoformat(),
end_date.isoformat()]),
'-d ' + ','.join(
['lon', str(bbox.min.lon),
str(bbox.max.lon)]),
'-d ' + ','.join(
['lat', str(bbox.min.lat),
str(bbox.max.lat)])
])
# Merge chunks
cat_args = [
'SKIP_SAME_TIME=1', 'cdo', 'mergetime',
os.path.join(self.locs['chunks'].dirname, '*'),
self.locs['data'].path
]
logging.info('Calling process %s', ' '.join(cat_args))
cat_process = subprocess.Popen(cat_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
cat_output, _ = cat_process.communicate()
def remove_variables(self):
###########################################################################
if len(self._rvars) > 0:
Nco().ncks(str(self._ofile),
output=str(self._ofile),
exclude=True,
force=True,
variable=",".join(self._rvars))
def test_specifying_input_files(testfiles8589):
"""
3.5 Specifying Input Files
ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra 8[56789].nc 8589.nc
ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra -n 5,2,1 85.nc 8589.nc
"""
nco = Nco(debug=True)
nco.ncra(input=testfiles8589, output='8589.nc')
nco.ncra(input=testfiles8589, output='8589.nc', nintap='5,2,1')
srcdir = os.path.dirname(testfiles8589[0])
basenames = [os.path.basename(x) for x in testfiles8589]
nco.ncra(input=basenames, output='8589.nc', path=srcdir)
regpath = os.path.join(srcdir, '8[56789].nc')
nco.ncra(input=regpath, output='8589.nc')
def test_specifying_input_files(testfiles8589):
"""
3.5 Specifying Input Files
ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra 8[56789].nc 8589.nc
ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra -n 5,2,1 85.nc 8589.nc
"""
inputs = ['85.nc', '86.nc', '87.nc', '88.nc', '89.nc']
nco = Nco(debug=True)
nco.ncra(input=inputs, ouptut='8589.nc')
nco.ncra(input='8[56789].nc', ouptut='8589.nc')
srcdir = os.path.split(inputs[0])
nco.ncra(input=inputs, ouptut='8589.nc', path=srcdir)
nco.ncra(input=inputs, ouptut='8589.nc', nintap='5,2,1')
def normalize_summary_depths(netcdf_file):
redimension = []
with CFDataset(netcdf_file, 'a') as nc:
nc.renameDimension('depth', 'z')
z_atts = nc.vatts('depth')
# Get list of variables to re-write with different dimensions
for vname, ov in nc.variables.items():
if 'depth' in ov.dimensions:
redimension.append(vname)
for vname in redimension:
ov = nc.variables[vname]
vatts = nc.vatts(vname)
nc.renameVariable(vname, '{}_old'.format(vname))
nc.sync()
v = nc.createVariable(vname, ov.dtype, ('time', ))
v.setncatts(vatts)
v[:] = ov[:, 0]
# Remove the old variables
remove_vars = ['{}_old'.format(vname) for vname in redimension]
remove_vars += ['depth']
o = Nco()
o.ncks(input=netcdf_file,
output=netcdf_file,
options=['-O', '-h', '-x', '-v', ','.join(remove_vars)])
# Add the new Z variable
with nc4.Dataset(netcdf_file, 'a') as nc:
z = nc.createVariable('depth', 'f4')
z[:] = None
z_atts.update({
'standard_name': 'depth',
'axis': 'Z',
'positive': 'down'
})
z.setncatts(z_atts)
def __check_soil(self, filename, latidx, lonidx, latdelta, londelta):
with nc(filename) as f:
lats, lons = f.variables['lat'][:], f.variables['lon'][:]
glatdelta, glondelta = abs(diff(lats)[0]), abs(diff(lons)[0])
minlat, maxlat, minlon, maxlon = self.__get_range(
latidx, lonidx, latdelta, londelta, glatdelta, glondelta)
nco = Nco()
options = '-h -d lat,%f,%f -d lon,%f,%f' % (minlat, maxlat, minlon,
maxlon)
nco.ncks(input=filename, output='tmp.nc4', options=options)
with nc('tmp.nc4') as f:
slsi = f.variables['slsi'][:]
hassoil = slsi.size and (not isMaskedArray(slsi)
or not slsi.mask.all())
remove('tmp.nc4')
return hassoil
def test_specifying_input_files(testfiles8589):
"""
3.5 Specifying Input Files
ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra 8[56789].nc 8589.nc
ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra -n 5,2,1 85.nc 8589.nc
"""
nco = Nco(debug=True)
nco.ncra(input=testfiles8589, output="8589.nc")
nco.ncra(input=testfiles8589, output="8589.nc", nintap="5,2,1")
srcdir = os.path.dirname(testfiles8589[0])
basenames = [os.path.basename(x) for x in testfiles8589]
nco.ncra(input=basenames, output="8589.nc", path=srcdir)
# unable to use brackets, perhaps because we're no longer using shell=True when calling subprocess()?
regpath = os.path.join(srcdir, "8[56789].nc")
nco.ncra(input=regpath, output="8589.nc", use_shell=True)
def normalize_ctd_depths(netcdf_file):
with CFDataset(netcdf_file, 'a') as nc:
depths = nc.variables['depth'][:][0]
z_atts = nc.vatts('depth')
# Remove the old depth variable so we can add a new one with no dimensions
o = Nco()
o.ncks(input=netcdf_file,
output=netcdf_file,
options=['-O', '-h', '-x', '-v', 'depth'])
# Add the new Z variable
with nc4.Dataset(netcdf_file, 'a') as nc:
z = nc.createVariable('depth', 'f4')
z[:] = depths
z_atts.update({
'standard_name': 'depth',
'axis': 'Z',
'positive': 'down'
})
z.setncatts(z_atts)
def normalize_locations(netcdf_file):
with CFDataset(netcdf_file) as nc:
y = nc.variables.get("lat")
y_data = y[:]
y_atts = nc.vatts('lat')
x = nc.variables.get("lon")
x_data = x[:]
x_atts = nc.vatts('lon')
# Remove the old x/y variable so we can add new ones with no dimensions
o = Nco()
o.ncks(
input=netcdf_file,
output=netcdf_file,
options=[
'-O',
'-h',
'-x',
'-v', 'lat,lon'
]
)
# Add the new X/Y variables
with nc4.Dataset(netcdf_file, 'a') as nc:
lat = nc.createVariable('lat', y_data.dtype)
lat[:] = y_data
y_atts.update({
'standard_name': 'latitude',
'axis': 'Y'
})
lat.setncatts(y_atts)
lon = nc.createVariable('lon', x_data.dtype)
lon[:] = x_data
x_atts.update({
'standard_name': 'longitude',
'axis': 'X'
})
lon.setncatts(x_atts)
def test_specifying_input_files(testfiles8589):
"""
3.5 Specifying Input Files
ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra 8[56789].nc 8589.nc
ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra -n 5,2,1 85.nc 8589.nc
"""
nco = Nco(debug=True)
nco.ncra(input=testfiles8589, output='8589.nc')
nco.ncra(input=testfiles8589, output='8589.nc', nintap='5,2,1')
srcdir = os.path.dirname(testfiles8589[0])
basenames = [ os.path.basename(x) for x in testfiles8589 ]
nco.ncra(input=basenames, output='8589.nc', path=srcdir)
regpath = os.path.join(srcdir, '8[56789].nc')
nco.ncra(input=regpath, output='8589.nc')
def test_specifying_input_files(testfiles8589):
"""
3.5 Specifying Input Files
ncra 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra 8[56789].nc 8589.nc
ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc
ncra -n 5,2,1 85.nc 8589.nc
"""
nco = Nco(debug=True)
nco.ncra(input=testfiles8589, output="8589.nc")
nco.ncra(input=testfiles8589, output="8589.nc", nintap="5,2,1")
srcdir = os.path.dirname(testfiles8589[0])
basenames = [os.path.basename(x) for x in testfiles8589]
nco.ncra(input=basenames, output="8589.nc", path=srcdir)
# unable to use brackets, perhaps because we're no longer using shell=True when calling subprocess()?
regpath = os.path.join(srcdir, "8[56789].nc")
nco.ncra(input=regpath, output="8589.nc", use_shell=True)
def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None):
""" Combine many files into a single file on disk. Defaults to using the 'time' dimension. """
nco = None
try:
nco = Nco()
except BaseException:
raise ImportError("NCO not found. The NCO python bindings are required to use 'Collection.combine'.")
if len(members) > 0 and hasattr(members[0], 'path'):
# A member DotDoct was passed in, we only need the paths
members = [ m.path for m in members ]
options = ['-4'] # NetCDF4
options += ['-L', '3'] # Level 3 compression
options += ['-h'] # Don't append to the history global attribute
if dimension is not None:
if start_index is None:
start_index = 0
if stop_index is None:
stop_index = ''
if stride is None:
stride = 1
options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)]
nco.ncrcat(input=members, output=output_file, options=options)
def dailyAve():
from nco import Nco
import datetime
nco = Nco()
for d in range(365):
dp = datetime.date(startY,1,1)+datetime.timedelta(d)
print "Averaging TRMM 3B42 for day "+dp.strftime('%j')+"..."
ifile = ' '.join("3B42_daily."+str(year)+"."+dp.strftime('%m')+"."+dp.strftime('%d')+".7.nc" for year in range(startY,endY))
ofile = "3B42_aver."+dp.strftime('%j')+".nc"
if not os.path.isfile(ofile):
nco.ncra(input=ifile, output=ofile)
nco.ncrcat(input="3B42_aver.*.nc", output="3B42_cat.nc", options="-d time,1,365")
nco.ncwa(input="3B42_cat.nc", output="3B42_MAP.nc", options='-N -a time')
return None
def test_ncks_hdf2nc3(hdf_file):
"""
1.6 netCDF2/3/4 and HDF4/5 Support
Obtaining a netCDF3 file from an HDF4 is now easy, even though the HDF4
file may contain netCDF4 atomic types (e.g., unsigned bytes,
64-bit integers):
ncks -3 fl.hdf fl.nc # HDF4->netCDF3 (NCO 4.4.0+, netCDF 4.3.1+)
ncks -7 -L 1 fl.hdf fl.nc # HDF4->netCDF4 (NCO 4.4.0+, netCDF 4.3.1+)
ncks --hdf4 -3 fl.hdf fl.nc # HDF4->netCDF3 (netCDF 4.3.0-)
ncks --hdf4 -7 fl.hdf fl.nc # HDF4->netCDF4 classic (netCDF 4.3.0-)
"""
nco = Nco(debug=True)
nco.ncks(input=hdf_file, output='foo.nc', options='-3')
nco.ncks(input=hdf_file, output='foo.nc', options='-7 -L 1')
nco.ncks(input=hdf_file, output='foo.nc', options='-3', hdf4=True)
nco.ncks(input=hdf_file, output='foo.nc', options='-7', hdf4=True)
def test_ncks_hdf2nc3(hdf_file):
"""
1.6 netCDF2/3/4 and HDF4/5 Support
Obtaining a netCDF3 file from an HDF4 is now easy, even though the HDF4
file may contain netCDF4 atomic types (e.g., unsigned bytes,
64-bit integers):
ncks -3 fl.hdf fl.nc # HDF4->netCDF3 (NCO 4.4.0+, netCDF 4.3.1+)
ncks -7 -L 1 fl.hdf fl.nc # HDF4->netCDF4 (NCO 4.4.0+, netCDF 4.3.1+)
ncks --hdf4 -3 fl.hdf fl.nc # HDF4->netCDF3 (netCDF 4.3.0-)
ncks --hdf4 -7 fl.hdf fl.nc # HDF4->netCDF4 classic (netCDF 4.3.0-)
"""
if hdf_file is None:
pytest.skip("Skipped because h5py is not installed")
nco = Nco(debug=True)
nco.ncks(input=hdf_file, output="foo.nc", options=["-3"])
nco.ncks(input=hdf_file, output="foo.nc", options=["-7 -L 1"])
nco.ncks(input=hdf_file, output="foo.nc", options=["-3"], hdf4=True)
nco.ncks(input=hdf_file, output="foo.nc", options=["-7"], hdf4=True)
def test_temp_output_files(foo_nc):
"""
2.3 Temporary Output Files
ncks in.nc out.nc # Default: create out.pid.tmp.nc then move to out.nc
ncks --wrt_tmp_fl in.nc out.nc # Same as default
ncks --no_tmp_fl in.nc out.nc # Create out.nc directly on disk
ncks --no_tmp_fl in.nc in.nc # ERROR-prone! Overwrite in.nc with itself
ncks --create_ram --no_tmp_fl in.nc in.nc # Create in RAM, write to disk
ncks --open_ram --no_tmp_fl in.nc in.nc # Read into RAM, write to disk
"""
nco = Nco(debug=True)
nco.ncks(input=foo_nc, output='bar.nc')
nco.ncks(input=foo_nc, output='bar.nc', wrt_tmp_fl=True)
nco.ncks(input=foo_nc, output='bar.nc', no_tmp_fl=True)
nco.ncks(input=foo_nc, output=foo_nc, no_tmp_fl=True, create_ram=True)
nco.ncks(input=foo_nc, output=foo_nc, no_tmp_fl=True, open_ram=True)
def test_hyperslabs(testfileglobal):
"""
3.15 Hyperslabs
# First and second indices of lon dimension
ncks -F -d lon,1,2 in.nc out.nc
# Second and third indices of lon dimension
ncks -d lon,1,2 in.nc out.nc
# All longitude values between 1 and 2 degrees
ncks -d lon,1.0,2.0 in.nc out.nc
# All longitude values between 1 and 2 degrees
ncks -F -d lon,1.0,2.0 in.nc out.nc
# Every other longitude value between 0 and 90 degrees
ncks -F -d lon,0.0,90.0,2 in.nc out.nc
# Last two indices of lon dimension
ncks -F -d lon,1,-2 in.nc out.nc
# Third-to-last to last index of lon dimension
ncks -F -d lon,-3,-1 in.nc out.nc
# Third-to-last to last index of lon dimension
ncks -F -d lon,-3, in.nc out.nc
"""
nco = Nco(debug=True)
nco.ncks(input=testfileglobal, output='out.nc', fortran=True,
dimension='lon,1,2')
nco.ncks(input=testfileglobal, output='out.nc',
dimension='lon,1,2')
nco.ncks(input=testfileglobal, output='out.nc', fortran=True,
dimension='lon,1.0,2.0')
nco.ncks(input=testfileglobal, output='out.nc', fortran=True,
dimension='lon,0.0,90.0,2')
nco.ncks(input=testfileglobal, output='out.nc', fortran=True,
dimension='lon,1,-2')
nco.ncks(input=testfileglobal, output='out.nc', fortran=True,
dimension='lon,-3,-1')
nco.ncks(input=testfileglobal, output='out.nc', fortran=True,
dimension='lon,-3')
def test_file_conversion(foo3c, foo4c):
"""
3.9.2 Determining File Format
ncks --fl_fmt=classic in.nc foo_3c.nc # netCDF3 classic
ncks --fl_fmt=64bit in.nc foo_364.nc # netCDF3 64bit
ncks --fl_fmt=netcdf4_classic in.nc foo_4c.nc # netCDF4 classic
ncks --fl_fmt=netcdf4 in.nc foo_4.nc # netCDF4
ncks -3 in.nc foo_3c.nc # netCDF3 classic
ncks --3 in.nc foo_3c.nc # netCDF3 classic
ncks -6 in.nc foo_364.nc # netCDF3 64bit
ncks --64 in.nc foo_364.nc # netCDF3 64bit
ncks -4 in.nc foo_4.nc # netCDF4
ncks --4 in.nc foo_4.nc # netCDF4
ncks -7 in.nc foo_4c.nc # netCDF4 classic
ncks --7 in.nc foo_4c.nc # netCDF4 classic
"""
nco = Nco(debug=True)
nco.ncks(input=foo4c, output='foo_3c.nc', fl_fmt='classic')
nco.ncks(input=foo4c, output='foo_364.nc', fl_fmt='64bit')
nco.ncks(input=foo3c, output='foo_4c.nc', fl_fmt='netcdf4_classic')
nco.ncks(input=foo3c, output='foo_4.nc', fl_fmt='netcdf4')
nco.ncks(input=foo4c, output='foo_3c.nc', options=['-3'])
nco.ncks(input=foo4c, output='foo_3c.nc', options=['--3'])
nco.ncks(input=foo3c, output='foo_364c.nc', options=['-6'])
nco.ncks(input=foo3c, output='foo_364c.nc', options=['--64bit_offset'])
nco.ncks(input=foo3c, output='foo_4.nc', options=['-4'])
nco.ncks(input=foo3c, output='foo_4.nc', options=['--4'])
nco.ncks(input=foo3c, output='foo_4c.nc', options=['-7'])
nco.ncks(input=foo3c, output='foo_4c.nc', options=['--7'])
def test_determining_file_format(foo3c, foo364, foo4c, hdf_file):
"""
3.9.2 Determining File Format
ncks -M foo_3c.nc
ncks -M foo_364.nc
ncks -M foo_4c.nc
ncks -M foo_4.nc
ncks -D 2 -M hdf.hdf
# ncks -D 2 -M http://thredds-test.ucar.edu/thredds/dodsC/testdods/in.nc
ncks -D 2 -M foo_4.nc
"""
nco = Nco(debug=True)
nco.ncks(input=foo3c, options=['-M'])
nco.ncks(input=foo364, options=['-M'])
nco.ncks(input=foo4c, options=['-M'])
nco.ncks(input=foo4c, options=['-D 2 -M'])
if hdf_file is not None:
assert os.path.isfile(hdf_file)
nco.ncks(input=hdf_file, options=['-D 2 -M'])
def test_use_list_inputs(foo_nc, bar_nc):
nco = Nco(debug=True)
infiles = [foo_nc, bar_nc]
nco.ncrcat(input=infiles, output="out.nc")
def test_ncra_single_file(foo_nc):
nco = Nco(debug=True)
infile = foo_nc
nco.ncra(input=infile, output="out.nc")
def goesr_nc_concat(fn_list, fn_concat=None, read_only=True, global_attrs=None, debug=False):
""" Returns a NetCDF object representing a concatenation of multiple files using NCO bindings.
Args:
fn_list (list): List of files.
e.g.: ['OR_SEIS-L1b-MPSL_G16_20032991650000_20032991650290_20151252148107.nc',
'OR_SEIS-L1b-MPSL_G16_20032991650300_20032991650590_20151252148107.nc']
Keyword Args:
fn_concat (string): Optionally save new NetCDF file permanently to this path (relative is ok).
e.g.: '/tmp/test_goesr_nc_concat.nc'
read_only (True): Only effective if 'fn_concat' supplied. True: returned NC object is writable.
False: returned NC object is read only.
global_attrs (list): Returns list of global attributes for each file concatenated. This is
a temporary step toward figuring out what to do with time varying global attributes.
debug (True): True: print debug on STDOUT. False: no STDOUT.
Returns:
NetCDF object reference to aggregated result.
None if an exception occurs.
Examples (also, see main() below):
1) Get MPS-LO:
>>> import glob, numpy as np, goesr_nc_concat
>>> fn_list = sorted( glob.glob(
... 'OR_SEIS-L1b-MPSL_G??_??????????????_??????????????_??????????????.nc') )
>>> nc_all = goesr_nc_concat.goesr_nc_concat( fn_list, debug=True )
2) Print Global Attributes:
>>> print( 'Global Attributes:' )
>>> nc_all_dict = nc_all.__dict__
>>> for attr in nc_all_dict: print( '\t%s: %s' % (attr, nc_all_dict[ attr ]) )
3) Save concatenation to file system permanently:
>>> fn_concat = '/tmp/test_goesr_nc_concat.nc'
>>> nc_all = goesr_nc_concat.goesr_nc_concat( fn_list, fn_concat=fn_concat, debug=True )
4) Use data:
>>> print( 'Mean Diff-e-flux: %f, %s' %
... ( np.mean( nc_all.variables['DiffElectronFluxes'] ),
... nc_all.variables['DiffElectronFluxes'].units ) )
Authors: R. Redmon (NOAA/NCEI),
"""
# TODO: Handle time varying global attributes. As sidecar dictionary? As new variables in returned object?
# TODO: Replace STDOUT debug with logger.
try:
' CONFIG '
my_name = 'goesr_nc_concat'
file_rw_access = 'r' if read_only else 'r+' # Read Only or Writable?
if 0 == len(fn_list): return None
' Imports '
import shutil, traceback
from netCDF4 import Dataset as NCDataset
from nco import Nco
' Concatenate to a Temporary File '
nco = Nco()
fn_tmp = nco.ncrcat(input=fn_list, options='--create_ram')
if debug: print(my_name + ': Aggregated %d files to temporary file: %s' % (len(fn_list), fn_tmp))
' Save Temporary File Permanently (User Choice) '
if fn_concat != None:
shutil.copy2(fn_tmp, fn_concat)
if debug: print(my_name + ': Saved aggregated file permanently as: %s' % fn_concat)
fn_tmp = fn_concat
' Open Concatenate as netcdf4-python::Dataset '
nc_all = NCDataset(fn_tmp, file_rw_access)
' Keep Global Attributes as Ordered List (User Choice) '
if global_attrs != None:
for fn in fn_list:
global_attrs.append(NCDataset(fn).__dict__)
' Return NC Object Reference '
return nc_all
except Exception as e:
# TODO: Exception prints stack trace.
print(my_name + ': EXCEPTION: %s' % str(e))
print(repr(traceback.format_stack()))
return None
def test_ncra_mult_files(foo_nc, bar_nc):
nco = Nco(debug=True)
infiles = [foo_nc, bar_nc]
nco.ncra(input=infiles, output="out.nc")
