def runTest(self):
"""testing multi-file dataset access"""
f = MFDataset(self.files, check=True)
f.set_auto_maskandscale(True) # issue570
f.set_always_mask(False)
assert f.history == 'created today'
assert_array_equal(np.arange(0, nx), f.variables['x'][:])
varin = f.variables['data']
datin = varin[:]
assert_array_equal(datin.mask, data.mask)
varin.set_auto_maskandscale(False)
data2 = data.filled()
assert varin.long_name == 'phony data'
assert len(varin) == nx
assert varin.shape == (nx, ydim, zdim)
assert varin.dimensions == ('x', 'y', 'z')
assert_array_equal(varin[4:-4:4, 3:5, 2:8], data2[4:-4:4, 3:5, 2:8])
assert varin[0, 0, 0] == data2[0, 0, 0]
assert_array_equal(varin[:], data2)
assert getattr(varin, 'nonexistantatt', None) == None
f.close()
# test master_file kwarg (issue #835).
f = MFDataset(self.files, master_file=self.files[-1], check=True)
assert_array_equal(np.arange(0, nx), f.variables['x'][:])
varin = f.variables['data']
assert_array_equal(varin[4:-4:4, 3:5, 2:8], data2[4:-4:4, 3:5, 2:8])
f.close()
# testing multi-file get_variables_by_attributes
f = MFDataset(self.files, check=True)
assert f.get_variables_by_attributes(axis='T') == []
f.get_variables_by_attributes(units='zlotys')[0] == f['x']
f.close()
def __init__(
self,
path="/RECH/skynet1_rech3/huziy/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.tmp.dat.nc",
var_name="tmp",
lazy=False):
self.times = None
self.var_data = None
self.times_var = None
self.kdtree = None
self.times_num = None
self.lons2d, self.lats2d = None, None
self.lazy = lazy
self.var_name = var_name
try:
with Dataset(path) as ds:
self._init_fields(ds)
# Cannot go into with, since it needs to be open
self.nc_dataset = Dataset(path)
except OSError as oserr:
with MFDataset(path) as ds:
self._init_fields(ds)
# Cannot go into with, since it needs to be open
self.nc_dataset = MFDataset(path)
self.nc_vars = ds.variables
def __init__(self,variables=None,**kwargs):
"""
Initialise the variables to outfile
"""
self.__dict__.update(kwargs)
# Open the file
if not self.multifile:
try:
self._nc = Dataset(self.ncurl)
except:
try:
self._nc = MFDataset(self.ncurl, aggdim=self.timedim)
except:
self._nc = MFDataset(self.ncurl,)
else:
self._nc = Dataset(self.ncurl[0])
self._ncfiles = self.ncurl
# Set the variables that need downloading
if variables==None and self.type=='ocean':
self.variables = self.oceanvars
elif variables==None and self.type=='atmosphere':
self.variables = self.airvars
else:
self.variables = variables
# For each variable:
# Get the coordinate variables
# Initiate the griddap class for each variable
# (this does most of the work)
self.init_var_grids()
def main():
from netCDF4 import Dataset, MFDataset
from numpy import unique, array
concfiles = ['temp/20161114/aqm.t12z.aconc.ncf']
grid = Dataset('MAY2014/aqm.t12z.grdcro2d.ncf')
concobj = MFDataset(concfiles)
d, index = get_dates(concobj)
jds = array([int(i.strftime('%j')) for i in d])
ujds = unique(jds)[1, 2]
m, x, y = load_conus_basemap(grid)
# get ozone
o3 = concobj.variables['O3'][index, 0, :, :].squeeze() * 1000.
# to make an image for each time loop through
for i, j in enumerate(d):
date = j.strftime('%m/%d/%Y %H')
make_spatial_plot(o3[i, :, :], x, y, date, m, levs=arange(0, 100, 10), cmap='viridis_r'):
plt.savefig(j.strftime('%Y%m%d%H_o3.jpg'), dpi=75)
concobj = MFDataset(['temp/20161114/aqm.t12z.aconc.ncf'])
pm25 = concobj.variables['PM25'][index, 0, :, :].squeeze()
# to make an image for each time loop through
for i, j in enumerate(d):
date = j.strftime('%m/%d/%Y %H')
make_spatial_plot(o3[i, :, :], x, y, date, m, levs=arange(0, 70, 10), cmap='viridis_r'):
plt.savefig(j.strftime('%Y%m%d%H_pm25.jpg'), dpi=75)
def get_data_dict(self):
for year, month in t.product(self.years, self.months):
dir_root = self.dirs[self.years.index(year)]
file_path = os.path.join(dir_root, year, month)
os.chdir(file_path)
for var in self.A1_vars:
file = 'GEOSFP.' + year + month + '**.A1.025x03125.NA.nc'
self.d[year + month + var] = \
MFDataset(file, aggdim='time').variables[var][1::3, self.idx_LAT, self.idx_LON]
print(year + month + var, self.d[year + month + var].shape[0])
for var in self.A3_vars:
file = 'GEOSFP.' + year + month + '**.A3dyn.025x03125.NA.nc'
for level in self.levels:
self.d[year + month + var + level] = \
MFDataset(file, aggdim='time').variables[var][:, int(level), self.idx_LAT, self.idx_LON]
print(year + month + var + level,
self.d[year + month + var + level].shape[0])
for var in self.I3_vars:
file = 'GEOSFP.' + year + month + '**.I3.025x03125.NA.nc'
if var == 'PS':
self.d[year + month + var] = \
MFDataset(file, aggdim='time').variables[var][:, self.idx_LAT, self.idx_LON]
else:
for level in self.levels:
self.d[year + month + var + level] = \
MFDataset(file, aggdim='time').variables[var][:, int(level), self.idx_LAT, self.idx_LON]
print(year + month + var + level,
self.d[year + month + var + level].shape[0])
return self.d
def get_time_nc(nc_file, tv='time'):
"""
returns all timestamps of given netcdf file as datetime list.
:param nc_file: NetCDF file(s)
:param tv: name of temporal dimension
:return format: netcdftime._datetime.datetime
"""
from netCDF4 import MFDataset, num2date
ds = MFDataset(nc_file)
try:
time = ds.variables[tv]
except:
tv = 'time_counter'
ds.close()
try:
ds = MFDataset(nc_file)
time = ds.variables[tv]
if (hasattr(time, 'units') and hasattr(time, 'calendar')) == True:
timestamps = num2date(time[:], time.units, time.calendar)
elif hasattr(time, 'units'):
timestamps = num2date(time[:], time.units)
else:
timestamps = num2date(time[:])
ds.close()
except Exception as e:
raise Exception
return timestamps
def ice_comp_model_to_osi(pathToModel,
modelYear,
modelIteration,
boundLat,
pathToOSI,
param='area',
threshold=0.15):
'''
Plot sea ice area from satellite data and several model iterations
'''
fsat = MFDataset(pathToOSI + '/OSI' + modelYear + '??.nc')
osi_lat = fsat.variables['lat'][:]
osi_lon = fsat.variables['lon'][:]
osi_ice = fsat.variables['ice_conc'][0, :, :]
area_osi = np.ones(osi_ice.shape) * 100
osi_area = []
for mm in range(12):
if param == 'area':
osi_area.append(calc_area(fsat.variables['ice_conc'][mm,:,:]/100,\
area_osi, osi_lat, blat=boundLat, threshold=threshold)/1e6)
elif param == 'extent':
osi_area.append(calc_extent(fsat.variables['ice_conc'][mm,:,:]/100,\
area_osi, osi_lat, blat=boundLat, threshold=threshold)/1e6)
g = Dataset('./grid.cdf')
dxc = g.variables['dxc'][0, :, :]
dyc = g.variables['dyc'][0, :, :]
lat = g.variables['yc'][0, :, :]
dxcXdyc = dxc * dyc
area_model = np.zeros((len(modelIteration), 12))
for (it, iteration) in enumerate(modelIteration):
fm = MFDataset(pathToModel + '/' + modelYear + '/' + 'it' +
str(iteration) + '/fw/*.cdf')
for mm in range(12):
if param == 'area':
area_model[it,mm] = calc_area(fm.variables['area'][mm,:,:],\
dxcXdyc, lat, blat=boundLat,threshold=threshold)/10e11
elif param == 'extent':
area_model[it,mm] = calc_extent(fm.variables['area'][mm,:,:],\
dxcXdyc, lat, blat=boundLat,threshold=threshold)/10e11
fm.close()
dates = pd.date_range(modelYear + '-01',
str(int(modelYear) + 1) + '-01',
freq='M')
dd = pd.DataFrame(index=dates)
dd['Satellite'] = osi_area
for (it, iteration) in enumerate(modelIteration):
dd['it' + str(iteration)] = area_model[it, :]
return dd.plot(figsize=(10, 5), lw=3)
def ice_comp_model_to_osi_table(pathToModel,
modelYears,
modelIteration,
boundLat,
pathToOSI,
param='area',
threshold=0.15):
diff_array = numpy.zeros((len(modelYears), 12))
for (nnum, yyear) in enumerate(modelYears):
fsat = MFDataset(pathToOSI + '/OSI' + yyear + '??.nc')
osi_lat = fsat.variables['lat'][:]
osi_lon = fsat.variables['lon'][:]
osi_ice = fsat.variables['ice_conc'][0, :, :]
area_osi = np.ones(osi_ice.shape) * 100
osi_area = []
for mm in range(12):
if param == 'area':
osi_area.append(calc_area(fsat.variables['ice_conc'][mm,:,:]/100,\
area_osi, osi_lat, blat=boundLat, threshold=threshold)/1e6)
elif param == 'extent':
osi_area.append(calc_extent(fsat.variables['ice_conc'][mm,:,:]/100,\
area_osi, osi_lat, blat=boundLat, threshold=threshold)/1e6)
g = Dataset('./grid.cdf')
dxc = g.variables['dxc'][0, :, :]
dyc = g.variables['dyc'][0, :, :]
lat = g.variables['yc'][0, :, :]
dxcXdyc = dxc * dyc
area_model = np.zeros((len(modelIteration), 12))
if modelIteration[0] == 'last':
gg = glob.glob(pathToModel + '/' + yyear + '/' + 'it*')
gg.sort()
lastit = [int(gg[-1].split('/')[-1].split('t')[-1])]
else:
lastit = modelIteration
for (it, iteration) in enumerate(lastit):
fm = MFDataset(pathToModel + '/' + yyear + '/' + 'it' +
str(iteration) + '/fw/*.cdf')
for mm in range(12):
if param == 'area':
area_model[it,mm] = calc_area(fm.variables['area'][mm,:,:],\
dxcXdyc, lat, blat=boundLat)/10e11
elif param == 'extent':
area_model[it,mm] = calc_extent(fm.variables['area'][mm,:,:],\
dxcXdyc, lat, blat=boundLat)/10e11
fm.close()
diff_array[nnum, :] = area_model[0, :] - osi_area[:]
return diff_array
def ice_comp_model_to_sat_table_rm(pathToModel, modelYears, modelIteration,\
boundLat, pathToOSI, param):
diff_array = numpy.zeros((len(modelYears), 12))
for (nnum, yyear) in enumerate(modelYears):
g = Dataset('./grid.cdf')
dxc = g.variables['dxc'][0, :, :]
dyc = g.variables['dyc'][0, :, :]
lat = g.variables['yc'][0, :, :]
topo = g.variables['topo'][0, :, :]
dxcXdyc = dxc * dyc
#area_model=np.zeros((len(modelIteration), 12))
if modelIteration[0] == 'last':
gg = glob.glob(pathToModel + '/' + yyear + '/' + 'it*')
gg.sort()
lastit = [int(gg[-1].split('/')[-1].split('t')[-1])]
else:
lastit = modelIteration
for (it, iteration) in enumerate(lastit):
fm = MFDataset(pathToModel + '/' + yyear + '/' + 'it' +
str(iteration) + '/fw/*.cdf')
fsat = MFDataset(pathToOSI + yyear + '??.nc')
for mm in range(12):
if param == 'area':
aa_model = np.ma.filled(fm.variables['area'][mm, :, :],
0) * dxcXdyc
bb_satel = (fsat.variables['ice'][mm, :, :]) * dxcXdyc
cc_diff = aa_model - bb_satel
diff_array[nnum, mm] = np.sqrt(cc_diff**2).sum()
if param == 'extent':
dmodel = np.ma.filled(fm.variables['area'][mm, :, :], 0)
dmodel[dmodel < 0.15] = 0
dmodel[dmodel >= 0.15] = 1
aa_model = dmodel * dxcXdyc
dsat = fsat.variables['ice'][mm, :, :]
dsat[dsat < 0.15] = 0
dsat[dsat >= 0.15] = 1
bb_satel = dsat * dxcXdyc
cc_diff = aa_model - bb_satel
diff_array[nnum, mm] = np.sqrt(cc_diff**2).sum()
fm.close()
fsat.close()
return diff_array
def get_data(path):
if os.path.isfile(path+'/ocean.stats.nc'):
if os.path.isfile(path+'/out1/ocean.stats.nc'):
s=MFDataset(path+'/out?/ocean.stats.nc')
else:
s=Dataset(path+'/ocean.stats.nc')
else:
s=MFDataset(path+'/out?/ocean.stats.nc')
print 'Reading exp', path
tmp = 365*2 # 2 years
dS_dt=(s.variables['Salt'][-1]-s.variables['Salt'][-tmp])*0.5 # kg/year
dT_dt=(s.variables['Heat'][-1]-s.variables['Heat'][-tmp])*0.5 # J/year
s.close()
return dS_dt, dT_dt
def runTest(self):
# The test files have no calendar attribute on the time variable.
calendar = 'standard'
# Get the real dates
# skip this until cftime pull request #55 is in a released
# version (1.0.1?). Otherwise, fix for issue #808 breaks this
if parse_version(cftime.__version__) >= parse_version('1.0.1'):
dates = []
for file in self.files:
f = Dataset(file)
t = f.variables['time']
dates.extend(num2date(t[:], t.units, calendar))
f.close()
# Compare with the MF dates
f = MFDataset(self.files, check=True)
t = f.variables['time']
T = MFTime(t, calendar=calendar)
assert_equal(T.calendar, calendar)
assert_equal(len(T), len(t))
assert_equal(T.shape, t.shape)
assert_equal(T.dimensions, t.dimensions)
assert_equal(T.typecode(), t.typecode())
# skip this until cftime pull request #55 is in a released
# version (1.0.1?). Otherwise, fix for issue #808 breaks this
if parse_version(cftime.__version__) >= parse_version('1.0.1'):
assert_array_equal(num2date(T[:], T.units, T.calendar), dates)
assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366)
f.close()
# Test exception is raised when no calendar attribute is available on the
# time variable.
with MFDataset(self.files, check=True) as ds:
with self.assertRaises(ValueError):
MFTime(ds.variables['time'])
# Test exception is raised when the calendar attribute is different on the
# variables. First, add calendar attributes to file. Note this will modify
# the files inplace.
calendars = ['standard', 'gregorian']
for idx, f in enumerate(self.files):
with Dataset(f, 'a') as ds:
ds.variables['time'].calendar = calendars[idx]
with MFDataset(self.files, check=True) as ds:
with self.assertRaises(ValueError):
MFTime(ds.variables['time'])
def get_variable(resources):
"""Guess main variables in a NetCDF file.
(compare nc.ocg_utils.get_variable)
:param resources: netCDF4.Dataset
:return list: names of main variables
Notes
-----
The main variables are the one with highest dimensionality and size. The
time, lon, lat variables and variables that are defined as bounds are
automatically ignored.
"""
if type(resources) == str:
ncdataset = Dataset(resources)
elif type(resources) == list:
ncdataset = MFDataset(resources)
else:
ncdataset = resources
# dims = [key for key in ncdataset.dimensions.keys()]
vars = [key for key in ncdataset.variables.keys()]
# assume that main variables are 3D or 4D
main_vars = []
for var in vars:
if len(ncdataset.variables[var].dimensions) >= 3:
main_vars.append(var)
if len(main_vars) > 1:
LOGGER.exception('more than one 3D or 4D variable in file')
if len(main_vars) == 0:
LOGGER.exception('No 3D or 4D variable detected')
return main_vars[0]
def get_data(path):
if os.path.isfile(path + '/prog.nc'):
if os.path.isfile(path + '/out1/prog.nc'):
s = MFDataset(path + '/out?/prog.nc')
tmp = 730
else:
s = Dataset(path + '/prog.nc')
tmp = len(s.variables['time'][:])
else:
s = MFDataset(path + '/out?/prog.nc')
tmp = 730
print 'Reading exp', path
MLD = (s.variables['ePBL_h_ML'][-tmp::, :]).mean() # m
s.close()
return MLD
def read_many_files(filenames, usr_variables, dim=None):
"""
Reads a single Variable from many NetCDF files. This method uses the netCDF4 MFDataset class and so is NOT
suitable for NetCDF4 datasets (only 'CLASSIC' netcdf).
:param filenames: A list of NetCDF filenames to read, or a string with wildcards.
:param usr_variables: A list of variable (dataset) names to read from the files.
The names must appear exactly as in in the NetCDF file.
:param dim: The name of the dimension on which to aggregate the data. None is the default
which tries to aggregate over the unlimited dimension
:return: A list of variable instances constructed from all of the input files
"""
from netCDF4 import MFDataset
from cis.exceptions import InvalidVariableError
usr_variables = listify(usr_variables)
try:
datafile = MFDataset(filenames, aggdim=dim)
except RuntimeError as e:
raise IOError(e)
data = {}
for variable in usr_variables:
# Get data.
try:
data[variable] = datafile.variables[variable]
except:
raise InvalidVariableError(
'Variable {} not found in file {}.'.format(
variable, filenames))
return data
def read_ecmwf_data(self):
"""
功能：读取ecmwf数据
返回
:return: self.data
"""
if isinstance(self.data_path, str):
print("读取ECMWF文件：{}".format(self.filename))
nc_file = Dataset(self.data_path)
elif isinstance(self.data_path, list):
# 读取多文件
print("读取ecmwf文件：{}".format(self.filenames))
nc_file = MFDataset(self.data_path)
else:
raise ValueError("不支持输入的格式")
# 读取各种变量，并存储
# 选取范围
lon_temp = nc_file.variables['longitude'][:]
lat_temp = nc_file.variables['latitude'][:]
lat_left_index = np.where(lat_temp <= self.extents[3])[0][0]
lat_right_index = np.where(lat_temp >= self.extents[2])[0][-1]
lon_left_index = np.where(lon_temp >= self.extents[0])[0][0]
lon_right_index = np.where(lon_temp <= self.extents[1])[0][-1]
lat = lat_temp[lat_left_index:lat_right_index + 1]
lon = lon_temp[lon_left_index:lon_right_index + 1]
if 'lon' in self.variables:
setattr(self.data, 'lon', lon)
if 'lat' in self.variables:
setattr(self.data, 'lat', lat)
if 'time' in self.variables:
time_temp = nc_file.variables['time'][:]
time_date = num2date(time_temp, nc_file.variables['time'].units)
time = pltdate.date2num(time_date)
x, y = np.unique(time, return_index=True)
setattr(self.data, 'time', x)
if 'wind' in self.variables:
u10 = nc_file.variables['u10'][y,
lat_left_index:lat_right_index + 1,
lon_left_index:lon_right_index + 1]
setattr(self.data, 'u10', u10)
v10 = nc_file.variables['v10'][y,
lat_left_index:lat_right_index + 1,
lon_left_index:lon_right_index + 1]
setattr(self.data, 'v10', v10)
if 'sst' in self.variables:
sst = nc_file.variables['sst'][y,
lat_left_index:lat_right_index + 1,
lon_left_index:lon_right_index + 1]
setattr(self.data, 'sst', sst - 273.15)
if 't2m' in self.variables:
t2m = nc_file.variables['t2m'][y,
lat_left_index:lat_right_index + 1,
lon_left_index:lon_right_index + 1]
setattr(self.data, 't2m', t2m - 273.15)
if 'slp' in self.variables:
slp = nc_file.variables['sp'][y,
lat_left_index:lat_right_index + 1,
lon_left_index:lon_right_index + 1]
setattr(self.data, 'slp', slp)
def get_area_avg_ts(self,
lake_mask,
start_year,
end_year,
varname="soicecov"):
series_list = []
i_arr, j_arr = np.where(lake_mask)
for y in range(start_year, end_year + 1):
with MFDataset(self.year_to_path[y]) as ds:
time_var = ds.variables["time_counter"]
data_var = ds.variables[varname]
time = num2date(time_var[:], time_var.units)
data = data_var[:]
data = data[:, j_arr, i_arr].mean(axis=1)
series_list.append(pd.Series(index=time, data=data))
series = pd.concat(series_list)
assert isinstance(series, pd.Series)
years = series.index.map(lambda d: d.year)
series.drop(series.index[(years > end_year) | (years < start_year)],
inplace=True)
return series
def read_raw_data(start_year, final_year):
""""Read ERA5 wind data for adjacent years.
Args:
start_year (int): Read data starting from this year.
final_year (int): Read data up to this year.
Returns:
tuple of MFDataset, ndarray, ndarray, ndarray, and ndarray: Tuple containing reading object of multiple wind
data (netCDF) files, longitudes of grid, latitudes of grid, model level numbers, and timestamps in hours since
1900-01-01 00:00:0.0.
"""
# Construct the list of input NetCDF files
netcdf_files = []
for y in range(start_year, final_year + 1):
for m in range(1, 13):
netcdf_files.append(
path_join(era5_data_dir, wind_file_name_format.format(y, m)))
# Load the data from the NetCDF files.
nc = MFDataset(netcdf_files)
# Read the variables from the netCDF file.
lons = nc.variables['longitude'][:]
lats = nc.variables['latitude'][:]
levels = nc.variables['level'][:] # Model level numbers.
hours = nc.variables[
'time'][:] # Hours since 1900-01-01 00:00:0.0, see: print(nc.variables['time']).
return nc, lons, lats, levels, hours
def runTest(self):
# Get the real dates
# skip this until cftime pull request #55 is in a released
# version (1.0.1?). Otherwise, fix for issue #808 breaks this
if parse_version(cftime.__version__) >= parse_version('1.0.1'):
dates = []
for file in self.files:
f = Dataset(file)
t = f.variables['time']
dates.extend(cftime.num2date(t[:], t.units, t.calendar))
f.close()
# Compare with the MF dates
f = MFDataset(self.files, check=True)
t = f.variables['time']
mfdates = cftime.num2date(t[:], t.units, t.calendar)
T = MFTime(t)
assert_equal(len(T), len(t))
assert_equal(T.shape, t.shape)
assert_equal(T.dimensions, t.dimensions)
assert_equal(T.typecode(), t.typecode())
# skip this until cftime pull request #55 is in a released
# version (1.0.1?). Otherwise, fix for issue #808 breaks this
if parse_version(cftime.__version__) >= parse_version('1.0.1'):
assert_array_equal(cftime.num2date(T[:], T.units, T.calendar),
dates)
assert_equal(cftime.date2index(datetime.datetime(1980, 1, 2), T), 366)
f.close()
def getGrid(self):
"""
get the grid info
"""
filelist = []
for t in self.daterange:
filestr = '%s/%s/NARR-%s_10_m_above_gnd-%s.nc' % (
self.basedir, self.variables[0], self.variables[0], t)
filelist.append(filestr)
#pdb.set_trace()
vn = MFDataset(filelist, 'r')
# print nc
tt = vn.variables['time']
timei = num2date(tt[:], tt.units)
self.time = timei
self.Nt = len(self.time)
print 'Downloading NARR wind from %s to %s\n'%(datetime.strftime(timei[0],\
'%Y-%m-%d %H:%M:%S'), datetime.strftime(timei[-1], '%Y-%m-%d %H:%M:%S'))
lon = vn.variables['longitude'][:]
lat = vn.variables['latitude'][:]
#bbox = [-97.458207, -96.72, 27.368078, 28.291049]
#25.868306, -97.954244
#27.551320, -96.905049
bbox = [-97.900, -96.91, 25.868306, 27.551320]
self.ind = []
for i in range(lon.shape[0]):
if bbox[2] < lat[i] < bbox[3] and bbox[0] < lon[i] < bbox[1]:
self.ind.append(i)
self.lon = lon[self.ind]
self.lat = lat[self.ind]
def load_netcdf4_file(filein=None, visObj=None):
""" Function doc """
#from netCDF4 import Dataset
#f = Dataset(filein)
#try:
from netCDF4 import MFDataset
f = MFDataset(filein)
#except:
# import netCDF4 as nc
# f= nc.Dataset(filein)
frames = []
for netcdf_frame in f.variables["coordinates"][:]:
try:
frame = []
for coords in netcdf_frame:
frame.append(coords[0])
frame.append(coords[1])
frame.append(coords[2])
#print (coords)
frame = np.array(frame, dtype=np.float32)
frames.append(frame)
frame = []
except:
print('netcdf error')
print('frames netcdf:', frames[0])
return frames
def __init__(self, *args, **kwargs):
verbose = kwargs.get('verbose', True)
aggdim = kwargs.get('aggdim', default_aggdim)
self.use_xarray = kwargs.get('use_xarray', have_xarray)
if self.use_xarray:
desc = 'with xarray'
else:
desc = 'with netcdf'
Nfiles = len(args)
self.filelist = []
for fpath in [fpath for fpath in args if os.path.isfile(fpath)]:
try:
Dataset(fpath)
except (IOError, OSError): # NetCDF: Unknown file format
pass
else:
self.filelist.append(fpath)
if self.use_xarray:
nc = xarray.open_mfdataset(self.filelist, concat_dim=aggdim)
self.Nt, self.Nz, self.Ny, self.Nx = nc.variables['U'].shape
self.Nx -= 1 # U is staggered in x
else:
nc = MFDataset(self.filelist, aggdim=aggdim)
self.Nt = len(nc.dimensions['time'])
self.Nx = len(nc.dimensions['west_east'])
self.Ny = len(nc.dimensions['south_north'])
self.Nz = len(nc.dimensions['bottom_top'])
self.varlist = list(nc.variables)
self._read_vars(nc)
def get_root(self):
self.link = get_thredds_opendap_link('hf_radar', 'hf_radar_ibiza-scb_codarssproc001', 1, 'dep0001',
'hf-radar-ibiza_scb-codarssproc001', self.year, self.month)
try:
self.root = Dataset(self.link)
except (RuntimeError, IOError):
logger.error('File does not exist. ' + self.link, exc_info=False)
buoy_definitions = c.settings.buoy_paths
buoy_links = []
for i in range(0, len(buoy_definitions['folder'])):
cur_folder = buoy_definitions['folder'][i]
cur_sub_folder = buoy_definitions['sub_folder'][i]
cur_station_name = buoy_definitions['station_name'][i]
globString = "/data/current/opendap/observational/mooring/currentmeter/" + cur_sub_folder + "/L1/" + str(self.year) + "/*" + str(self.year) + "-" + str(self.month).zfill(2) + ".nc"
opendapContents = glob.glob(globString)
logger.info('FILE LIST ' + str(opendapContents))
if opendapContents is None or not opendapContents:
continue
try:
self.buoy_root = MFDataset(globString)
except RuntimeError:
logger.info('Dataset Runtime Error. ', exc_info=False)
if self.buoy_root is None:
logger.error('No buoy root loaded for {0} {1}. Will skip this month.'.format(self.year, self.month))
raise RuntimeError('Check the buoy input. Dataset with month/year file may not exist.')
def get_index_lat(resource, variable=None):
"""
returns the dimension index of the latiude values
:param resource: list of path(s) to netCDF file(s) of one Dataset
:param variable: variable name
:return int: index
"""
if variable is None:
variable = get_variable(resource)
if type(resource) != list:
resource = [resource]
if len(resource) == 1:
ds = Dataset(resource[0])
else:
ds = MFDataset(resource)
var = ds.variables[variable]
dims = list(var.dimensions)
if 'rlat' in dims:
index = dims.index('rlat')
if 'lat' in dims:
index = dims.index('lat')
if 'latitude' in dims:
index = dims.index('latitude')
if 'y' in dims:
index = dims.index('y')
return index
def get_values(resource, variable=None):
"""
returns the values for a list of files of files belonging to one dataset
:param resource: list of files
:param variable: variable to be picked from the files (if not set, variable will be detected)
:returs numpy.array: values
"""
from numpy import squeeze
if variable is None:
variable = get_variable(resource)
# Python 2-3
try:
basestring
except NameError:
basestring = str
if isinstance(resource, basestring):
ds = Dataset(resource)
elif len(resource) == 1:
ds = Dataset(resource)
else:
ds = MFDataset(resource)
vals = squeeze(ds.variables[variable][:])
return vals
def __init__(self, myFile):
CoverageReader.__init__(self,myFile);
if os.path.isfile(self.filename):
self.ncfile = Dataset(self.filename, 'r')
elif os.path.isdir(self.filename):
self.ncfile = MFDataset(os.path.join(self.filename, "*.nc"), 'r')
def get_calendar(resource, variable=None):
"""
returns the calendar and units in wich the timestamps are stored
:param resource: netCDF file or files of one Dataset
:return str: calendar, unit
"""
if type(resource) != list:
resource = [resource]
try:
if len(resource) > 1:
ds = MFDataset(resource)
else:
ds = Dataset(resource[0])
time = ds.variables['time']
except:
msg = 'failed to get time'
LOGGER.exception(msg)
raise Exception(msg)
if hasattr(time, 'units') is True:
unit = time.units
else:
unit = None
if hasattr(time, 'calendar') is True:
calendar = time.calendar
else:
calendar = None
return str(calendar), str(unit)
def get_nc(dloc,name,var_list,lev_list,years,months):
from netCDF4 import Dataset, MFDataset,num2date,date2num
file_list=[]
for year in years:
for month in months:
if name=='aerosol-concentration':
file_list.append(dloc + '%s/ace-cpc_summit_%s%s_%s_v1.nc'%(name,year,str(month).zfill(2),name))
else:
file_list.append(dloc + '%s/ace-tower_summit_%s%s_%s_v1.nc'%(name,year,str(month).zfill(2),name))
if len(file_list)>1:
nc = MFDataset(file_list,'r',aggdim='time')
elif len(file_list)==1:
nc = Dataset(file_list[0],'r')
else:
print('No data')
times = pd.to_datetime(nc.variables['time'][:],origin='unix',unit='s')
all_vars=[]
for i in range(0,len(var_list)):
if lev_list[i]==-1:
all_vars.append(nc.variables[var_list[i]][:])
else:
all_vars.append(nc.variables[var_list[i]][:,lev_list[i]])
return times, all_vars
def get_seasonal_mean_lst(self,
start_year=None,
end_year=None,
season_to_months=None):
"""
:param start_year:
:param end_year:
:param season_to_months:
:return: dict(year -> season -> field)
"""
import pandas as pd
month_to_season = defaultdict(lambda: "no-season")
for season, months in season_to_months.items():
for the_month in months:
month_to_season[the_month] = season
print("months_to_season = {}".format(month_to_season))
result = {}
for the_year in range(start_year, end_year + 1):
result[the_year] = {}
data_path = self.year_to_path[the_year]
with MFDataset(data_path) as ds:
# sst = ds.variables["isstempe"][:]
# ist = ds.variables["isnotem2"][:]
# ice_f = ds.variables["iiceconc"][:]
#
# if hasattr(ice_f, "mask"):
# ice_f[ice_f.mask] = 0
# Calculate lake surface temperature
# lst = sst * (1.0 - ice_f) + ist * ice_f
lst = ds.variables["sosstsst"][:]
time_var = ds.variables["time_counter"]
dates = num2date(time_var[:], time_var.units).tolist()
print(", ".join([str(d) for d in dates[:10]]) +
", ..., {}".format(dates[-1]))
panel = pd.Panel(data=lst,
items=dates,
major_axis=range(lst.shape[1]),
minor_axis=range(lst.shape[2]))
seasonal_panel = panel.groupby(
lambda d: month_to_season[d.month], axis="items").mean()
for the_season in season_to_months:
print(seasonal_panel)
# in the files the dimensions are ordered as (t, y, x) -> hence the transpose below
result[the_year][the_season] = seasonal_panel[
the_season, :, :].values.T
return result
def get_time_range(files, time_range=None, temporal_var_name='time'):
'''
If time_range is None, this function will get a time range from input files.
Else, this function will adjust the time_range by setting hour value to datetime.datetime objects.
:param files: netCDF file(s) (including OPeNDAP URL(s))
:type files: list of str
:param time_range: time_range (default: None)
:type time_range: list two datetime objects: [begin, end]
:param temporal_var_name: name of temporal variable from netCDF file (default: "time")
:type temporal_var_name: str
:rtype
Returns a time range: a list two datetime objects: [begin, end], where "begin" is the first date, and "end" is the last.
'''
try:
nc = Dataset(files[0], 'r')
except RuntimeError:
raise MissingIcclimInputError("Failed to access dataset: " + files[0])
time = nc.variables[temporal_var_name]
try:
calend = time.calendar
except:
calend = 'gregorian'
units = time.units
t = netcdftime.utime(units, calend)
any_dt = t.num2date(time[0])
nc.close()
if time_range != None:
time_range = harmonize_hourly_timestamp(time_range, calend, any_dt)
else:
try:
nc = MFDataset(files, 'r', aggdim='time')
except RuntimeError:
raise MissingIcclimInputError("Failed to access dataset: " + files)
time_arr = nc.variables[temporal_var_name][:]
nc.close()
begin_num = min(time_arr)
end_num = max(time_arr)
begin_dt = num2date(begin_num, calend, units)
end_dt = num2date(end_num, calend, units)
time_range = [begin_dt, end_dt]
return time_range
def update(self, FileName):
'''
Change nc Dataset to point to a new nc file or url without reinitializing everything (retain grid info)
'''
if isinstance(FileName, list):
self.Dataset = MFDataset(FileName)
else:
self.Dataset = Dataset(FileName)
