def read_ncdf(name,path='/work/gg0877/KST/tide_gaughes/ncdf',origin='2012-01-01 00:00:00'):
ncname = '_'.join(word[0].upper()+word[1:] for word in name.split('_'))
if path[-6:]=='hawaii':
nc = netCDF4.MFDataset(path+'/'+ncname+'*.nc')
else:
nc = netCDF4.Dataset(path+'/'+ncname+'.nc')
ncv = nc.variables
ot = utime('seconds since '+origin)
ut = utime(ncv['time'].units)
time = ot.date2num(ut.num2date(ncv['time'][:]))
print(ncv.keys())
scale = {'cm':100.,'m':1.0,'millimeters':1000.,'mm':1000.}
if 'elev' in ncv:
if 'units' in ncv['elev'].ncattrs():
scale_factor = scale[ncv['elev'].units]
else:
if ncv['elev'][:].std()>100.:
scale_factor=1000.
elif ncv['elev'][:].std()>10.:
scale_factor=100.
else:
scale_factor=1.0
elev = ncv['elev'][:].squeeze()/scale_factor
elif 'sea_surface_height_above_reference_level' in ncv:
if 'units' in ncv['sea_surface_height_above_reference_level'].ncattrs():
scale_factor = scale[ncv['sea_surface_height_above_reference_level'].units]
else:
scale_factor = 1000.
elev = ncv['sea_surface_height_above_reference_level'][:].squeeze()/scale_factor
nc.close()
return time,elev
def create_dataset_timeunits_cmam(file1, newfile1):
"""
This function take two arguments as file names of netcdf files
file1: is a netcdf with time units to change
newfile1: is a name for a new netcdf file that will be the file1
but with the units 'days since 1850-01-01 00:00:00'
and calendar standard.
"""
new_time_units = 'days since 1850-01-01 00:00:00'
src = Dataset(file1, 'r')
dst = Dataset(newfile1, 'w')
cmam_time_units = src.variables['time'].units
for namedimen in src.dimensions.keys():
dimension = src.dimensions[namedimen]
if not dimension.isunlimited():
dst.createDimension(namedimen, size=len(dimension))
else:
dst.createDimension(namedimen, size=None)
for varname in src.variables.keys():
variable = src.variables[varname]
if varname == 'some_variable':
continue
if varname == 'time':
x = dst.createVariable(varname,
variable.datatype,
dimensions=variable.dimensions)
x.units = new_time_units
x.calendar = variable.calendar
time_obj = utime(
str(cmam_time_units.replace('00:00',
'00:00:00')).replace('"', ''))
time_x = time_obj.num2date(variable[:])
new_time_obj = utime(new_time_units)
new_cmam_tim = new_time_obj.date2num(time_x)
x[:] = new_cmam_tim
else:
x = dst.createVariable(varname,
variable.datatype,
dimensions=variable.dimensions)
x[:] = src.variables[varname][:]
src.close()
dst.close()
return
def units_checker(units):
try:
u = utime(units)
return u.unit_string
except ValueError, TypeError:
msg = 'Invalid time units format - Available format is "{}"'.format(TIME_UNITS_FORMAT)
raise ArgumentTypeError(msg)
def read_Elgar_pressure_data(flow_data):
print ' > Elgar pressure data;'
dir1 = data_dir + '/elgar/*_filter_abs.nc'
flist = glob.glob(dir1)
flist.sort()
for filename in flist[:]:
ncf = netCDF4.Dataset(filename,'r')
ncvar = ncf.variables
time = ncvar['time']
utime = netcdftime.utime(time.units)
dates = utime.num2date(time[:])
sta_name = filename.split('/')[-1][:3]
if 'p' in sta_name or 'q' in sta_name:
print ' > read > Station name: ', sta_name
water_depth = ncvar['water_depth'][:]
data = pd.DataFrame(data = water_depth, columns = ['water_depth'], index = dates)
data = data.dropna()
data = data.resample('H') #hourly mean
date_tmp = ps.datetime64todatetime(data.index.values)
datenum = [ps.datetime2datenum(datei) for datei in date_tmp]
flow_data[sta_name]['name_long'] = 'Bottom pressure '+sta_name
flow_data[sta_name]['lat'] = ncvar['lat'][:]
flow_data[sta_name]['lon'] = ncvar['lon'][:]
flow_data[sta_name]['elev'] = (data['water_depth'] - data['water_depth'].mean()).values
flow_data[sta_name]['date'] = date_tmp
flow_data[sta_name]['datenum_mat'] = datenum
def num2date(tnum,tunits,calendar='standard'): from netcdftime import utime #return utime(tunits,calendar).num2date(tnum) # for numpy 1.15.3 there are problems with masked arrays without mask! # for some reason, removing mask or dividing by 1 solves the problem!!! return utime(tunits,calendar).num2date(tnum/1)
def read_ncom(flow_data):
print ' > NCOM;'
filename = data_dir + '/ncom/ncom_glb_regp01_2012-3-4-5.nc'
ncf = netCDF4.Dataset(filename,'r')
ncvar = ncf.variables
time = ncvar['time']
utime = netcdftime.utime(time.units)
dates = utime.num2date(time[:])
sta_name = filename.split('/')[-1][:3]
lona,lata = np.meshgrid(ncvar['lon'][:],ncvar['lat'][:])
dist = ((lona - lon_orig)**2 + (lata - lat_orig)**2)
[i,j] = np.where(dist==dist.min())
elev = ncvar['surf_el'][:,i,j].flatten()
data = pd.DataFrame(data = elev, columns = ['elev'], index = dates)
data = data.dropna()
data = data.resample('H') #hourly mean
date_tmp = ps.datetime64todatetime(data.index.values)
datenum = [ps.datetime2datenum(datei) for datei in date_tmp]
flow_data[sta_name]['name_long'] = 'Elevation from NCOM model '
flow_data[sta_name]['lat'] = lata[i,j].item()
flow_data[sta_name]['lon'] = lona[i,j].item()
flow_data[sta_name]['elev'] = data['elev'].values
flow_data[sta_name]['date'] = date_tmp
flow_data[sta_name]['datenum_mat'] = datenum
def date2num(dt, calend, units):
'''
type dt: datetime object
'''
t = utime(units, calend)
dt_num = t.date2num(dt)
return dt_num
def _fix_netcdftime_datetime(dt, unit):
# Fix netcdftime bug: netcdftime.datetime(2000, 03, 31).dayofyr
# Happens to work with datetime objects too.
fix_unit = "days since {}".format(str(dt))
fix_utime = netcdftime.utime(fix_unit, calendar=unit.calendar)
dt = fix_utime.num2date(0)
return dt
def extract_netcdf_time(cls, bandName, calendar):
"""Convert netcdf time to datetime using appropriate library"""
from netcdftime import utime
epoch, units = cls.extract_epoch_units(bandName)
cdftime = utime(units, calendar)
timestamps = cdftime.num2date([epoch])
return timestamps[0]
def get_list_dates_from_nc(nc, type_dates):
'''
Returns list of dates from NetCDF dataset.
:param nc: NetCDF dataset
:type nc: netCDF4.Dataset
:param type_dates: type of dates ('dt' for datetime objects, 'num' for float objects)
:type type_dates: str
:rtype: list of datetime/float
'''
var_time = nc.variables['time']
time_units = var_time.units # str (ex.: 'days since 1850-01-01 00:00:00')
try:
time_calend = var_time.calendar # str (ex.: 'standard'/'gregorian'/...)
except:
time_calend = 'gregorian'
if type_dates == 'num':
arr_dt = var_time[:]
list_dt = arr_dt.tolist() # numpy array -> list
if type_dates == 'dt':
t = netcdftime.utime(time_units, time_calend)
arr_dt = t.num2date(var_time[:])
list_dt = arr_dt.tolist() # numpy array -> list
del arr_dt
return list_dt
def plot_time_series(data_in, param_in, title_in):
'''
plot_time_series ...
'''
# Read time and param vars
time = data_in.variables['time'][:]
param = data_in.variables[param_in][:]
# Scale var
[scal_req, scale_factor, add_offset] = findScaleOffset(data_in, param_in)
param_scaled = (scale_factor*param)+add_offset
# create time vector
time_uni = data_in.variables['time'].units
time_cal = data_in.variables['time'].calendar
cdftime = utime(time_uni, calendar=time_cal)
date = [cdftime.num2date(t) for t in time]
plt.figure()
plt.plot(date, param_scaled[:, 0, 0], c='r')
plt.ylabel("%s (%s)" % (data_in.variables[param_in].long_name,
data_in.variables[param_in].units))
plt.ticklabel_format(useOffset=False, axis='y')
plt.xlabel("Time")
plt.title('Maximum '+data_in.variables[param_in].long_name + ' in the '+region+' region')
plt.grid()
def read_d3d_time(netcdf_vars):
"""
Read time from netcdf file and return Datetime vector
"""
utime = netcdftime.utime(netcdf_vars['time'].units)
dates = utime.num2date(netcdf_vars['time'][:])
return dates
def conTimes(time_str="",calendar="",times=np.empty(1),safe=False):
"""
This function takes a time string (e.g. hours since 0001-01-01 00:00:00),
along with a numpy array of times (hours since base) and returns: year, month, day.
Inefficient at present (with mulitple vectorize calls), but sufficient
"""
timeObj=utime(time_str,calendar=calendar)
# set up the functions
f_times = np.vectorize(lambda x: timeObj.num2date(x))
f_year = np.vectorize(lambda x: x.year)
f_mon = np.vectorize(lambda x: x.month)
f_day = np.vectorize(lambda x: x.day)
f_hour = np.vectorize(lambda x: x.hour)
# Call them
pyTimes = f_times(times); year = f_year(pyTimes); mon = f_mon(pyTimes)
day = f_day(pyTimes); hour = f_hour(pyTimes)
# check that 'proper' datetime has been returned:
if safe:
year=np.atleast_1d(year); mon=np.atleast_1d(mon); day=np.atleast_1d(day)
pyTimes = np.array([datetime(year[ii],mon[ii],day[ii]) for ii in \
range(len(year))])
return year,mon,day,hour,pyTimes
def __init__(self,ncfile='na_aviso_2012.nc'):
self.sshname='adt_mean'
self.lonname='lon'
self.latname='lat'
self.timename='time'
nc = netCDF4.Dataset(ncfile)
sv = nc.variables
self.ncv = sv
xslice=slice(None)
yslice=slice(None)
self.lon = sv[self.lonname][xslice]
self.lat = sv[self.latname][yslice]
self.lon2,self.lat2 = np.meshgrid(self.lon,self.lat)
self.time = sv[self.timename][:]
self.timeunits = sv[self.timename].units
self.ut = netcdftime.utime(self.timeunits)
self.dates = self.ut.num2date(self.time)
self.ssh = sv[self.sshname][yslice,xslice]
print(' build spatial tree')
self.mask = self.ssh.mask.squeeze()
self.water = where(self.mask==False)
vlon2 = self.lon2[self.water]
vlat2 = self.lat2[self.water]
self.tree = cKDTree(zip(vlon2,vlat2))
def plot_time_series(data_in, param_in, region):
'''
plot_time_series of RCP4.5 scenario ...
'''
# Read time and param vars
time = data_in.variables['time'][:]
param = data_in.variables[param_in][:]
# Scale var
[scal_req, scale_factor, add_offset] = findScaleOffset(data_in, param_in)
param_scaled = (scale_factor*param)+add_offset
# create time vector
time_uni = data_in.variables['time'].units
time_cal = data_in.variables['time'].calendar
cdftime = utime(time_uni, calendar=time_cal)
date = [cdftime.num2date(t) for t in time]
# ############# A plot of field mean ##############
plt.figure(region+' '+param_in, figsize=(15, 6))
plt.plot(date, param_scaled[:, 0, 0], label=model) # PLOT ORIGINAL DATA
# ############ PLOT SMOOTHED ###############
# window = 8 # date [x:-y], where x+y = window - 1
# param_scaled_smoothed = moving_average(arr=param_scaled[:, 0, 0], win=window)
# plt.plot(date[4:-3], param_scaled_smoothed, label=model) # Plot data smoothed
plt.ylabel("%s (%s)" % (data_in.variables[param_in].long_name,
data_in.variables[param_in].units))
plt.ticklabel_format(useOffset=False, axis='y')
plt.xlabel("Time")
plt.title('RCP 8.5 - Annual Average '+data_in.variables[param_in].long_name
+ ' in the '+region+' region', fontweight='bold')
def get_list_dates_from_nc(nc, type_dates):
'''
Returns list of dates from NetCDF dataset.
:param nc: NetCDF dataset
:type nc: netCDF4.Dataset
:param type_dates: type of dates ('dt' for datetime objects, 'num' for float objects)
:type type_dates: str
:rtype: list of datetime/float
'''
var_time = nc.variables['time']
time_units = var_time.units # str (ex.: 'days since 1850-01-01 00:00:00')
try:
time_calend = var_time.calendar # str (ex.: 'standard'/'gregorian'/...)
except:
time_calend = 'gregorian'
if type_dates == 'num':
arr_dt = var_time[:]
list_dt = arr_dt.tolist() # numpy array -> list
if type_dates == 'dt':
t = netcdftime.utime(time_units, time_calend)
arr_dt = t.num2date(var_time[:])
list_dt = arr_dt.tolist() # numpy array -> list
del arr_dt
return list_dt
def num2date(tnum, tunits, calendar='standard'):
from netcdftime import utime
#return utime(tunits,calendar).num2date(tnum)
# for numpy 1.15.3 there are problems with masked arrays without mask!
# for some reason, removing mask or dividing by 1 solves the problem!!!
return utime(tunits, calendar).num2date(tnum / 1)
def get_list_dates(ifile, type_dates):
'''
Returns list of dates from one file.
:param ifile: NetCDF file
:type ifile: str
:param type_dates: type of dates ('dt' for datetime objects, 'num' for float objects)
:type type_dates: str
:rtype: list of datetime/float
'''
nc = Dataset(ifile, 'r')
var_time = nc.variables['time']
time_units = var_time.units # str (ex.: 'days since 1850-01-01 00:00:00')
time_calend = var_time.calendar # str (ex.: 'standard'/'gregorian'/...)
if type_dates == 'num':
arr_dt = var_time[:]
list_dt = arr_dt.tolist() # numpy array -> list
if type_dates == 'dt':
t = utime(time_units, time_calend) # <netcdftime.utime instance at 0xecae18>
arr_dt = t.num2date(var_time[:]) # arr_dt: numpy array of dates datetime; var_time[:]: time values (ex.: [49323.5, 49353, 49382.5, ...])
list_dt = arr_dt.tolist() # numpy array -> list
del arr_dt
nc.close()
return list_dt
def num2date(num, calend, units):
'''
type num: float date
'''
t = utime(units, calend)
dt = t.num2date(num)
return dt
def extract_netcdf_time(cls, bandName, calendar):
"""Convert netcdf time to datetime using appropriate library"""
from netcdftime import utime
epoch, units = cls.extract_epoch_units(bandName)
cdftime = utime(units, calendar)
timestamps = cdftime.num2date([epoch])
return timestamps[0]
def plot_time_series(data_in, param_in, region):
'''
plot_time_series ...
'''
# Read time and param vars
time = data_in.variables['time'][:] # read the time
param = data_in.variables[param_in][:]
# Scale var
[scal_req, scale_factor, add_offset] = findScaleOffset(data_in, param_in)
param_scaled = (scale_factor*param)+add_offset
# create time vector
time_uni = data_in.variables['time'].units # get the time units
time_cal = data_in.variables['time'].calendar # read calendar
cdftime = utime(time_uni, calendar=time_cal)
date = [cdftime.num2date(t) for t in time]
# ############# A plot of Maximum precipitation ##############
plt.figure(region+' '+param_in, figsize=(15, 6))
plt.plot(date, param_scaled[:, 0, 0], label=model)
# window = 10 # date [x:-y], where x+y = window - 1
# param_scaled_smoothed = moving_average(arr=param_scaled[:, 0, 0], win=window)
# plt.plot(date[5:-4], param_scaled_smoothed, label=model+'_smoothed') ##
plt.ylabel("%s Anomaly (%s)" % (data_in.variables[param_in].long_name,
data_in.variables[param_in].units))
plt.ticklabel_format(useOffset=False, axis='y')
plt.xlabel("Time")
plt.title('Annual '+data_in.variables[param_in].long_name+' Anomaly ' + region + ' region', fontweight='bold')
def read_d3d_time(netcdf_vars):
"""
Read time from netcdf file and return Datetime vector
"""
utime = netcdftime.utime(netcdf_vars['time'].units)
dates = utime.num2date(netcdf_vars['time'][:])
return dates
def read_net(year, month, day, lats, lons, dataset=None):
'''
datasets should be wv(water vapur), aot (aerosol), tco (Ozone)
'''
if dataset == 'wv':
fname = glob.glob('ECMWF/Total_column_water_vapour_%d*.nc' % year)[0]
elif dataset == 'aot':
fname = glob.glob('ECMWF/Total_Aerosol_Optical_Depth_at_550nm_%d*.nc' %
year)[0]
elif dataset == 'tco':
fname = glob.glob('ECMWF/GEMS_Total_column_ozone_%d*.nc' % year)[0]
else:
print 'Wrong dataset is given, please see the doc string!'
net = Dataset(fname, 'r')
lon, lat, time, val = net.variables.keys()
lat_ind, lon_ind = ((lats - net[lat][0]) /
(-0.125)).astype(int), ((lons - net[lon][0]) /
0.125).astype(int)
cdf_time = utime(net[time].units)
date = cdf_time.num2date(net[time][:])
time_ind = np.where(
date == datetime.datetime(year, month, day, 3, 0))[0][0]
return net[val][time_ind, lat_ind, lon_ind]
def day_number_to_datetime_array(time_in, calendar_type, units_in):
cdftime = utime(units_in, calendar=calendar_type)
date_out = cdftime.num2date(time_in)
return date_out
def load_time(self): tnum=self.nc.variables['time'][-self.ntimes_max:] tunits=self.nc.variables['time'].units self.time=utime(tunits,calendar='standard').num2date(tnum) ind0=self.nc.variables['time'].size-self.ntimes_max ind1=self.nc.variables['time'].size self.time_inds=range(ind0,ind1)
def _process(self, inputs):
import numpy as np
if sys.version[0] == '3':
name_scenario = [*inputs][0]
else:
name_scenario = inputs.keys()[0]
if self.count == 0:
self.time = inputs[name_scenario][0]
var = inputs[name_scenario][1]
self.mat = np.zeros((self.nb_scenario, len(var)))
#We move from num to date format because if we send a netcdf variable format we have this error:
#*** NotImplementedError: Variable is not picklable
import netcdftime
t = netcdftime.utime(self.time.units, self.time.calendar)
self.time = t.num2date(self.time[:])
#update_monitoring_file(self.name)
self.mat[self.count, :] = inputs[name_scenario][1]
self.count += 1
if self.count == (self.nb_scenario):
self.write('output', (self.time, self.mat))
def convert(cls, value, unit, axis):
"""
Converts value, if it is not already a number or sequence of numbers,
with :func:`netcdftime.utime().date2num`.
"""
if isinstance(value, np.ndarray):
# Don't do anything with numeric types.
if value.dtype != np.object:
return value
first_value = value[0]
else:
# Don't do anything with numeric types.
if munits.ConversionInterface.is_numlike(value):
return value
first_value = value
if not isinstance(first_value, CalendarDateTime):
raise ValueError('The values must be numbers or instances of '
'"nc_time_axis.CalendarDateTime".')
if not isinstance(first_value.datetime, netcdftime.datetime):
raise ValueError('The datetime attribute of the CalendarDateTime '
'object must be of type `netcdftime.datetime`.')
ut = netcdftime.utime(cls.standard_unit, calendar=first_value.calendar)
if isinstance(value, CalendarDateTime):
value = [value]
return ut.date2num([v.datetime for v in value])
def check(self, max_n_ticks, num1, num2):
locator = NetCDFTimeDateLocator(max_n_ticks=max_n_ticks,
calendar=self.calendar,
date_unit=self.date_unit)
utime = netcdftime.utime(self.date_unit, self.calendar)
return locator.compute_resolution(num1, num2, utime.num2date(num1),
utime.num2date(num2))
def setUp(self):
self.cdftime_mixed = utime("hours since 0001-01-01 00:00:00")
self.cdftime_mixed_tz = utime("hours since 0001-01-01 00:00:00 -06:00")
self.cdftime_pg = utime("seconds since 0001-01-01 00:00:00", calendar="proleptic_gregorian")
self.cdftime_noleap = utime("days since 1600-02-28 00:00:00", calendar="noleap")
self.cdftime_leap = utime("days since 1600-02-29 00:00:00", calendar="all_leap")
self.cdftime_360day = utime("days since 1600-02-30 00:00:00", calendar="360_day")
self.cdftime_jul = utime("hours since 1000-01-01 00:00:00", calendar="julian")
self.cdftime_iso = utime("seconds since 1970-01-01T00:00:00Z")
def get_time_range(files, time_range=None, temporal_var_name='time'):
'''
If time_range is None, this function will get a time range from input files.
Else, this function will adjust the time_range by setting hour value to datetime.datetime objects.
:param files: netCDF file(s) (including OPeNDAP URL(s))
:type files: list of str
:param time_range: time_range (default: None)
:type time_range: list two datetime objects: [begin, end]
:param temporal_var_name: name of temporal variable from netCDF file (default: "time")
:type temporal_var_name: str
:rtype
Returns a time range: a list two datetime objects: [begin, end], where "begin" is the first date, and "end" is the last.
'''
try:
nc = Dataset(files[0], 'r')
except RuntimeError:
raise MissingIcclimInputError("Failed to access dataset: " + files[0])
time = nc.variables[temporal_var_name]
try:
calend = time.calendar
except:
calend = 'gregorian'
units = time.units
t = netcdftime.utime(units, calend)
any_dt = t.num2date(time[0])
nc.close()
if time_range != None:
time_range = harmonize_hourly_timestamp(time_range, calend, any_dt)
else:
try:
nc = MFDataset(files, 'r', aggdim='time')
except RuntimeError:
raise MissingIcclimInputError("Failed to access dataset: " + files)
time_arr = nc.variables[temporal_var_name][:]
nc.close()
begin_num = min(time_arr)
end_num = max(time_arr)
begin_dt = num2date(begin_num, calend, units)
end_dt = num2date(end_num, calend, units)
time_range = [begin_dt, end_dt]
return time_range
def __new__(self, ncfile, name='time', units=None, calendar='standard'):
self._nc = octant.io.Dataset(ncfile)
data = self._nc.variables[name][:]
data = data.view(time)
if units == None:
units = self._nc.variables[name].units
data.utime = netcdftime.utime(units, calendar=calendar)
return data
def units_checker(units):
try:
u = utime(units)
return u.unit_string
except ValueError, TypeError:
msg = 'Invalid time units format - Available format is "{}"'.format(
TIME_UNITS_FORMAT)
raise ArgumentTypeError(msg)
def get_time_converter(time_var):
try:
time_var.units
except AttributeError:
raise AttributeError('netcdf time variable is missing a \'units\' '
'attribute')
return utime(time_var.units,
calendar=getattr(time_var, 'calendar', 'standard'))
def __new__(self, ncfile, name='time', units=None, calendar='standard'):
self._nc = pyroms.io.Dataset(ncfile)
data = self._nc.variables[name][:]
data = data.view(time)
if units == None:
units = self._nc.variables[name].units
data.utime = netcdftime.utime(units, calendar=calendar)
return data
def setUp(self):
self.cdftime_mixed = utime('hours since 0001-01-01 00:00:00')
self.cdftime_julian = utime(
'hours since 0001-01-01 00:00:00', calendar='julian')
self.cdftime_mixed_tz = utime('hours since 0001-01-01 00:00:00 -06:00')
self.cdftime_pg = utime('seconds since 0001-01-01 00:00:00',
calendar='proleptic_gregorian')
self.cdftime_noleap = utime(
'days since 1600-02-28 00:00:00', calendar='noleap')
self.cdftime_leap = utime(
'days since 1600-02-29 00:00:00', calendar='all_leap')
self.cdftime_360day = utime(
'days since 1600-02-30 00:00:00', calendar='360_day')
self.cdftime_jul = utime(
'hours since 1000-01-01 00:00:00', calendar='julian')
self.cdftime_iso = utime("seconds since 1970-01-01T00:00:00Z")
self.cdftime_leading_space = utime("days since 850-01-01 00:00:00")
def __init__(self,vname,plon,plat,dset='GSOP_GLORYS2V4',\
syr=1993,eyr=2009,\
path='/home/uotilap/tiede/ORA-IP/annual_mean/'):
print "Reading %s" % dset
self.dset = dset
self.vname = vname # T or S
self.path = path
self.lat = plat
self.lon = plon
self.syr = syr
self.eyr = eyr
self.level_bounds = LevelBounds[vname]
self.depth = np.hstack((self.level_bounds[:, 0], 4000))
if self.vname == 'T':
fn = "%s_ORCA025_HC.nc" % (self.dset)
varname = 'heatc'
else:
fn = "%s_ORCA025_SC.nc" % (self.dset)
varname = 'saltc'
fp = nc.Dataset(os.path.join(self.path, fn))
lat = np.array(fp.variables['lat'][:])
lon = np.array(fp.variables['lon'][:])
# transfer negative lons to positive
lon[np.where(lon < 0.)] += 360.
iy = np.where(
np.abs(lat - self.lat) == np.min(np.abs(lat - self.lat)))[0][0]
ix = np.where(
np.abs(lon - self.lon) == np.min(np.abs(lon - self.lon)))[0][0]
time = fp.variables['time']
cdftime = utime(time.units, calendar=time.calendar)
dates = [cdftime.num2date(t) for t in time[:]]
data = [[] for i in self.level_bounds[:, 0]]
for li, lb in enumerate(self.level_bounds):
ncnameu = "z%d%s" % (lb[0], varname)
if lb[1] == 6000:
ncnamel = "zbot%s" % (varname)
else:
ncnamel = "z%d%s" % (lb[1], varname)
ldata = []
for i, t in enumerate(time[:]):
date = dates[i]
if date.year in range(self.syr, self.eyr + 1):
ldatal = fp.variables[ncnamel][i, iy, ix]
if ncnameu in ['z0heatc', 'z0saltc']:
ldata.append(ldatal / lb[1])
else:
ldatau = fp.variables[ncnameu][i, iy, ix]
ldata.append((ldatal - ldatau) / (lb[1] - lb[0]))
data[li].append(np.ma.mean(ldata))
fp.close()
if vname == 'T':
self.data = np.ma.masked_equal(np.ma.squeeze(data), 0)
else: #S
self.data = np.ma.masked_less_equal(np.ma.squeeze(data), 32)
if self.data[-1] < self.data[-2]:
self.data[-1] = self.data[-2]
if self.data[-2] < self.data[-3]:
self.data[-1] = self.data[-2] = self.data[-3]
def setUp(self):
self.cdftime_mixed = utime('hours since 0001-01-01 00:00:00')
self.cdftime_mixed_tz = utime('hours since 0001-01-01 00:00:00 -06:00')
self.cdftime_pg = utime('seconds since 0001-01-01 00:00:00',
calendar='proleptic_gregorian')
self.cdftime_noleap = utime('days since 1600-02-28 00:00:00',calendar='noleap')
self.cdftime_leap = utime('days since 1600-02-29 00:00:00',calendar='all_leap')
self.cdftime_360day = utime('days since 1600-02-30 00:00:00',calendar='360_day')
self.cdftime_jul = utime('hours since 1000-01-01 00:00:00',calendar='julian')
self.cdftime_iso = utime("seconds since 1970-01-01T00:00:00Z")
def __new__(self, ncfile, name='time', units=None, calendar='standard'):
raise DeprecationWarning, 'Use netCDF4.num2date instead. Wrapper function in octant.roms.nc_time'
self._nc = octant.io.Dataset(ncfile)
data = self._nc.variables[name][:]
data = data.view(time)
if units == None:
units = self._nc.variables[name].units
data.utime = netcdftime.utime(units, calendar=calendar)
return data
def test_date2num(feb3_noon, dmy_feb_3_noon):
for cal, cal_num in calendars.items():
for unit, unit_num in units.items():
unit_string = '{0} since 0001-01-01'.format(unit)
ut = utime(unit_string, calendar=cal)
expected = ut.date2num(feb3_noon)
actual = vic_lib.date2num(ut._jd0, dmy_feb_3_noon, 0, cal_num,
unit_num)
np.testing.assert_allclose(actual, expected)
def test_date2num(feb3_noon, dmy_feb_3_noon):
for cal, cal_num in calendars.items():
for unit, unit_num in units.items():
unit_string = '{0} since 0001-01-01'.format(unit)
ut = utime(unit_string, calendar=cal)
expected = ut.date2num(feb3_noon)
actual = vic_lib.date2num(ut._jd0, dmy_feb_3_noon, 0,
cal_num, unit_num)
np.testing.assert_allclose(actual, expected)
def FindDayOfYear(dateStruc, dateUnits, calendar):
import netcdftime as nct
nDays = len(dateStruc)
t = nct.utime(dateUnits, calendar=calendar)
dateLoc = np.zeros_like(dateStruc)
for d in range(nDays):
dateLoc[d] = nct.datetime(1, dateStruc[d].month, dateStruc[d].day)
dayOfYear = t.date2num(dateLoc)
return dayOfYear
def test_extract_conversion_factors_from_string(self):
u = utime('hours since 1970-01-01 00:00:00')
t_origin = api.Calendar(u.tzoffset).time(
api.YMDhms(u.origin.year, u.origin.month, u.origin.day, u.origin.hour, u.origin.minute, u.origin.second))
delta_t_dic = {'days': api.deltahours(24), 'hours': api.deltahours(1), 'minutes': api.deltaminutes(1)}
delta_t = delta_t_dic[u.units]
self.assertIsNotNone(u)
self.assertEqual(delta_t, api.deltahours(1))
self.assertEqual(t_origin, 0)
def __new__(self, ncfile, name='time', units=None, calendar='standard'):
raise DeprecationWarning, 'Use netCDF4.num2date instead. Wrapper function in octant.roms.nc_time'
self._nc = octant.io.Dataset(ncfile)
data = self._nc.variables[name][:]
data = data.view(time)
if units == None:
units = self._nc.variables[name].units
data.utime = netcdftime.utime(units, calendar=calendar)
return data
def create_grid(nc, inc):
nc.createDimension('time', None)
tv = nc.createVariable('time', 'f8', ('time'))
tv.long_name = 'Time'
tv.standard_name = 'time'
tv.units = 'days since 1948-01-01 00:00:00'
tv.base_date = [1948, 1, 1, 0]
ut = netcdftime.utime(tv.units)
incv = inc.variables
# copy some global attributes
for attr in ['experiment_id', 'references']:
nc.setncattr(attr, inc.getncattr(attr))
hstr = dt.strftime(dt.now(),
'%a %b %d %H:%M:%S %Y') + ': create_schism_sflux.py\n'
nc.setncattr('history', unicode(hstr + inc.getncattr('history')))
# write time
iut = netcdftime.utime(incv['time'].units)
tv[0:len(inc.dimensions['time'])] = ut.date2num(
iut.num2date(incv['time'][:]))
# write grid
nc.createDimension('nx_grid', len(inc.dimensions['lon']))
nc.createDimension('ny_grid', len(inc.dimensions['lat']))
lon = incv['lon'][:]
lat = incv['lat'][:]
gridlon, gridlat = np.meshgrid(lon, lat)
lv = nc.createVariable('lon', 'f4', ('ny_grid', 'nx_grid'))
lv.long_name = 'Longitude'
lv.standard_name = 'longitude'
lv.units = 'degrees_east'
lv[:] = gridlon
lv = nc.createVariable('lat', 'f4', ('ny_grid', 'nx_grid'))
lv.long_name = 'Latitude'
lv.standard_name = 'latitude'
lv.units = 'degrees_north'
lv[:] = gridlat
nc.sync()
def calculate_normals(time_var, value_arr, units, variable_name, start_time,
end_time, month):
'''
Calculates the means (or totals) through the given time period.
'''
# Parse the time units
# The calendar in some CMIP6 model outputs is "360 day"
# so you can't take the "days since 1850-01-01" literally.
cdftime = netcdftime.utime(time_var.units, time_var.calendar)
if time_var.size == 12:
#
# In this case, the file only contains 12 time indexes,
# so they are not actual moments in time and the data
# is already aggregated by month.
#
if not month:
filtered_time_indexes = np.arange(0, 12)
else:
filtered_time_indexes = np.array([month - 1])
else:
if time_var.calendar == '360_day' and end_time.day == 31:
end_time = end_time - timedelta(days=1)
# Determine start and end times in the time units
start = cdftime.date2num(start_time)
end = cdftime.date2num(end_time)
earliest_time = cdftime.num2date(time_var[0])
if earliest_time.month != 1:
print(
'Warning: Earliest time does not start in January (starting on %s). '
'High probability of the time variable being wrong.'
'Check variables[\'time\'].calendar' % earliest_time,
file=sys.stderr)
# Determine the time indexes that correspond for this range and month
time_arr = time_var[:]
time_indexes_range = np.where((time_arr >= start)
& (time_arr <= end))[0]
if month:
filtered_time_indexes = []
for time_i in time_indexes_range:
time_value = time_var[time_i]
time = cdftime.num2date(time_value)
if time.month == month:
filtered_time_indexes.append(time_i)
else:
filtered_time_indexes = time_indexes_range
# Filter the data by the time indexes
filtered_values = value_arr[filtered_time_indexes, :, :]
# Compute the average for each coordinate through time axis
return filtered_values.mean(axis=0)
def create_grid(nc,inc):
nc.createDimension('time',None)
tv = nc.createVariable('time','f8',('time'))
tv.long_name = 'Time'
tv.standard_name = 'time'
tv.units = 'days since 1948-01-01 00:00:00'
tv.base_date = [1948,1,1,0]
ut = netcdftime.utime(tv.units)
incv = inc.variables
# copy some global attributes
for attr in ['experiment_id','references']:
nc.setncattr(attr,inc.getncattr(attr))
hstr = dt.strftime(dt.now(),'%a %b %d %H:%M:%S %Y')+': create_schism_sflux.py\n'
nc.setncattr('history',unicode(hstr+inc.getncattr('history')))
# write time
iut = netcdftime.utime(incv['time'].units)
tv[0:len(inc.dimensions['time'])] = ut.date2num(iut.num2date(incv['time'][:]))
# write grid
nc.createDimension('nx_grid',len(inc.dimensions['lon']))
nc.createDimension('ny_grid',len(inc.dimensions['lat']))
lon = incv['lon'][:]
lat = incv['lat'][:]
gridlon,gridlat = np.meshgrid(lon,lat)
lv = nc.createVariable('lon','f4',('ny_grid','nx_grid'))
lv.long_name = 'Longitude'
lv.standard_name = 'longitude'
lv.units = 'degrees_east'
lv[:] = gridlon
lv = nc.createVariable('lat','f4',('ny_grid','nx_grid'))
lv.long_name = 'Latitude'
lv.standard_name = 'latitude'
lv.units = 'degrees_north'
lv[:] = gridlat
nc.sync()
def Convert2Days(time, units, calendar):
"""
"""
import netCDF4 as nc
import netcdftime as nct
date = nc.num2date(time, units, calendar)
unitArray = units.split()
dayUnits = units.replace(unitArray[0], 'days')
t = nct.utime(dayUnits, calendar=calendar)
return t.date2num(date)
def read_times(filename):
"""Read times from a netcdf file and convert to datetime objects"""
time_data=load_time_var(filename)
cdftime = utime(time_data.units,time_data.calendar)
dates = [cdftime.num2date(time_data.data[i]) for i in range(time_data.data.shape[0])]
return dates
def print_nc_dates(ref_grd,time_key):
import netCDF4
import netcdftime
#time_key='ocean_time'
nc=netCDF4.Dataset(ref_grd)
ncv_obs=nc.variables
utim_obs=netcdftime.utime(ncv_obs[time_key].units)
sec_obs=ncv_obs[time_key][:]
date_obs=utim_obs.num2date(sec_obs)
print(date_obs)
return date_obs
def time_slice(time, grid, dataset):
"""\
Slice according to time request (WMS-T). Since it is very expensive to
convert the entire grid time array to datetime objects we do the opposite
and convert the requested time steps to the unit used in the grid time
array.
If input time is None the nearest timestep is returned.
"""
if time is None:
find_nearest = True
else:
find_nearest = False
for dim in grid.maps.values():
if ' since ' in dim.attributes.get('units', ''):
calendar = dim.attributes.get('calendar', 'standard')
try:
values = np.array(np.asarray(dim))
break
except:
pass
if len(values.shape) == 0:
# Input field has no time dimension
l = Ellipsis
else:
l = np.zeros(values.shape, bool) # get no data by default
if time is None:
time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
tokens = time.split(',')
for token in tokens:
if '/' in token: # range
start, end = token.strip().split('/')
start = iso8601.parse_date(start, default_timezone=None)
end = iso8601.parse_date(end, default_timezone=None)
l[(values >= start) & (values <= end)] = True
# Convert to index (we do not support multiple timesteps)
l = np.where(l == True)[0][0]
else:
instant = iso8601.parse_date(token.strip().rstrip('Z'), default_timezone=None)
utime = netcdftime.utime(dim.units, calendar=calendar)
instant = utime.date2num(instant)
# TODO: Calculate index directly
epoch = values[0]
values = values - epoch
instant = instant - epoch
if not find_nearest:
# Require almost exact match
l = np.isclose(values, instant)
# Convert array to index
l = np.where(l == True)[0][0]
else:
l = (np.abs(values-instant)).argmin()
return l
def print_nc_dates(ref_grd, time_key):
import netCDF4
import netcdftime
#time_key='ocean_time'
nc = netCDF4.Dataset(ref_grd)
ncv_obs = nc.variables
utim_obs = netcdftime.utime(ncv_obs[time_key].units)
sec_obs = ncv_obs[time_key][:]
date_obs = utim_obs.num2date(sec_obs)
print(date_obs)
return date_obs
def Read_NetCDF(fname, KeyVar):
'Read NetCDF file'
if "*" in fname: nc = MFDataset(fname, 'r')
else: nc = Dataset(fname, 'r')
## Get time using the time variable
Time_Var = nc.variables['time']
dt = Time_Var[:][1] - Time_Var[:][0]
Time_H = np.arange(Time_Var[:][0], Time_Var[:][0] + dt * Time_Var[:].size,
dt)
try:
Tref = netcdftime.utime(Time_Var.units, calendar=Time_Var.calendar)
except:
Tref = netcdftime.utime(Time_Var.units, calendar="gregorian")
Time = Tref.num2date(Time_H)
print "====================++++"
## Get Coordinates
try:
Lon = nc.variables['longitude'][:]
Lat = nc.variables['latitude'][:]
LON, LAT = np.meshgrid(Lon, Lat)
except:
LON = nc.variables['lon'][:]
LAT = nc.variables['lat'][:]
## Get Variable
dum = nc.variables[KeyVar]
Var = dum[:]
ind = (Var == dum._FillValue)
Var[ind] = np.nan
#Var = Var / dum.scale_factor + dum.add_offset
ind = (np.isnan(Var))
Var[ind] = -9999999
print Time[0], Time[-1], Var.shape, Time.shape, np.sum(ind)
try:
return Time, LON, LAT, Var, dum.units, dum.long_name
except:
return Time, LON, LAT, Var, dum.units, dum.standard_name
def test_initialize_time():
for cal, cal_num in calendars.items():
for unit, unit_num in units.items():
unit_string = '{0} since 0001-01-01'.format(unit)
ut = utime(unit_string, calendar=cal)
vic_lib.global_param.calendar = cal_num
vic_lib.global_param.time_units = unit_num
assert vic_lib.initialize_time() is None
assert vic_lib.global_param.time_origin_num != -99999
print(unit_string, cal)
np.testing.assert_allclose(vic_lib.global_param.time_origin_num,
ut._jd0)
def return_perc_array_2_compute_bootstrapping(dt_arr, arr_filled,
t_units, t_calendar, ytd, window_width, percentile,
interpolation, ignore_Feb29th, only_leap_years, bootstrapping):
#Get time config
nc_time = netcdftime.utime(t_units, t_calendar)
#From datetime to numerical format
dt_arr_num = nc_time.date2num(dt_arr)
#Find first day of ytd in dt_arr_num, make it equel to ind_ytd_start
if bootstrapping:
ind_ytd_start = get_first_ytd_day(dt_arr, dt_arr_num, nc_time, ytd)
else:
ind_ytd_start = 0
#Get the length of the year depending on the calendar type and 29th Feb option. I.e 360, 365 ot 366 days
len_ytd = util_dt.check_calend(t_calendar, ignore_Feb29th)
#List the year to be computed and duplicated ytd
list_year = get_list_year(ytd, dt_arr)
#Width of half of the window
window_wide = int(window_width/2)
#Definition of the percentile array to fill and return
percentile_array = np.zeros((len_ytd, arr_filled.shape[1], arr_filled.shape[2]))
#Datetime for the first index of the year to duplicate => Starting date
t = dt_arr[ind_ytd_start]
#variable i to be iterated in the while statement represents each day of the year
i=0
#We iterate across day over all year long related to the year length
while i<len_ytd:
# t is the centered day which we perform the calculation on
t = dt_arr[ind_ytd_start+i]
#ind_2_calc returns the index we want to perform the percentile on
ind_2_calc = indices_to_return_for_percentile_calc(dt_arr_num, dt_arr, list_year, window_wide, i, len_ytd, ytd,
nc_time, t_calendar, t, ignore_Feb29th, only_leap_years, bootstrapping,
ind_ytd_start)
#Percentile calculation
percentile_array[i,:,:] = np.percentile(arr_filled[ind_2_calc,:,:], percentile, axis=0, interpolation=interpolation)
#Iteration
i+=1
return percentile_array
def year_clip(data, var, begin, end):
tcon = utime('days since 1900-01-01', calendar='noleap')
dates = tcon.num2date(data.variables['time'][:])
yvals = np.zeros((len(dates))) # create array to hold year values
# year array
for j in range(len(dates)):
yvals[j] = dates[j].year
# aggregate by year
var_data = data.variables[var][((yvals > begin - 1) & (yvals < end + 1)), :, :]
return var_data
def get_daterange(ncin):
nc = Dataset(ncin, 'r')
# get the time variable (note that it must be named 'time'
time = nc.variables['time']
# initialize utime, which allows conversion of the time axis
nctime = netcdftime.utime(time.units)
start = nctime.num2date(time[0])
end = nctime.num2date(time[-1])
nc.close()
return (start, end)
def open_precip(file_path = EV['REANALYSIS'] + 'prate.mon.mean.nc'):
from netCDF4 import Dataset
from netcdftime import utime
from pandas import DatetimeIndex
dat = Dataset(file_path)
lat = dat['lat']
lon = dat['lon']
time = dat['time']
prate = dat['prate']
#Get pandas index from time
cdftime = utime(time.units); dates = cdftime.num2date(time[:])
index = DatetimeIndex(dates)
return prate, lat, lon, index
