def plot_histograms(self, save=False, save_dir=None, fancy=True):
'''Create an timeseries plot of specific variables.
Parameters::
----------
save = boolean
True to save an output plot, False displays on screen.
fancy = boolean
If set to true uses ggplot style, otherwise default is used.
'''
if fancy:
matplotlib.style.use('ggplot')
# Set the time string format
fmt = '%H:%M:%S'
# Create figure/axes instance
fig, axs = plt.subplots(nrows=4, ncols=3, figsize=(8, 11), \
sharey="row")
for ax, fills in zip(axs.flat, self.dts):
# Calculate start and end times
stime = netCDF4.datetime(fills[0][0], fills[0][1], fills[0][2],\
fills[0][3], fills[0][4], fills[0][5])
etime = netCDF4.datetime(fills[1][0], fills[1][1], fills[1][2],\
fills[1][3], fills[1][4], fills[1][5])
# Make title from these times
figtitle = stime.strftime(fmt) + ' - ' + etime.strftime(fmt)
##if self.pcasp[
# Calculate Poisson error from Cai et al. AMT (2013), Equation A4
yerr = self.pcaspf.between_time(stime, etime).sum(axis=0).divide(self.flow3s.between_time(stime, etime).sum(axis=0)**2).pow(0.5).mul(1./self.dlog10d)
# Calculate the dNd(log10{D}) value for this time period,
# sum at each diameter (bin), and replace zero values with NaN
dndlogp = self.pcaspf.between_time(stime, etime).sum(axis=0).divide(\
(self.flow3s.between_time(stime, etime).sum(axis=0) * self.dlog10d))
# Create a pandas Series for plotting
dndlogpf = pd.Series(data=dndlogp.replace(to_replace=0., value=np.nan), \
index=self.pcasp_bin_mid, name=figtitle)
xlabels = self.pcasp_bin_mid.astype('|S5')
dndlogpf.plot(ax=ax, logx=True, logy=True, yerr=yerr, legend=False, x=xlabels)
ax.set_title(figtitle)
ax.set_xlim(left=0.1, right=10.)
ax.set_ylim(bottom=0.01, top=10000.)
ax.xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter())
ax.yaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter())
ax.set_xlabel(r'D ($\mu$m)')
ax.set_ylabel(r"dN/dlog$_{10}$D (cm$^{-3}$)")
fig.tight_layout()
if save:
# Set the output file name
fname = "pcasp_timehist_" + os.path.splitext(self.filename)[0] + ".png"
outfile = os.sep.join([save_dir, fname])
plt.savefig(outfile, format='png')
print "Creating image: " + outfile
else:
plt.show()
def _draw_time_region(self, ax):
'''A function to highlight time regions for further analysis.
Parameters::
----------
ax = Matplotlib axis instance
Used for drawing polygons on that axis.
'''
# Get the y-limits to match the polycollection
ylim = ax.get_ylim()
# Loop through the times and create
for fills in self.dts:
stime = netCDF4.datetime(fills[0][0], fills[0][1], fills[0][2],\
fills[0][3], fills[0][4], fills[0][5])
etime = netCDF4.datetime(fills[1][0], fills[1][1], fills[1][2],\
fills[1][3], fills[1][4], fills[1][5])
ax.fill_between(self.time, ylim[0], ylim[1], \
where=(self.time>=stime)&(self.time<=etime), alpha=0.3)
def shift_time(infilename, outfilename, amount, shift_units):
"""
Do the actual time shift
:param infilename: the name of the input file (full or relative path)
:param outfilename: the name of the output file (full or relative path)
:param amount: amount to shift the time
:type amount: floating point number, of a string version
:param shift_units: units of the time shift
:type shift_units: string
"""
units = str(shift_units).strip()
# make a copy of the file if it's different
if not infilename == outfilename:
print "creating new file:", outfilename
shutil.copyfile(infilename, outfilename)
# pull the time units out of the source copy:
nc = netCDF4.Dataset(infilename, mode='r')
time_var = nc.variables['time']
time_units = time_var.units
print "original time units", time_units
nc.close()
units, epoch = time_units.split('since')
# iso datetimes can be separated with whitesapce or "T"
try:
date, time = epoch.strip().split()
except ValueError:
date, time = epoch.strip().split("T")
year, month, day = [int(i) for i in date.split('-')]
hour, minute, second = [int(i) for i in time.split(':')]
dt = netCDF4.datetime(year, month, day, hour, minute, second)
try:
shift = netCDF4.timedelta(**{shift_units: float(amount)})
except TypeError:
raise ValueError(
"Time shift must units options are: days, seconds, minutes, hours, or weeks"
)
dt += shift
new_units = "%s since %s" % (units.strip(), dt.isoformat())
print "new time units:", new_units
## write out to the new file:
nc = netCDF4.Dataset(outfilename, mode='r+')
time_var = nc.variables['time']
time_var.units = new_units
nc.close()
print "Done"
def _draw_time_region(self, ax):
'''A function to highlight time regions for further analysis.
Parameters::
----------
ax = Matplotlib axis instance
Used for drawing polygons on that axis.
'''
# Get the y-limits to match the polycollection
ylim = ax.get_ylim()
# Loop through the times and create
for fills in self.dts:
stime = netCDF4.datetime(fills[0][0], fills[0][1], fills[0][2],\
fills[0][3], fills[0][4], fills[0][5])
etime = netCDF4.datetime(fills[1][0], fills[1][1], fills[1][2],\
fills[1][3], fills[1][4], fills[1][5])
ax.fill_between(self.time, ylim[0], ylim[1], \
where=(self.time>=stime)&(self.time<=etime), alpha=0.3)
def sel_season(var, dates, season):
#----------------------------------------------------------------------------------------
#print('____________________________________________________________________________________________________________________')
#print('Selecting only {0} data'.format(season))
#----------------------------------------------------------------------------------------
dates_pdh = pd.to_datetime(dates)
if season == 'DJF': #ONLY DEC-JAN-FEB
m = [12, 1, 2]
mask = (dates_pdh.month == 12) | (dates_pdh.month
== 1) | (dates_pdh.month == 2)
elif season == 'DJFM': #ONLY DEC-JAN-FEB-MAR
m = [12, 1, 2, 3]
mask = (dates_pdh.month == 12) | (dates_pdh.month == 1) | (
dates_pdh.month == 2) | (dates_pdh.month == 3)
elif season == 'NDJFM': #ONLY NOV-DEC-JAN-FEB-MAR
m = [11, 12, 1, 2, 3]
mask = (dates_pdh.month == 11) | (dates_pdh.month == 12) | (
dates_pdh.month == 1) | (dates_pdh.month == 2) | (dates_pdh.month
== 3)
elif season == 'JJA': #ONLY JUN-JUL-AUG
m = [6, 7, 8]
mask = (dates_pdh.month == 6) | (dates_pdh.month
== 7) | (dates_pdh.month == 8)
else:
print('season is not one of the following: DJF, DJFM, NDJFM, JJA')
var_season = var[mask, :, :]
dates_season = dates[mask]
if (12 in m) or (1 in m):
#REMOVING THE FIRST MONTHS (for the first year) because there is no previuos december
start = int(
np.where(dates_season ==
datetime(dates_pdh.year[0], m[0], dates_pdh.day[0],
dates_pdh.hour[0], dates_pdh.minute[0]))[0])
#REMOVING THE LAST MONTHS (for the last year) because there is no following january
end = int(
np.where(dates_season ==
datetime(dates_pdh.year[-1], m[0], dates_pdh.day[0],
dates_pdh.hour[0], dates_pdh.minute[0]))[0])
var_season = var_season[start:end, :, :]
dates_season = dates_season[start:end]
return var_season, dates_season
def parseDate(dat):
dateRe = r'^(\d{4})(?:-|\.|\/)?(0[1-9]|1[0-2])(?:-|\.|\/)?(0[1-9]|[1-2]\d|3[01])(?:T?([01]\d|2[0-3])(?::?([0-5]\d)(?::?([0-5]\d))?)?)?$'
parDat = re.match(dateRe, dat)
if parDat:
parDat = [int(e) for e in parDat.groups(default=0)]
#If it is efectively a date
return datetime(*parDat)
else:
return None
def shift_time(infilename, outfilename, amount, shift_units):
"""
Do the actual time shift
:param infilename: the name of the input file (full or relative path)
:param outfilename: the name of the output file (full or relative path)
:param amount: amount to shift the time
:type amount: floating point number, of a string version
:param shift_units: units of the time shift
:type shift_units: string
"""
units = str(shift_units).strip()
# make a copy of the file if it's different
if not infilename == outfilename:
print("creating new file:", outfilename)
shutil.copyfile(infilename, outfilename)
# pull the time units out of the source copy:
nc = netCDF4.Dataset(infilename, mode='r')
time_var = nc.variables['time']
time_units = time_var.units
print("original time units", time_units)
nc.close()
units, epoch = time_units.split('since')
# iso datetimes can be separated with whitesapce or "T"
try:
date, time = epoch.strip().split()
except ValueError:
date, time = epoch.strip().split("T")
year, month, day = [int(i) for i in date.split('-')]
hour, minute, second = [int(i) for i in time.split(':')]
dt = netCDF4.datetime(year, month, day, hour, minute, second)
try:
shift = netCDF4.timedelta(**{shift_units:float(amount)})
except TypeError:
raise ValueError("Time shift must units options are: days, seconds, minutes, hours, or weeks")
dt+=shift
new_units = "%s since %s"%(units.strip(), dt.isoformat())
print("new time units:", new_units)
## write out to the new file:
nc = netCDF4.Dataset(outfilename, mode='r+')
time_var = nc.variables['time']
time_var.units = new_units
nc.close()
print("Done")
def seltimeframe(scenario, t_coord, t_units, t_cal):
if scenario == 'historical':
yi = 1860
yf = 2005
elif (scenario == 'rcp45') or (scenario == 'rcp85'):
yi = 2005
yf = 2100
elif scenario == '1pctCO2':
yi_tmp = t_units[
11:15] # time axis for 1pctCO2 varies from one model to another
if yi_tmp == "1-1-":
yi = 1
else:
yi = int(yi_tmp)
yf = yi + 139
ti = netcdf.date2num(netcdf.datetime(yi, 1, 1), t_units, t_cal)
if t_cal == '360_day':
tf = netcdf.date2num(netcdf.datetime(yf, 12, 30), t_units, t_cal)
else:
tf = netcdf.date2num(netcdf.datetime(yf, 12, 31), t_units, t_cal)
ind_t = np.where(np.bitwise_and(t_coord >= ti, t_coord <= tf + 1))
return ind_t
def fetch_radar_time_profile(sonde_dset,
radar,
time_key='time',
height_key='height',
nvars=None):
"""
Extract the correct profile from a interpolated sonde.
This is an ARM specific method which extract the correct profile out of
netCDF Variables from a Interpolated Sonde VAP for the volume start time
of a radar object.
Parameters
----------
sonde_dset : Dataset
Interpolate sonde Dataset.
radar : Radar
Radar object from which the nearest profile will be found.
time_key : string, optional
Key to find a CF startard time variable
height_key : string, optional
Key to find profile height data
nvars : list, optional
NetCDF variable to generated profiles for. If None (the default) all
variables with dimension of time, height will be found in ncvars.
Returns
-------
return_dic : dict
Profiles at the start time of the radar
"""
ncvars = sonde_dset.variables
if nvars is None:
time_height_shape = (len(ncvars[time_key]), len(ncvars[height_key]))
nvars = [k for k, v in ncvars.items() if v.shape == time_height_shape]
radar_start = netCDF4.num2date(radar.time['data'][0], radar.time['units'])
radar_day_start = netCDF4.datetime(radar_start.year, radar_start.month,
radar_start.day)
seconds_since_start_of_day = (radar_start - radar_day_start).seconds
time_index = abs(ncvars[time_key][:] - seconds_since_start_of_day).argmin()
return_dic = dict([(key, ncvars[key][time_index, :]) for key in nvars])
return_dic[height_key] = ncvars[height_key][:]
return return_dic
def fetch_radar_time_profile(sonde_dset, radar, time_key='time',
height_key='height', nvars=None):
"""
Extract the correct profile from a interpolated sonde.
This is an ARM specific method which extract the correct profile out of
netCDF Variables from a Interpolated Sonde VAP for the volume start time
of a radar object.
Parameters
----------
sonde_dset : Dataset
Interpolate sonde Dataset.
radar : Radar
Radar object from which the nearest profile will be found.
time_key : string, optional
Key to find a CF startard time variable
height_key : string, optional
Key to find profile height data
nvars : list, optional
NetCDF variable to generated profiles for. If None (the default) all
variables with dimension of time, height will be found in ncvars.
Returns
-------
return_dic : dict
Profiles at the start time of the radar
"""
ncvars = sonde_dset.variables
if nvars is None:
time_height_shape = (len(ncvars[time_key]), len(ncvars[height_key]))
nvars = [k for k, v in ncvars.items() if v.shape == time_height_shape]
radar_start = num2date(radar.time['data'][0], radar.time['units'])
radar_day_start = datetime(radar_start.year, radar_start.month,
radar_start.day)
seconds_since_start_of_day = (radar_start - radar_day_start).seconds
time_index = abs(ncvars[time_key][:] - seconds_since_start_of_day).argmin()
return_dic = dict([(key, ncvars[key][time_index, :]) for key in nvars])
return_dic[height_key] = ncvars[height_key][:]
return return_dic
##########################################################################################
##########################################################################################
season_names = ['djf', 'mam', 'jja', 'son', 'ondjfm']
season = 'djf'
#season='jja'
#season='annual'
##########################################################################################
##########################################################################################
##########################################################################################
# (YYYY,MM,DD)
hist_start = netCDF4.datetime(1979, 12, 1)
hist_end = netCDF4.datetime(2010, 2, 28)
sst_histmonths_list = []
sst_histclim_list = []
hist_stdevs_list = []
#ncfile = Dataset('/Users/baird/google_drive/_data_original/era_interim/u200_MERRA_monthly_197901-201412_invertlat_96x144regrid.nc', 'r', format='NETCDF4')
ncfile = netCDF4.Dataset(
'/ninod/baird/cmip5/obs/MERRA/u200/MERRA_197901-201412_u200_monthly_2.5x2.5regrid.nc',
'r',
format='NETCDF4')
u200_hist_data = ncfile.variables['u200'][:]
u200_lat = ncfile.variables['lat'][:]
u200_lon = ncfile.variables['lon'][:]
def get2Ddata(grdROMS, grdMODEL, myvar, indatatype, year, month, day, varNames, dataPath):
indexROMS_SSH = (grdROMS.eta_rho, grdROMS.xi_rho)
if indatatype == "NORESM":
if myvar == 'ageice': varN = 5; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'uice': varN = 6; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'vice': varN = 7; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'aice': varN = 8; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'hice': varN = 9; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'snow_thick': varN = 10; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if indatatype == "GLORYS":
if myvar == 'uice': varN = 5; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'vice': varN = 6; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'aice': varN = 7; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'hice': varN = 8; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'ssh': varN = 2; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if grdMODEL.useESMF:
if indatatype == "SODA":
filename = getSODAfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[varNames[varN]][0,:,:]
if indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][:,:]
if indatatype == "WOAMONTHLY":
filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][month - 1, :,:]
if indatatype == "NORESM":
cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath))
#myunits = cdf.variables[str(varNames[varN])].units
# For NORESM data are 12 months of data stored in ice files. Use ID as month indicator to get data.
if myvar in ['ageice','uice','vice','aice','hice','snow_thick']:
data = np.squeeze(cdf.variables[str(varNames[varN])][month-1, :,:])
else:
data = np.squeeze(cdf.variables[str(varNames[varN])][0, :,:])
data=np.where(data.mask,grdROMS.fill_value,data)
if indatatype == "GLORYS":
cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath))
data = np.squeeze(cdf.variables[str(varNames[varN])][0, :,:])
data=np.where(data.mask,grdROMS.fill_value,data)
if indatatype == "NS8KMZ":
filename, readFromOneFile = getNS8KMZfilename(year, month, day, varNames[varN], dataPath)
cdf = Dataset(filename)
print "Reading from file", filename
if (readFromOneFile):
jdref = date2num(datetime(1948,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar)
currentdate = datetime(year, month, day, 12)
jd = date2num(currentdate, cdf.variables["time"].units, calendar="gregorian")
timesteps = (cdf.variables["time"][:]).tolist()
timeindex = timesteps.index(jd)
data = np.squeeze(cdf.variables[str(varNames[varN])][timeindex,:,:])
else:
data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:])
data=np.where(data.mask,grdROMS.fill_value,data)
print "Extracted raw data: %s min: %s max: %s"%(myvar,np.min(data),np.max(data))
cdf.close()
else:
if grdMODEL.splitExtract is True:
if indatatype == "SODA":
filename = getSODAfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data1 = cdf.variables[varNames[varN]][0,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
data2 = cdf.variables[varNames[varN]][0,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]
if indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data1 = cdf.variables[str(varNames[varN])][
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
data2 = cdf.variables[str(varNames[varN])][
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]
if indatatype == "GLORYS":
cdf = Dataset(getGLORYSfilename(year, month, day, '2D', dataPath))
data1 = np.squeeze(cdf.variables[str(varNames[varN])][0,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
data2 = np.squeeze(cdf.variables[str(varNames[varN])][0,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])])
if indatatype == "NORESM":
cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath))
if myvar in ['ageice','uice','vice','aice','hice','snow_thick']:
data1 = np.squeeze(cdf.variables[str(varNames[varN])][month-1,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
data2 = np.squeeze(cdf.variables[str(varNames[varN])][month-1,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])])
else:
data1 = np.squeeze(cdf.variables[str(varNames[varN])][0,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
data2 = np.squeeze(cdf.variables[str(varNames[varN])][0,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])])
cdf.close()
data = np.concatenate((data1, data2), axis=1)
else:
if indatatype == "SODA":
filename = getSODAfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][0,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
if indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
if indatatype == "GLORYS":
cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath))
data = np.squeeze(cdf.variables[str(varNames[varN])][0,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
if indatatype == "NORESM":
cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath))
if myvar in ['ageice','uice','vice','aice','hice','snow_thick']:
data = np.squeeze(cdf.variables[str(varNames[varN])][month-1,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
else:
data = np.squeeze(cdf.variables[str(varNames[varN])][0,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
cdf.close()
# data = np.where(abs(data) == grdMODEL.fill_value, grdROMS.fill_value, data)
#data = np.ma.where(abs(data) > 1000, grdROMS.fill_value, data)
if indatatype == "GLORYS":
data = np.where(data <= -32.767, grdROMS.fill_value, data)
data = np.ma.masked_where(data <= grdROMS.fill_value, data)
if __debug__:
print "Data range of %s just after extracting from netcdf file: %s - %s" % (str(varNames[varN]),
data.min(), data.max())
return data
def main():
print '\n--------------------------\n'
print 'Started ' + time.ctime(time.time())
# EDIT ===================================================================
# Set show_progress to "False" if you do not want to see the progress
# indicator for horizontal interpolation.
show_progress = True
# Set compileAll to True if you want automatic re-compilation of all the
# fortran files necessary to run model2roms. Options are "gfortran" or "ifort". Edit
# compile.py to add other Fortran compilers.
compileAll = False
if compileAll is True:
import compile
compile.compileAll("ifort")
# Extract time-series of data for given longitude/latitude
extractStations = False
# Define a set of longitude/latitude positions with names to extract into
# station files (using extractStations)
if (extractStations):
stationNames = [
'NorthSea', 'Iceland', 'EastandWestGreenland', 'Lofoten',
'Georges Bank'
]
lonlist = [2.4301, -22.6001, -47.0801, 13.3801, -67.2001]
latlist = [54.5601, 63.7010, 60.4201, 67.5001, 41.6423]
# Create the bry, init, and clim files for a given grid and input data
createOceanForcing = True
# Create atmospheric forcing for the given grid
createAtmosForcing = False
# Create a smaller resolution grid based on your original. Decimates every second for
# each time run
decimateGridfile = False
# Write ice values to file (for Arctic regions)
writeIce = True
# Use ESMF for the interpolation. This requires that you have ESMF and ESMPy installed (import ESMF)
useESMF = True
# Apply filter to smooth the 2D fields after interpolation (time consuming)
useFilter = True
# Format to write the ouput to: 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', or 'NETCDF3_CLASSIC'
# Using NETCDF4 automatically turns on compression of files (ZLIB)
myformat = 'NETCDF4'
# Frequency of the input data: usually monthly
timeFrequencyOfInputData = "month" #, "month", "hour"
# Subset input data. If you have global data you may want to seubset these to speed up reading. Make
# sure that your input data are cartesian (0-360 or -180:180, -90:90)
subsetIndata = False
if subsetIndata:
subset = defineSubsetForIndata()
# Set the input data MODEL indatatype
indatatype = 'SODA'
indatatype = 'SODAMONTHLY'
indatatype = 'WOAMONTHLY'
indatatype = 'NORESM'
indatatype = 'GLORYS'
#indatatype = 'NS8KM'
#indatatype = 'NS8KMZ'
# GRIDTYPES ------------------------------------------------------------------------------
# Define what grid type you wnat to interpolate to
outgrid = "NS8KM"
#outgrid = "REGSCEN"
#outgrid = "KINO"
# outgrid= "AA10KM"
# Define what grid type you wnat to interpolate from: Can be Z for SIGMA for ROMS
# vertical coordinate system or ZLEVEL
ingridtype = "SIGMA"
ingridtype = "ZLEVEL"
outgridtype = "ROMS"
# PATH TO DATA ----------------------------------------------------------------------------
# Define the path to the input data
if indatatype == 'SODA':
modelpath = "/Volumes/MacintoshHD2/Datasets/SODA/"
if indatatype == 'SODAMONTHLY':
modelpath = "/Volumes/MacintoshHD2/Datasets/SODAMonthly/"
if indatatype == 'GLORYS':
modelpath = "/Volumes/MacintoshHD2/Datasets/GLOBAL_REANALYSIS_PHYS_001_009/"
modelpath = "/Users/trondkr/Projects/is4dvar/GLORYS2V3/"
modelpath = "/work/shared/imr/NS8KM/FORCING/GLORYS2V3/ftp.myocean.mercator-ocean.fr/Core/GLOBAL_REANALYSIS_PHYS_001_009/"
if indatatype == 'NORESM':
modelpath = "/Users/trondkr/Projects/RegScen/NRCP45AERCN_f19_g16_CLE_01/"
#modelpath = "/work/users/trondk/REGSCEN/NRCP45AERCN_f19_g16_CLE_01/"
#modelpath = "/work/users/trondk/REGSCEN/N20TRAERCN/"
if createAtmosForcing:
atmospath = "/Users/trondkr/Projects/RegScen/model2roms/TESTFILES/"
if indatatype == 'NS8KM':
modelpath = "/Users/trondkr/Projects/is4dvar/grid2lonlat/RESULTS/"
if indatatype == 'NS8KMZ':
modelpath = "/Users/trondkr/Projects/NOWMAPS/delivery-COPERNICUS/"
#modelpath = "/work/shared/imr/NS8KM/Z-LEVEL-ASSIMILATION2010-2013/"
if indatatype == 'WOAMONTHLY':
modelpath = "/Users/trondkr/Projects/is4dvar/createSSS/"
# PATH TO GRID -----------------------------------------------------------------------------
# Define the path to the grid file
if outgrid == "NS8KM":
romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_smoothed02022015.nc"
romsgridpath = "/work/shared/imr/NS8KM/FORCING/GRID/nordsjoen_8km_grid_hmax20m_v4.nc"
if outgrid == "KINO":
romsgridpath = "/work/users/trondk/KINO/GRID/kino_1600m_19022016_vf20.nc"
#romsgridpath = "/Users/trondkr/Projects/KINO/GRID/kino_1600m_07082015_vf20.nc"
if outgrid == "REGSCEN":
romsgridpath = "/Users/trondkr/Projects/RegScen/Grid/AA_10km_grid_noest.nc"
#romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_grid_hmax20m_v3.nc"
#romsgridpath = "/work/users/trondk/REGSCEN/GRID/AA_10km_grid_noest.nc"
if outgrid == "GREENLAND":
romsgridpath = "/Users/trondkr/Projects/RegScen/Grid/Sermilik_grid_4000m.nc"
romsgridpath = "/Users/trondkr/Projects/RegScen/model2roms/polarlowr_grid.nc"
if outgrid == "AA10KM":
romsgridpath = "/work-common/shared/imr/AA10KM/GRID/AA_10km_grid.nc"
if indatatype == 'WOAMONTHLY': isClimatology = True
else: isClimatology = False
# DETAILS -----------------------------------------------------------------------------------
# Define the period to create forcing for
start_year = 2013
end_year = 2014
start_month = 11
end_month = 12
start_day = 15
end_day = 15
if (int(calendar.monthrange(start_year, start_month)[1]) < start_day):
start_day = int(calendar.monthrange(start_year, start_month)[1])
if (int(calendar.monthrange(end_year, end_month)[1]) < end_day):
end_day = int(calendar.monthrange(end_year, end_month)[1])
startdate = datetime(start_year, start_month, start_day)
enddate = datetime(end_year, end_month, end_day)
years = [start_year + year for year in xrange(end_year + 1 - start_year)]
# Define what and name of variables to include in the forcing files
# -> myvars is the name model2roms uses to identify variables
# -> varNames is the name of the variable found in the NetCDF input files
if indatatype == 'SODA':
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = ['TEMP', 'SALT', 'SSH', 'U', 'V']
if indatatype == 'SODAMONTHLY':
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = ['temp', 'salt', 'ssh', 'u', 'v']
if indatatype == 'NS8KM':
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty']
if indatatype == 'NS8KMZ':
fileNameIn, readFromOneFile = model2roms.getNS8KMZfilename(
startdate.year, startdate.month, startdate.day, "S", modelpath)
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
if (readFromOneFile):
varNames = ['temp', 'salt', 'zeta', 'u_eastward', 'v_northward']
else:
varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty']
if indatatype == 'GLORYS':
if (writeIce):
myvars = [
'temperature', 'salinity', 'ssh', 'uvel', 'vvel', 'uice',
'vice', 'aice', 'hice'
]
varNames = [
'votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty',
'iicevelu', 'iicevelv', 'ileadfra', 'iicethic'
]
else:
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = [
'votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty'
]
if indatatype == 'WOAMONTHLY':
myvars = ['temperature', 'salinity']
varNames = ['t_an', 's_an']
if indatatype == 'NORESM':
myvars = [
'temperature', 'salinity', 'ssh', 'uvel', 'vvel', 'ageice', 'uice',
'vice', 'aice', 'hice', 'snow_thick'
]
varNames = [
'templvl', 'salnlvl', 'sealv', 'uvellvl', 'vvellvl', 'iage',
'uvel', 'vvel', 'aice', 'hi', 'hs'
]
# NO EDIT BELOW ====================================================================================================
abbreviation = defineAbbreviation(outgrid)
climName, initName, bryName = defineOutputFilenames(
abbreviation, start_year, end_year, start_month, end_month, start_day,
end_day, indatatype)
if isClimatology is True:
climName = abbreviation + '_' + str(indatatype) + '_climatology.nc'
# Create the grid object for the output grid
grdROMS = grd.grdClass(romsgridpath, "ROMS", outgridtype, useESMF, 'ocean',
outgrid)
grdROMS.vars = myvars
if (useESMF):
import ESMF
manager = ESMF.Manager(logkind=ESMF.LogKind.MULTI, debug=True)
if createOceanForcing:
showInfo(myvars, romsgridpath, climName, initName, bryName, start_year,
end_year, isClimatology, useESMF, myformat)
model2roms.convertMODEL2ROMS(years,
startdate,
enddate,
timeFrequencyOfInputData,
climName,
initName,
modelpath,
romsgridpath,
myvars,
varNames,
show_progress,
indatatype,
outgridtype,
isClimatology,
writeIce,
useESMF,
useFilter,
myformat,
subsetIndata,
outgrid,
subset=None)
clim2bry.writeBry(grdROMS, start_year, bryName, climName, writeIce,
indatatype, myformat)
if createAtmosForcing:
atmosForcing.createAtmosFileUV(grdROMS, modelpath, atmospath,
startdate, enddate, useESMF, myformat,
abbreviation, indatatype, gridtype,
show_progress)
#if decimateGridfile:
#decimateGrid.createGrid(grdROMS, "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015.nc", "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015v2.nc", 2)
if extractStations:
print "Running in station mode and extracting pre-defined station locations"
IOstation.getStationData(years, IDS, modelpath, latlist, lonlist,
stationNames)
print 'Finished ' + time.ctime(time.time())
def main(in_dir="/RESCUE/skynet3_rech1/huziy/anusplin_links", out_dir="/HOME/huziy/skynet3_rech1/hail/anusplin_ts"):
out_dir_p = Path(out_dir)
in_dir_p = Path(in_dir)
lon0 = -114.0708
lat0 = 51.0486
vname = "daily_precipitation_accumulation"
vname_alternatives = ["daily_accumulation_precipitation"]
vname_alternatives.append(vname)
var_list = [vname]
fname_hint = "pcp"
spatial_ind = None
varname_to_list_of_frames = {vname: [] for vname in var_list}
for fin in in_dir_p.iterdir():
if fin.name.lower().endswith("ret"):
continue
if fin.name.lower().endswith("verif"):
continue
if fname_hint not in fin.name.lower():
continue
if not fin.name.endswith(".nc"):
continue
print(fin)
year, month = get_ym_from_path(fin)
with Dataset(str(fin)) as ds:
if spatial_ind is None:
lons, lats = ds.variables["lon"][:], ds.variables["lat"][:]
x, y, z = lat_lon.lon_lat_to_cartesian(lons.flatten(), lats.flatten())
ktree = KDTree(list(zip(x, y, z)))
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon0, lat0)
dist, spatial_ind = ktree.query((x0, y0, z0))
for vname_alt in vname_alternatives:
try:
values = ds[vname_alt][:]
values = [field.flatten()[spatial_ind] for field in values]
break
except IndexError as ierr:
pass
dates = [datetime(year, month, int(d)) for d in ds["time"][:]]
varname_to_list_of_frames[vname].append(pd.DataFrame(index=dates, data=values))
for vname in var_list:
df = pd.concat(varname_to_list_of_frames[vname])
assert isinstance(df, pd.DataFrame)
df.sort_index(inplace=True)
df.to_csv(str(out_dir_p.joinpath("{}.csv".format(vname))), float_format="%.3f", index_label="Time")
def get2ddata(confM2R, myvar, year, month, day):
indexROMS_SSH = (confM2R.grdROMS.eta_rho, confM2R.grdROMS.xi_rho)
if confM2R.indatatype == "NORESM":
if myvar == 'ageice':
varN = 5
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'uice':
varN = 6
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'vice':
varN = 7
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'aice':
varN = 8
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'hice':
varN = 9
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'snow_thick':
varN = 10
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if confM2R.indatatype == "GLORYS":
if myvar == 'uice':
varN = 5
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'vice':
varN = 6
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'aice':
varN = 7
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'hice':
varN = 8
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
print("Current type %s and variable %s" % (confM2R.indatatype, myvar))
if confM2R.indatatype == "SODA3":
if myvar == 'aice':
varN = 5
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'hice':
varN = 6
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'snow_thick':
varN = 7
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if myvar == 'ssh':
varN = 2
SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64)
if confM2R.useesmf:
if confM2R.indatatype == "SODA":
filename = getSODAfilename(confM2R, year, month, day, None)
cdf = Dataset(filename)
data = cdf.variables[confM2R.inputdatavarnames[varN]][0, :, :]
if confM2R.indatatype == "SODA3":
filename = getSODA3filename(confM2R, year, month,
confM2R.inputdatavarnames[varN])
print("Trying to open file %s" % (filename))
cdf = Dataset(filename)
if myvar == 'aice':
# We only extract the first thickness concentration. Need to fix this so all 5 classes can be extracted.
# http://www.atmos.umd.edu/~ocean/index_files/soda3_readme.htm
# hi: sea ice thickness [m ice]
# mi: sea ice mass [kg/m^2]
# hs: snow thickness [m snow]
# {cn1,cn2,cn3,cn4,cn5}: sea ice concentration [0:1] in five ice thickness classes
data = cdf.variables[confM2R.inputdatavarnames[varN]][
int(month - 1), 0, :, :]
else:
data = cdf.variables[confM2R.inputdatavarnames[varN]][
int(month - 1), :, :]
if confM2R.indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(confM2R, year, month, None)
cdf = Dataset(filename)
data = cdf.variables[str(confM2R.inputdatavarnames[varN])][:, :]
if confM2R.indatatype == "WOAMONTHLY":
filename = getWOAMONTHLYfilename(confM2R, year, month, myvar)
cdf = Dataset(filename)
data = cdf.variables[str(confM2R.inputdatavarnames[varN])][month -
1, :, :]
if confM2R.indatatype == "NORESM":
cdf = Dataset(
getNORESMfilename(confM2R, year, month,
confM2R.inputdatavarnames[varN]))
# myunits = cdf.variables[str(grdROMS.varNames[varN])].units
# For NORESM data are 12 months of data stored in ice files. Use ID as month indicator to get data.
if myvar in [
'ageice', 'uice', 'vice', 'aice', 'hice', 'snow_thick'
]:
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][month - 1, :, :])
else:
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][0, :, :])
data = np.where(data.mask, confM2R.grdROMS.fillval, data)
if confM2R.indatatype == "GLORYS":
cdf = Dataset(
getGLORYSfilename(confM2R, year, month,
confM2R.inputdatavarnames[varN]))
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][0, :, :])
data = np.where(data.mask, confM2R.grdROMS.fillval, data)
if confM2R.indatatype == "NS8KMZ":
filename, readFromOneFile = getNS8KMZfilename(
confM2R, year, month, confM2R.inputdatavarnames[varN])
cdf = Dataset(filename)
print("Reading from file", filename)
if (readFromOneFile):
jdref = date2num(datetime(1948, 1, 1),
cdf.variables["time"].units,
calendar=cdf.variables["time"].calendar)
currentdate = datetime(year, month, day, 12)
jd = date2num(currentdate,
cdf.variables["time"].units,
calendar="gregorian")
timesteps = (cdf.variables["time"][:]).tolist()
timeindex = timesteps.index(jd)
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][timeindex, :, :])
else:
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][0, :, :])
data = np.where(data.mask, confM2R.grdROMS.fillval, data)
print("Extracted raw data: %s min: %s max: %s" %
(myvar, np.min(data), np.max(data)))
cdf.close()
if __debug__:
print(
"Data range of %s just after extracting from netcdf file: %s - %s"
%
(str(confM2R.inputdatavarnames[varN]), data.min(), data.max()))
return data
def getTime(confM2R, year, month, day):
"""
Create a date object to keep track of Julian dates etc.
Also create a reference date starting at 1948/01/01.
Go here to check results:http://lena.gsfc.nasa.gov/lenaDEV/html/doy_conv.html
"""
if confM2R.indatatype == 'SODA':
filename = getSODAfilename(confM2R, year, month, None)
if confM2R.indatatype == 'SODA3':
filename = getSODA3filename(confM2R, year, month, None)
if confM2R.indatatype == 'SODAMONTHLY':
filename = getSODAMONTHLYfilename(confM2R, year, month, None)
if confM2R.indatatype == 'GLORYS':
filename = getGLORYSfilename(confM2R, year, month, "S")
if confM2R.indatatype == 'WOAMONTHLY':
filename = getWOAMONTHLYfilename(confM2R, year, month, "temperature")
if confM2R.indatatype == 'NORESM':
filename = getNORESMfilename(confM2R, year, month, "saln")
if confM2R.indatatype == 'NS8KM':
filename = getNS8KMfilename(confM2R, year, month, "salt")
if confM2R.indatatype == 'NS8KMZ':
filename, readFromOneFile = getNS8KMZfilename(confM2R, year, month,
"salt")
# Now open the input file and get the time
cdf = Dataset(filename)
if confM2R.indatatype == 'NORESM':
jdref = date2num(datetime(1800, 1, 1),
cdf.variables["time"].units,
calendar=cdf.variables["time"].calendar)
elif confM2R.indatatype == 'NS8KMZ':
jdref = date2num(datetime(1948, 1, 1),
units="days since 1948-01-01 00:00:00",
calendar="standard")
elif confM2R.indatatype == 'GLORYS':
jdref = date2num(datetime(1948, 1, 1),
cdf.variables["time_counter"].units,
calendar=cdf.variables["time_counter"].calendar)
elif confM2R.indatatype == 'NS8KM':
jdref = date2num(datetime(1948, 1, 1),
cdf.variables["ocean_time"].units,
calendar=cdf.variables["ocean_time"].calendar)
elif confM2R.indatatype == 'SODA3':
jdref = date2num(datetime(1948, 1, 1),
units="days since 1948-01-01 00:00:00",
calendar="standard")
else:
jdref = date2num(datetime(1948, 1, 1),
cdf.variables["time"].units,
calendar=cdf.variables["time"].calendar)
if confM2R.indatatype == 'SODAMONTHLY':
# Find the day and month that the SODAMONTHLY file respresents based on the year and ID number.
# Each SODA file represents a 1 month average.
mycalendar = cdf.variables["time"].calendar
myunits = cdf.variables["time"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate, myunits, calendar=mycalendar)
if confM2R.indatatype == 'SODA3':
# Each SODA file represents 12 month averages.
myunits = cdf.variables["time"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate,
units="days since 1948-01-01 00:00:00",
calendar="standard")
if confM2R.indatatype == 'GLORYS':
# Find the day and month that the GLORYS file respresents based on the year and ID number.
# Each file represents a 1 month average.
mycalendar = cdf.variables["time_counter"].calendar
myunits = cdf.variables["time_counter"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate, myunits, calendar=mycalendar)
if confM2R.indatatype == 'NS8KM':
# Find the day and month that the GLORYS file respresents based on the year and ID number.
# Each file represents a 1 month average.
mycalendar = cdf.variables["ocean_time"].calendar
myunits = cdf.variables["ocean_time"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate, myunits, calendar=mycalendar)
if confM2R.indatatype == 'NS8KMZ':
# Find the day and month that the GLORYS file respresents based on the year and ID number.
# Each file represents a 1 month average.
currentdate = datetime(year, month, day)
myunits = cdf.variables["time"].units
jd = date2num(currentdate, myunits, calendar="gregorian")
print("Days:", jd, currentdate, year, month, day)
if confM2R.indatatype == 'NORESM':
# Find the day and month that the NORESM file. We need to use the time modules from
# netcdf4 for python as they handle calendars that are no_leap.
# http://www.esrl.noaa.gov/psd/people/jeffrey.s.whitaker/python/netcdftime.html#datetime
mydays = cdf.variables["time"][0]
mycalendar = cdf.variables["time"].calendar
myunits = cdf.variables["time"].units
currentdate = num2date(mydays, units=myunits, calendar=mycalendar)
jd = date2num(currentdate, myunits, calendar='noleap')
confM2R.grdROMS.time = (jd - jdref)
confM2R.grdROMS.reftime = jdref
confM2R.grdROMS.timeunits = myunits
cdf.close()
print("-------------------------------")
print('\nCurrent time of %s file : %s' % (confM2R.indatatype, currentdate))
print("-------------------------------")
def date2dec(calendar = 'standard', units=None,
excelerr = True, yr=1,
mo=1, dy=1, hr=0, mi=0, sc=0,
ascii=None, en=None, eng=None):
"""
Convert scalar and array_like with calendar dates into decimal
dates. Supported calendar formats are standard, gregorian, julian,
proleptic_gregorian, excel1900, excel1904, 365_day, noleap, 366_day,
all_leap, 360_day, decimal, or decimal360.
Input is year, month day, hour, minute, second or a combination of them.
Input as date string is possible.
Output is decimal date with day as unit.
Parameters
----------
yr : array_like, optional
years (default: 1)
mo : array_like, optional
month (default: 1)
dy : array_like, optional
days (default: 1)
hr : array_like, optional
hours (default: 0)
mi : array_like, optional
minutes (default: 0)
sc : array_like, optional
seconds (default: 0)
ascii : array_like, optional
strings of the format 'dd.mm.yyyy hh:mm:ss'.
Missing hour, minutes and/or seconds are set
to their default values (0).
`ascii` overwrites all other keywords.
`ascii` and `eng` are mutually exclusive.
en : array_like, optional
strings of the format 'yyyy-mm-dd hh:mm:ss'.
Missing hour, minutes and/or seconds are set
to their default values (0).
`en` overwrites all other keywords.
`en` and `ascii` are mutually exclusive.
eng : array_like, optional
Same as en: obsolete.
calendar : str, optional
Calendar of output dates (default: 'standard').
Possible values are:
'standard', 'gregorian' = julian calendar from
01.01.-4712 12:00:00 (BC) until 05.03.1583 00:00:00 and
gregorian calendar from 15.03.1583 00:00:00 until now.
Missing 10 days do not exsist.
'julian' = julian calendar from 01.01.-4712 12:00:00 (BC)
until now.
'proleptic_gregorian' = gregorian calendar from
01.01.0001 00:00:00 until now.
'excel1900' = Excel dates with origin at
01.01.1900 00:00:00.
'excel1904' = Excel 1904 (Lotus) format.
Same as excel1904 but with origin at
01.01.1904 00:00:00.
'365_day', 'noleap' = 365 days format,
i.e. common years only (no leap years)
with origin at 01.01.0001 00:00:00.
'366_day', 'all_leap' = 366 days format,
i.e. leap years only (no common years)
with origin at 01.01.0001 00:00:00.
'360_day' = 360 days format,
i.e. years with only 360 days (30 days per month)
with origin at 01.01.0001 00:00:00.
'decimal' = decimal year instead of decimal days.
'decimal360' = decimal year with a year of 360 days, i.e. 12 month with 30 days each.
units : str, optional
User set units of output dates. Must be a
string in the format 'days since yyyy-mm-dd hh:mm:ss'.
Default values are set automatically depending on `calendar`.
excelerr : bool, optional
In Excel, the year 1900 is normally considered a leap year,
which it was not. By default, this error is taken into account
if `calendar=='excel1900'` (default: True).
1900 is not considered a leap year if `excelerr==False`.
Returns
-------
array_like
array_like with decimal dates. The type of output will be the same as the input type.
Notes
-----
Most versions of `datetime` do not support negative years,
i.e. Julian days < 1721423.5 = 01.01.0001 00:00:00.
There is an issue in `netcdftime` version < 0.9.5 in proleptic_gregorian for dates before year 301:
dec2date(date2dec(ascii='01.01.0300 00:00:00', calendar='proleptic_gregorian'), calendar='proleptic_gregorian')
[300, 1, 2, 0, 0, 0]
dec2date(date2dec(ascii='01.01.0301 00:00:00', calendar='proleptic_gregorian'), calendar='proleptic_gregorian')
[301, 1, 1, 0, 0, 0]
List input is only supported up to 2 dimensions.
Requires `netcdftime.py` from module `netcdftime` available at:
http://netcdf4-python.googlecode.com
Examples
--------
# Some implementations of datetime do not support negative years
>>> import datetime
>>> if datetime.MINYEAR > 0:
... print('The minimum year in your datetime implementation is ', datetime.MINYEAR)
... print('i.e. it does not support negative years (BC).')
The minimum year in your datetime implementation is 1
i.e. it does not support negative years (BC).
>>> if datetime.MINYEAR > 0:
... year = np.array([2000, 1810, 1630, 1510, 1271, 619, 2, 1])
... else:
... year = np.array([2000, 1810, 1630, 1510, 1271, 619, -1579, -4712])
>>> month = np.array([1, 4, 7, 9, 3, 8, 8, 1])
>>> day = np.array([5, 24, 15, 20, 18, 27, 23, 1])
>>> hour = np.array([12, 16, 10, 14, 19, 11, 20, 12])
>>> minute = np.array([30, 15, 20, 35, 41, 8, 3, 0])
>>> second = np.array([15, 10, 40, 50, 34, 37, 41, 0])
>>> decimal = date2dec(calendar = 'standard', yr=year, mo=month, dy=day, hr=hour, mi=minute, sc=second)
>>> nn = year.size
>>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2]))
2.45154902100694e+06 2.38226217719907e+06 2.31660093101852e+06 2.27284810821759e+06
>>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2]))
2.18536732053241e+06 1.94738596431713e+06
>>> decimal = date2dec(calendar='standard', yr=year, mo=6, dy=15, hr=12, mi=minute, sc=second)
>>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2]))
2.45171102100694e+06 2.38231401053241e+06 2.31657101435185e+06 2.27275102488426e+06
>>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2]))
2.18545602886574e+06 1.94731300598380e+06
# ascii input
>>> if datetime.MINYEAR > 0:
... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50',
... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.0002 20:03:41', '01.01.0001 12:00:00'])
... else:
... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50',
... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.-1579 20:03:41', '01.01.-4712 12:00:00'])
>>> decimal = date2dec(calendar='standard', ascii=a)
>>> nn = a.size
>>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2]))
2.45154902100694e+06 2.38226217719907e+06 2.31660093101852e+06 2.27284810821759e+06
>>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2]))
2.18536732053241e+06 1.94738596431713e+06
# calendar = 'julian'
>>> decimal = date2dec(calendar='julian', ascii=a)
>>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2]))
2.45156202100694e+06 2.38227417719907e+06 2.31661093101852e+06 2.27284810821759e+06
>>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2]))
2.18536732053241e+06 1.94738596431713e+06
# calendar = 'proleptic_gregorian'
>>> decimal = date2dec(calendar='proleptic_gregorian', ascii=a)
>>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2]))
730123.5210069 660836.6771991 595175.4310185 551412.6082176
>>> print('{:.7f} {:.7f}'.format(*decimal[nn//2:nn-2]))
463934.8205324 225957.4643171
# calendar = 'excel1900' WITH excelerr=True -> 1900 considered as leap year
>>> d = np.array(['05.01.2000 12:30:15', '27.05.1950 16:25:10', '13.08.1910 10:40:55',
... '01.03.1900 00:00:00', '29.02.1900 00:00:00', '28.02.1900 00:00:00',
... '01.01.1900 00:00:00'])
>>> decimal = date2dec(calendar='excel1900', ascii=d)
>>> nn = d.size
>>> print('{:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2]))
36530.5210069 18410.6841435 3878.4450810
>>> print('{:.1f} {:.1f} {:.1f} {:.1f}'.format(*decimal[nn//2:]))
61.0 60.0 59.0 1.0
# calendar = 'excel1900' WITH excelerr = False -> 1900 is NO leap year
>>> decimal = date2dec(calendar='excel1900', ascii=d, excelerr=False)
>>> print('{:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2]))
36529.5210069 18409.6841435 3877.4450810
>>> print('{:.1f} {:.1f} {:.1f} {:.1f}'.format(*decimal[nn//2:]))
60.0 60.0 59.0 1.0
# calendar = 'excel1904'
>>> decimal = date2dec(calendar='excel1904', ascii=d[:nn//2])
>>> print('{:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2]))
35069.5210069 16949.6841435 2417.4450810
# calendar = '365_day'
>>> g = np.array(['18.08.1972 12:30:15', '25.10.0986 12:30:15', '28.11.0493 22:20:40', '01.01.0001 00:00:00'])
>>> decimal = date2dec(calendar='365_day', ascii=g)
>>> nn = g.size
>>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn]))
719644.5210069 359822.5210069 179911.9310185 0.0000000
# calendar = '366_day'
>>> decimal = date2dec(calendar='366_day', ascii=g)
>>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn]))
721616.5210069 360808.5210069 180404.9310185 0.0000000
# 360_day does not work with netcdftime.py version equal or below 0.9.2
# calendar = '360_day'
>>> decimal = date2dec(calendar='360_day', ascii=g)
>>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn]))
709787.5210069 354894.5210069 177447.9310185 0.0000000
>>> print('{:.7f}'.format(date2dec(yr=1992, mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal')))
1992.0685337
>>> print('{:.7f}'.format(date2dec(ascii='26.01.1992 02:00', calendar='decimal360')))
1992.0696759
>>> print('{:.7f} {:.7f}'.format(*date2dec(ascii=['26.01.1992 02:00','26.01.1992 02:00'], calendar='decimal360')))
1992.0696759 1992.0696759
>>> print('{:.7f} {:.7f}'.format(*date2dec(yr=[1992,1992], mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360')))
1992.0696759 1992.0696759
>>> print('{:.7f} {:.7f}'.format(*date2dec(yr=np.array([1992,1992]), mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360')))
1992.0696759 1992.0696759
>>> decimal = date2dec(ascii=[['26.01.1992 02:00','26.01.1992 02:00'],
... ['26.01.1992 02:00','26.01.1992 02:00'],
... ['26.01.1992 02:00','26.01.1992 02:00']],
... calendar='decimal360')
>>> print('{:.7f} {:.7f}'.format(*decimal[0]))
1992.0696759 1992.0696759
>>> print('{:.7f} {:.7f}'.format(*decimal[2]))
1992.0696759 1992.0696759
>>> print((date2dec(ascii='01.03.2003 00:00:00') - date2dec(ascii='01.03.2003')) == 0.)
True
# en
>>> decimal = date2dec(en='1992-01-26 02:00', calendar='decimal360')
>>> print('{:.7f}'.format(decimal))
1992.0696759
>>> decimal = date2dec(eng='1992-01-26 02:00', calendar='decimal360')
>>> print('{:.7f}'.format(decimal))
1992.0696759
History
-------
Written Arndt Piayda, Jun 2010
Modified Matthias Cuntz, Feb 2012 - All input can be scalar, list or array, also a mix
- Changed checks for easier extension
- decimal, decimal360
Matthias Cuntz, Dec 2012 - change unit of proleptic_gregorian
from 'days since 0001-01-01 00:00:00'
to 'days since 0001-01-00 00:00:00'
Matthias Cuntz, Feb 2013 - solved Excel leap year problem.
Matthias Cuntz, Feb 2013 - ported to Python 3
Matthias Cuntz, Jul 2013 - ascii/eng without time defaults to 00:00:00
Matthias Cuntz, Oct 2013 - Excel starts at 1 not at 0
Matthias Cuntz, Oct 2013 - units bugs, e.g. 01.01.0001 was substracted if Julian calendar even with units
Matthias Cuntz, Nov 2013 - removed remnant of time treatment before time check in eng keyword
Matthias Cuntz, Jun 2015 - adapted to new netCDF4/netcdftime (>= v1.0) and datetime (>= Python v2.7.9)
Matthias Cuntz, Oct 2015 - call date2num with list instead of single netCDF4.datetime objects
Matthias Cuntz, Oct 2016 - netcdftime provided even with netCDF4 > 1.0.0; make mo for months always integer
Matthias Cuntz, Nov 2016 - 00, 01, etc. for integers not accepted by Python3
Matthias Cuntz, May 2020 - numpy docstring format
Matthias Cuntz, Jul 2020 - en for eng
Matthias Cuntz, Jul 2020 - use proleptic_gregorian for Excel dates
"""
#
# Checks
calendars = ['standard', 'gregorian', 'julian', 'proleptic_gregorian',
'excel1900', 'excel1904', '365_day', 'noleap', '366_day',
'all_leap', '360_day', 'decimal', 'decimal360']
import netCDF4 as nt
try:
tst = nt.date2num
tst = nt.datetime
except:
try:
import netcdftime as nt
if ((nt.__version__ <= '0.9.2') & (calendar == '360_day')):
raise ValueError("date2dec error: Your version of netcdftime.py is equal"
" or below 0.9.2. The 360_day calendar does not work with"
" arrays here. Please download a newer one.")
except:
import cftime as nt
#
calendar = calendar.lower()
if (calendar not in calendars):
raise ValueError("date2dec error: Wrong calendar!"
" Choose: "+''.join([i+' ' for i in calendars]))
# obsolete eng
if (eng is not None):
if (en is not None):
raise ValueError("date2dec error: 'eng' was succeeded by 'en'. Only one can be given.")
else:
en = eng
# if ascii input is given by user, other input will be neglected
# calculation of input size and shape
if (ascii is not None) and (en is not None):
raise ValueError("date2dec error: 'ascii' and 'en' mutually exclusive")
if (ascii is not None):
islist = type(ascii) != type(np.array(ascii))
isarr = np.ndim(ascii)
if (islist & (isarr > 2)):
raise ValueError("date2dec error: ascii input is list > 2D; Use array input")
if isarr == 0:
ascii = np.array([ascii])
else:
ascii = np.array(ascii)
insize = ascii.size
outsize = insize
outshape = ascii.shape
asciifl = ascii.flatten()
timeobj = np.zeros(insize, dtype=object)
# slicing of ascii strings to implement in datetime object. missing times
# will be set to 0.
yr = np.zeros(insize, dtype=np.int)
mo = np.zeros(insize, dtype=np.int)
dy = np.zeros(insize, dtype=np.int)
hr = np.zeros(insize, dtype=np.int)
mi = np.zeros(insize, dtype=np.int)
sc = np.zeros(insize, dtype=np.int)
for i in range(insize):
aa = asciifl[i].split('.')
dy[i] = int(aa[0])
mo[i] = int(aa[1])
tail = aa[2].split()
yr[i] = int(tail[0])
if len(tail) > 1:
tim = tail[1].split(':')
hr[i] = int(tim[0])
if len(tim) > 1:
mi[i] = int(tim[1])
else:
mi[i] = 0
if len(tim) > 2:
sc[i] = int(tim[2])
else:
sc[i] = 0
else:
hr[i] = 0
mi[i] = 0
sc[i] = 0
if ((yr[i]==1900) & (mo[i]==2) & (dy[i]==29)):
timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i])
else:
timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i], sc[i])
if (en is not None):
islist = type(en) != type(np.array(en))
isarr = np.ndim(en)
if isarr == 0:
en = np.array([en])
else:
en = np.array(en)
if (islist & (isarr > 2)):
raise ValueError("date2dec error: en input is list > 2D; Use array input")
en = np.array(en)
insize = en.size
outsize = insize
outshape = en.shape
enfl = en.flatten()
timeobj = np.zeros(insize, dtype=object)
# slicing of en strings to implement in datetime object. missing times
# will be set to 0.
yr = np.zeros(insize, dtype=np.int)
mo = np.zeros(insize, dtype=np.int)
dy = np.zeros(insize, dtype=np.int)
hr = np.zeros(insize, dtype=np.int)
mi = np.zeros(insize, dtype=np.int)
sc = np.zeros(insize, dtype=np.int)
for i in range(insize):
ee = enfl[i].split('-')
yr[i] = int(ee[0])
mo[i] = int(ee[1])
tail = ee[2].split()
dy[i] = int(tail[0])
if len(tail) > 1:
tim = tail[1].split(':')
hr[i] = int(tim[0])
if len(tim) > 1:
mi[i] = int(tim[1])
else:
mi[i] = 0
if len(tim) > 2:
sc[i] = int(tim[2])
else:
sc[i] = 0
else:
hr[i] = 0
mi[i] = 0
sc[i] = 0
if ((yr[i]==1900) & (mo[i]==2) & (dy[i]==29)):
timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i])
else:
timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i], sc[i])
# if no ascii input, other inputs will be concidered
# calculation of input sizes, shapes and number of axis
if (ascii is None) and (en is None):
isnlist1 = type(yr) == type(np.array(yr))
isarr1 = np.ndim(yr)
if isarr1 == 0: yr = np.array([yr])
else: yr = np.array(yr)
isnlist2 = type(mo) == type(np.array(mo))
isarr2 = np.ndim(mo)
if isarr2 == 0: mo = np.array([mo], dtype=np.int)
else: mo = np.array(mo, dtype=np.int)
isnlist3 = type(dy) == type(np.array(dy))
isarr3 = np.ndim(dy)
if isarr3 == 0: dy = np.array([dy])
else: dy = np.array(dy)
isnlist4 = type(hr) == type(np.array(hr))
isarr4 = np.ndim(hr)
if isarr4 == 0: hr = np.array([hr])
else: hr = np.array(hr)
isnlist5 = type(mi) == type(np.array(mi))
isarr5 = np.ndim(mi)
if isarr5 == 0: mi = np.array([mi])
else: mi = np.array(mi)
isnlist6 = type(sc) == type(np.array(sc))
isarr6 = np.ndim(sc)
if isarr6 == 0: sc = np.array([sc])
else: sc = np.array(sc)
islist = not (isnlist1 | isnlist2 | isnlist3 | isnlist4 | isnlist5 | isnlist6)
isarr = isarr1 + isarr2 + isarr3 + isarr4 + isarr5 + isarr6
shapes = [np.shape(yr), np.shape(mo), np.shape(dy), np.shape(hr), np.shape(mi), np.shape(sc)]
nyr = np.size(yr)
nmo = np.size(mo)
ndy = np.size(dy)
nhr = np.size(hr)
nmi = np.size(mi)
nsc = np.size(sc)
sizes = [nyr,nmo,ndy,nhr,nmi,nsc]
nmax = np.amax(sizes)
ii = np.argmax(sizes)
outshape = shapes[ii]
if (islist & (np.size(outshape) > 2)):
raise ValueError("date2dec error: input is list > 2D; Use array input.")
if nyr < nmax:
if nyr == 1: yr = np.ones(outshape,)*yr
else: raise ValueError("date2dec error: size of yr != max input or 1.")
if nmo < nmax:
if nmo == 1: mo = np.ones(outshape, dtype=np.int)*mo
else: raise ValueError("date2dec error: size of mo != max input or 1.")
if ndy < nmax:
if ndy == 1: dy = np.ones(outshape)*dy
else: raise ValueError("date2dec error: size of dy != max input or 1.")
if nhr < nmax:
if nhr == 1: hr = np.ones(outshape)*hr
else: raise ValueError("date2dec error: size of hr != max input or 1.")
if nmi < nmax:
if nmi == 1: mi = np.ones(outshape)*mi
else: raise ValueError("date2dec error: size of mi != max input or 1.")
if nsc < nmax:
if nsc == 1: sc = np.ones(outshape)*sc
else: raise ValueError("date2dec error: size of sc != max input or 1.")
indate = [yr, mo, dy, hr, mi, sc]
insize = [np.size(i) for i in indate]
inshape = [np.shape(i) for i in indate]
dimnum = [len(i) for i in inshape]
# calculation of maximum input size and maximum number of axis for
# reshaping the output
indate = [i.flatten() for i in indate]
outsize = max(insize)
timeobj = np.zeros(outsize, dtype=object)
# datetime object is constructed
for i in range(outsize):
iyr = int(indate[0][i])
imo = int(indate[1][i])
idy = int(indate[2][i])
ihr = int(indate[3][i])
imi = int(indate[4][i])
isc = int(indate[5][i])
if ((iyr==1900) & (imo==2) & (idy==29)):
timeobj[i] = nt.datetime(iyr, 3, 1, ihr, imi, isc)
else:
timeobj[i] = nt.datetime(iyr, imo, idy, ihr, imi, isc)
# depending on chosen calendar and optional set of the time units
# decimal date is calculated
output = np.zeros(outsize)
t0 = nt.datetime(1582, 10, 5, 0, 0, 0)
t1 = nt.datetime(1582, 10, 15, 0, 0, 0)
is121 = True if (min(timeobj)<t0) and (max(timeobj)>=t1) else False
if (calendar == 'standard') or (calendar == 'gregorian'):
if not units:
units = 'days since 0001-01-01 12:00:00'
dec0 = 1721424
else:
dec0 = 0
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='gregorian')+dec0
else:
output = nt.date2num(timeobj, units, calendar='gregorian')+dec0
elif calendar == 'julian':
if not units:
units = 'days since 0001-01-01 12:00:00'
dec0 = 1721424
else:
dec0 = 0
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='julian')+dec0
else:
output = nt.date2num(timeobj, units, calendar='julian')+dec0
elif calendar == 'proleptic_gregorian':
if not units: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='proleptic_gregorian')
else:
output = nt.date2num(timeobj, units, calendar='proleptic_gregorian')
elif calendar == 'excel1900':
doerr = False
if not units:
units = 'days since 1899-12-31 00:00:00'
if excelerr: doerr = True
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='proleptic_gregorian')
else:
output = nt.date2num(timeobj, units, calendar='proleptic_gregorian')
if doerr:
output = np.where(output >= 60., output+1., output)
# date2num treats 29.02.1900 as 01.03.1990, i.e. is the same decimal number
if np.any((output >= 61.) & (output < 62.)):
for i in range(outsize):
# if (timeobj[i].year==1900) & (timeobj[i].month==2) & (timeobj[i].day==29):
# output[i] -= 1.
if (yr[i]==1900) & (mo[i]==2) & (dy[i]==29):
output[i] -= 1.
elif calendar == 'excel1904':
if not units: units = 'days since 1903-12-31 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='proleptic_gregorian')
else:
output = nt.date2num(timeobj, units, calendar='proleptic_gregorian')
elif (calendar == '365_day') or (calendar == 'noleap'):
if not units: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='365_day')
else:
output = nt.date2num(timeobj, units, calendar='365_day')
elif (calendar == '366_day') or (calendar == 'all_leap'):
if not units: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='366_day')
else:
output = nt.date2num(timeobj, units, calendar='366_day')
elif calendar == '360_day':
if not units: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='360_day')
else:
output = nt.date2num(timeobj, units, calendar='360_day')
elif calendar == 'decimal':
ntime = np.size(yr)
leap = np.array((((yr%4)==0) & ((yr%100)!=0)) | ((yr%400)==0)).astype(np.int)
tdy = np.array(dy, dtype=np.float)
diy = np.array([ [-9,0, 31, 59, 90,120,151,181,212,243,273,304,334,365],
[-9,0, 31, 60, 91,121,152,182,213,244,274,305,335,366] ])
for i in range(ntime):
tdy[i] = tdy[i] + np.array(diy[leap[i],mo[i]], dtype=np.float)
days_year = 365.
output = ( np.array(yr, dtype=np.float) +
((tdy-1.)*24. + np.array(hr, dtype=np.float) +
np.array(mi, dtype=np.float)/60. +
np.array(sc, dtype=np.float)/3600.) /
((days_year+np.array(leap, dtype=np.float))*24.) )
# for numerical stability, i.e. back and forth transforms
output += 1e-08 # 1/3 sec
elif calendar == 'decimal360':
ntime = np.size(yr)
tdy = np.array(dy, dtype=np.float)
diy = np.array([-9, 0, 30, 60, 90,120,150,180,210,240,270,300,330,360])
for i in range(ntime):
tdy[i] = tdy[i] + np.array(diy[mo[i]], dtype=np.float)
days_year = 360.
output = ( np.array(yr, dtype=np.float) +
((tdy-1.)*24. + np.array(hr, dtype=np.float) +
np.array(mi, dtype=np.float)/60. +
np.array(sc, dtype=np.float)/3600.) /
(days_year*24.) )
# for numerical stability, i.e. back and forth transforms
output += 1e-08 # 1/3 sec
else:
raise ValueError("date2dec error: calendar not implemented; should have been catched before.")
# return of reshaped output
output = np.reshape(output, outshape)
if isarr == 0:
output = np.float(output)
else:
if islist:
ns = np.size(outshape)
if ns == 1:
output = [i for i in output]
else:
loutput = [ i for i in output[:,0]]
for i in range(np.size(output[:,0])):
loutput[i] = list(np.squeeze(output[i,:]))
output = loutput
return output
import pandas as pd
import matplotlib.path as mpath
import matplotlib.patches as mpatches
from matplotlib.collections import PatchCollection
import matplotlib.pyplot as plt
import ogr
import osr
from math import *
from scipy.ndimage.filters import gaussian_filter
import mpl_util
import laplace_filter
import cmocean
__author__ = 'Trond Kristiansen'
__email__ = '*****@*****.**'
__created__ = datetime(2017, 5, 16)
__modified__ = datetime(2017, 5, 16)
__version__ = "1.0"
__status__ = "Development, modified on 16.05.2017"
def getPathForPolygon(ring, mymap):
codes = []
x = [ring.GetX(j) for j in range(ring.GetPointCount())]
y = [ring.GetY(j) for j in range(ring.GetPointCount())]
codes += [mpath.Path.MOVETO] + (len(x) - 1) * [mpath.Path.LINETO]
pathX, pathY = mymap(x, y)
mymappath = mpath.Path(np.column_stack((pathX, pathY)), codes)
return mymappath
def plot_histograms(self, save=False, save_dir=None, fancy=True):
'''Create an timeseries plot of specific variables.
Parameters::
----------
save = boolean
True to save an output plot, False displays on screen.
fancy = boolean
If set to true uses ggplot style, otherwise default is used.
'''
if fancy:
matplotlib.style.use('ggplot')
# Set the time string format
fmt = '%H:%M:%S'
# Create figure/axes instance
fig, axs = plt.subplots(nrows=4, ncols=3, figsize=(8, 11), \
sharey="row")
for ax, fills in zip(axs.flat, self.dts):
# Calculate start and end times
stime = netCDF4.datetime(fills[0][0], fills[0][1], fills[0][2],\
fills[0][3], fills[0][4], fills[0][5])
etime = netCDF4.datetime(fills[1][0], fills[1][1], fills[1][2],\
fills[1][3], fills[1][4], fills[1][5])
# Make title from these times
figtitle = stime.strftime(fmt) + ' - ' + etime.strftime(fmt)
##if self.pcasp[
# Calculate Poisson error from Cai et al. AMT (2013), Equation A4
yerr = self.pcaspf.between_time(stime, etime).sum(axis=0).divide(
self.flow3s.between_time(stime,
etime).sum(axis=0)**2).pow(0.5).mul(
1. / self.dlog10d)
# Calculate the dNd(log10{D}) value for this time period,
# sum at each diameter (bin), and replace zero values with NaN
dndlogp = self.pcaspf.between_time(stime, etime).sum(axis=0).divide(\
(self.flow3s.between_time(stime, etime).sum(axis=0) * self.dlog10d))
# Create a pandas Series for plotting
dndlogpf = pd.Series(data=dndlogp.replace(to_replace=0., value=np.nan), \
index=self.pcasp_bin_mid, name=figtitle)
xlabels = self.pcasp_bin_mid.astype('|S5')
dndlogpf.plot(ax=ax,
logx=True,
logy=True,
yerr=yerr,
legend=False,
x=xlabels)
ax.set_title(figtitle)
ax.set_xlim(left=0.1, right=10.)
ax.set_ylim(bottom=0.01, top=10000.)
ax.xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter())
ax.yaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter())
ax.set_xlabel(r'D ($\mu$m)')
ax.set_ylabel(r"dN/dlog$_{10}$D (cm$^{-3}$)")
fig.tight_layout()
if save:
# Set the output file name
fname = "pcasp_timehist_" + os.path.splitext(
self.filename)[0] + ".png"
outfile = os.sep.join([save_dir, fname])
plt.savefig(outfile, format='png')
print "Creating image: " + outfile
else:
plt.show()
def date2dec(calendar='standard',
units=None,
excelerr=True,
yr=1,
mo=1,
dy=1,
hr=0,
mi=0,
sc=0,
ascii=None,
eng=None):
"""
Converts numpy arrays with calendar date into
numpy arrays with decimal date. Supported calendar
formats are standard, gregorian, julian, proleptic_gregorian,
excel1900, excel1904, 365_day, noleap, 366_day, all_leap,
360_day, decimal, or decimal360
Input is year, month day, hour, minute,
second or a combination of them. ASCII input
is possible, too.
Output is decimal date with day as unit.
Requires 'netcdftime.py' from the module
netcdftime available at:
http://netcdf4-python.googlecode.com
Definition
----------
def date2dec(calendar = 'standard', units=None,
excelerr = True, yr=1,
mo=1, dy=1, hr=0, mi=0, sc=0,
ascii=None, eng=None):
Input
-----
yr -> input array with year
mo -> input array with month
dy -> input array with day
hr -> input array with hour
mi -> input array with minute
sc -> input array with second
ascii -> input array with strings of the format
'dd.mm.yyyy hh:mm:ss'. If hour, minutes
and/or seconds are missing, they will be
set to 00.
If ascii input is chosen by user,
other inputs will be neglected.
ascii and eng are mutually exclusive.
eng -> input array with strings of the format
'yyyy-mm-dd hh:mm:ss'. If hour, minutes
and/or seconds are missing, they will be
set to 00.
If eng input is chosen, other inputs will
be neglected.
ascii and eng are mutually exclusive.
Parameters
----------
calendar -> Input date format. Default value is
'standard'.
'standard', 'gregorian'
= Input date is standard format.
Input is in julian calendar from
01.01.-4712 12:00:00 (BC) until
05.03.1583 00:00:00 and gregorian
calendar from 15.03.1583 00:00:00
until now. Missing 10 days don't
exsist.
'julian' = Input date is julian format.
Input is in julian calendar from
01.01.-4712 12:00:00 (BC) until now.
'proleptic_gregorian'
= Input date is gregorian format.
Input is in gregorian calendar from
01.01.0000 00:00:00 until now.
'excel1900' = Input date is excel 1900 format.
Input date is excel date with its
units at 01.01.1900 00:00:00 until
now.
'excel1904' = Input date is excel 1904 (lotus) format.
Input date is excel date with its
units at 01.01.1904 00:00:00 until now.
'365_day', 'noleap'
= Input date is 365 days format. Input date
consists of common years only (No leap years)
with its units at 01.01.0001 00:00:00 until now.
'366_day', 'all_leap'
= Input date is 366 days format. Input date
consists of leap years only (No common years)
with its units at 01.01.0001 00:00:00 until now.
'360_day' = Input date is 360 days format. Input
date consists of years with only 360 days
(30 days per month)with its units at
'decimal' = Output is decimal year.
'decimal360' = Output is decimal year with a year of 360 days, i.e. 12 month with 30 days each.
Optional Arguments
------------------
units -> Time units can be set by user. Input must be a
string in the format 'days since yyyy-mm-dd hh:mm:ss'.
Default values are set automatically.
excelerr -> In Excel the year 1900 is normally considered
as leap year, which is wrong. By default, this
error is taken into account (excelerr = True).
For excelerr = False, 1900 is considered as no
leap year.
Output
------
output -> Output numpy array with decimal date.
Restrictions
------------
Most versions of datetime do not support neagtive years,
i.e. Julian days < 1721423.5 = 01.01.0001 00:00.
There is an issue in netcdftime version < 0.9.5 in proleptic_gregorian for dates before year 301:
jams.dec2date(jams.date2dec(ascii='01.01.0300 00:00:00', calendar='proleptic_gregorian'),
calendar='proleptic_gregorian')
[300, 1, 2, 0, 0, 0]
jams.dec2date(jams.date2dec(ascii='01.01.0301 00:00:00', calendar='proleptic_gregorian'),
calendar='proleptic_gregorian')
[301, 1, 1, 0, 0, 0]
List input is only supported up to 2 dimensions.
Requires 'netcdftime.py' from module netcdftime available at:
http://netcdf4-python.googlecode.com
Examples
--------
#calendar = 'standard'
# Some implementations of datetime have problems with negative years
>>> import datetime
>>> if datetime.MINYEAR > 0:
... print('The minimum year in your datetime implementation is ', datetime.MINYEAR)
... print('i.e. it does not support negative years (BC).')
>>> if datetime.MINYEAR > 0:
... year = np.array([2000, 1810, 1630, 1510, 1271, 619, 2, 1])
... else:
... year = np.array([2000, 1810, 1630, 1510, 1271, 619, -1579, -4712])
>>> month = np.array([1, 4, 7, 9, 3, 8, 8, 1])
>>> day = np.array([5, 24, 15, 20, 18, 27, 23, 1])
>>> hour = np.array([12, 16, 10, 14, 19, 11, 20, 12])
>>> minute = np.array([30, 15, 20, 35, 41, 8, 3, 0])
>>> second = np.array([15, 10, 40, 50, 34, 37, 41, 0])
>>> decimal = date2dec(calendar = 'standard', yr=year, mo=month, dy=day, hr=hour, mi=minute, sc=second)
>>> from autostring import astr
>>> nn = year.size
>>> print(astr(decimal[:nn//2], 14, pp=True))
['2.45154902100694e+06' '2.38226217719907e+06' '2.31660093101852e+06' '2.27284810821759e+06']
>>> print(astr(decimal[nn//2:nn-2], 14,pp=True))
['2.18536732053241e+06' '1.94738596431713e+06']
>>> decimal = date2dec(calendar='standard', yr=year, mo=6, dy=15, hr=12, mi=minute, sc=second)
>>> print(astr(decimal[:nn//2],14,pp=True))
['2.45171102100694e+06' '2.38231401053241e+06' '2.31657101435185e+06' '2.27275102488426e+06']
>>> print(astr(decimal[nn//2:nn-2],14,pp=True))
['2.18545602886574e+06' '1.94731300598380e+06']
# ascii input
>>> if datetime.MINYEAR > 0:
... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50',
... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.0002 20:03:41', '01.01.0001 12:00:00'])
... else:
... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50',
... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.-1579 20:03:41', '01.01.-4712 12:00:00'])
>>> decimal = date2dec(calendar='standard', ascii=a)
>>> nn = a.size
>>> print(astr(decimal[:nn//2],14,pp=True))
['2.45154902100694e+06' '2.38226217719907e+06' '2.31660093101852e+06' '2.27284810821759e+06']
>>> print(astr(decimal[nn//2:nn-2],14,pp=True))
['2.18536732053241e+06' '1.94738596431713e+06']
# calendar = 'julian'
>>> decimal = date2dec(calendar='julian', ascii=a)
>>> print(astr(decimal[:nn//2],14,pp=True))
['2.45156202100694e+06' '2.38227417719907e+06' '2.31661093101852e+06' '2.27284810821759e+06']
>>> print(astr(decimal[nn//2:nn-2],14,pp=True))
['2.18536732053241e+06' '1.94738596431713e+06']
# calendar = 'proleptic_gregorian'
>>> decimal = date2dec(calendar='proleptic_gregorian', ascii=a)
>>> print(astr(decimal[:nn//2], 7, pp=True))
['730123.5210069' '660836.6771991' '595175.4310185' '551412.6082176']
>>> print(astr(decimal[nn//2:nn-2], 7, pp=True))
['463934.8205324' '225957.4643171']
# calendar = 'excel1900' WITH excelerr=True -> 1900 considered as leap year
>>> d = np.array(['05.01.2000 12:30:15', '27.05.1950 16:25:10', '13.08.1910 10:40:55',
... '01.03.1900 00:00:00', '29.02.1900 00:00:00', '28.02.1900 00:00:00',
... '01.01.1900 00:00:00'])
>>> decimal = date2dec(calendar='excel1900', ascii=d)
>>> nn = d.size
>>> print(astr(decimal[:nn//2], 7, pp=True))
['36530.5210069' '18410.6841435' ' 3878.4450810']
>>> print(astr(decimal[nn//2:],14,pp=True))
['61.00000000000000' '60.00000000000000' '59.00000000000000' ' 1.00000000000000']
# calendar = 'excel1900' WITH excelerr = False -> 1900 is NO leap year
>>> decimal = date2dec(calendar='excel1900', ascii=d, excelerr=False)
>>> print(astr(decimal[:nn//2], 7, pp=True))
['36529.5210069' '18409.6841435' ' 3877.4450810']
>>> print(astr(decimal[nn//2:],14,pp=True))
['60.00000000000000' '60.00000000000000' '59.00000000000000' ' 1.00000000000000']
# calendar = 'excel1904'
>>> decimal = date2dec(calendar='excel1904', ascii=d[:nn//2])
>>> print(astr(decimal[:nn//2], 7, pp=True))
['35069.5210069' '16949.6841435' ' 2417.4450810']
# calendar = '365_day'
>>> g = np.array(['18.08.1972 12:30:15', '25.10.0986 12:30:15', '28.11.0493 22:20:40', '01.01.0001 00:00:00'])
>>> decimal = date2dec(calendar='365_day', ascii=g)
>>> nn = g.size
>>> print(astr(decimal[:nn],14,pp=True))
['719644.52100694458932' '359822.52100694435649' '179911.93101851851679' ' 0.00000000000000']
# calendar = '366_day'
>>> decimal = date2dec(calendar='366_day', ascii=g)
>>> print(astr(decimal[:nn],14,pp=True))
['721616.52100694458932' '360808.52100694435649' '180404.93101851851679' ' 0.00000000000000']
# 360_day does not work with netcdftime.py version equal or below 0.9.2
# calendar = '360_day'
>>> decimal = date2dec(calendar='360_day', ascii=g)
>>> print(astr(decimal[:nn],14,pp=True))
['709787.52100694458932' '354894.52100694435649' '177447.93101851851679' ' 0.00000000000000']
>>> print(astr(date2dec(yr=1992, mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal'),14,pp=True))
1992.06853370763201
>>> print(astr(date2dec(ascii='26.01.1992 02:00', calendar='decimal360'),14,pp=True))
1992.06967593592572
>>> print(astr(date2dec(ascii=['26.01.1992 02:00','26.01.1992 02:00'], calendar='decimal360'),14,pp=True))
['1992.06967593592572' '1992.06967593592572']
>>> print(astr(date2dec(yr=[1992,1992], mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360'),14,pp=True))
['1992.06967593592572' '1992.06967593592572']
>>> print(astr(date2dec(yr=np.array([1992,1992]), mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360'),
... 14,pp=True))
['1992.06967593592572' '1992.06967593592572']
>>> print(astr(date2dec(ascii=[['26.01.1992 02:00','26.01.1992 02:00'],
... ['26.01.1992 02:00','26.01.1992 02:00'],
... ['26.01.1992 02:00','26.01.1992 02:00']], calendar='decimal360'),14,pp=True))
[['1992.06967593592572' '1992.06967593592572']
['1992.06967593592572' '1992.06967593592572']
['1992.06967593592572' '1992.06967593592572']]
>>> print((date2dec(ascii='01.03.2003 00:00:00') - date2dec(ascii='01.03.2003')) == 0.)
True
License
-------
This file is part of the JAMS Python package, distributed under the MIT
License. The JAMS Python package originates from the former UFZ Python library,
Department of Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany.
Copyright (c) 2010-2016 Arndt Piayda, Matthias Cuntz - mc (at) macu (dot) de
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
History
-------
Written AP, Jun 2010
Modified MC, Feb 2012 - All input can be scalar, list or array, also a mix
- Changed checks for easier extension
- decimal, decimal360
MC, Dec 2012 - change unit of proleptic_gregorian
from 'days since 0001-01-01 00:00:00'
to 'days since 0001-01-00 00:00:00'
MC, Feb 2013 - solved Excel leap year problem.
MC, Feb 2013 - ported to Python 3
MC, Jul 2013 - ascii/eng without time defaults to 00:00:00
MC, Oct 2013 - Excel starts at 1 not at 0
MC, Oct 2013 - units bugs, e.g. 01.01.0001 was substracted if Julian calendar even with units
MC, Nov 2013 - removed remnant of time treatment before time check in eng keyword
MC, Jun 2015 - adapted to new netCDF4/netcdftime (>= v1.0) and datetime (>= Python v2.7.9)
MC, Oct 2015 - call date2num with list instead of single netCDF4.datetime objects
MC, Oct 2016 - netcdftime provided even with netCDF4 > 1.0.0; make mo for months always integer
MC, Nov 2016 - 00, 01, etc. for integers not accepted by Python3
"""
#
# Checks
calendars = [
'standard', 'gregorian', 'julian', 'proleptic_gregorian', 'excel1900',
'excel1904', '365_day', 'noleap', '366_day', 'all_leap', '360_day',
'decimal', 'decimal360'
]
import netCDF4 as nt
try:
tst = nt.date2num
tst = nt.datetime
except:
try:
import netcdftime as nt
if ((nt.__version__ <= '0.9.2') & (calendar == '360_day')):
raise ValueError(
"date2dec error: Your version of netcdftime.py is equal"
" or below 0.9.2. The 360_day calendar does not work with"
" arrays here. Please download a newer one.")
except:
import cftime as nt
#
calendar = calendar.lower()
if (calendar not in calendars):
raise ValueError("date2dec error: Wrong calendar!"
" Choose: " + ''.join([i + ' ' for i in calendars]))
# if ascii input is given by user, other input will be neglected
# calculation of input size and shape
if (ascii is not None) and (eng is not None):
raise ValueError(
"date2dec error: 'ascii' and 'eng' mutually exclusive")
if ascii is not None:
islist = type(ascii) != type(np.array(ascii))
isarr = np.ndim(ascii)
if (islist & (isarr > 2)):
raise ValueError(
"date2dec error: ascii input is list > 2D; Use array input")
if isarr == 0: ascii = np.array([ascii])
else: ascii = np.array(ascii)
insize = ascii.size
outsize = insize
outshape = ascii.shape
asciifl = ascii.flatten()
timeobj = np.zeros(insize, dtype=object)
# slicing of ascii strings to implement in datetime object. missing times
# will be set to 0.
yr = np.zeros(insize, dtype=np.int)
mo = np.zeros(insize, dtype=np.int)
dy = np.zeros(insize, dtype=np.int)
hr = np.zeros(insize, dtype=np.int)
mi = np.zeros(insize, dtype=np.int)
sc = np.zeros(insize, dtype=np.int)
for i in range(insize):
aa = asciifl[i].split('.')
dy[i] = int(aa[0])
mo[i] = int(aa[1])
tail = aa[2].split()
yr[i] = int(tail[0])
if len(tail) > 1:
tim = tail[1].split(':')
hr[i] = int(tim[0])
if len(tim) > 1:
mi[i] = int(tim[1])
else:
mi[i] = 0
if len(tim) > 2:
sc[i] = int(tim[2])
else:
sc[i] = 0
else:
hr[i] = 0
mi[i] = 0
sc[i] = 0
if ((yr[i] == 1900) & (mo[i] == 2) & (dy[i] == 29)):
timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i])
else:
timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i],
sc[i])
if eng is not None:
islist = type(eng) != type(np.array(eng))
isarr = np.ndim(eng)
if isarr == 0:
eng = np.array([eng])
else:
eng = np.array(eng)
if (islist & (isarr > 2)):
raise ValueError(
"date2dec error: eng input is list > 2D; Use array input")
eng = np.array(eng)
insize = eng.size
outsize = insize
outshape = eng.shape
engfl = eng.flatten()
timeobj = np.zeros(insize, dtype=object)
# slicing of eng strings to implement in datetime object. missing times
# will be set to 0.
yr = np.zeros(insize, dtype=np.int)
mo = np.zeros(insize, dtype=np.int)
dy = np.zeros(insize, dtype=np.int)
hr = np.zeros(insize, dtype=np.int)
mi = np.zeros(insize, dtype=np.int)
sc = np.zeros(insize, dtype=np.int)
for i in range(insize):
ee = engfl[i].split('-')
yr[i] = int(ee[0])
mo[i] = int(ee[1])
tail = ee[2].split()
dy[i] = int(tail[0])
if len(tail) > 1:
tim = tail[1].split(':')
hr[i] = int(tim[0])
if len(tim) > 1:
mi[i] = int(tim[1])
else:
mi[i] = 0
if len(tim) > 2:
sc[i] = int(tim[2])
else:
sc[i] = 0
else:
hr[i] = 0
mi[i] = 0
sc[i] = 0
if ((yr[i] == 1900) & (mo[i] == 2) & (dy[i] == 29)):
timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i])
else:
timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i],
sc[i])
# if no ascii input, other inputs will be concidered
# calculation of input sizes, shapes and number of axis
if ((ascii is None) and (eng is None)):
isnlist1 = type(yr) == type(np.array(yr))
isarr1 = np.ndim(yr)
if isarr1 == 0: yr = np.array([yr])
else: yr = np.array(yr)
isnlist2 = type(mo) == type(np.array(mo))
isarr2 = np.ndim(mo)
if isarr2 == 0: mo = np.array([mo], dtype=np.int)
else: mo = np.array(mo, dtype=np.int)
isnlist3 = type(dy) == type(np.array(dy))
isarr3 = np.ndim(dy)
if isarr3 == 0: dy = np.array([dy])
else: dy = np.array(dy)
isnlist4 = type(hr) == type(np.array(hr))
isarr4 = np.ndim(hr)
if isarr4 == 0: hr = np.array([hr])
else: hr = np.array(hr)
isnlist5 = type(mi) == type(np.array(mi))
isarr5 = np.ndim(mi)
if isarr5 == 0: mi = np.array([mi])
else: mi = np.array(mi)
isnlist6 = type(sc) == type(np.array(sc))
isarr6 = np.ndim(sc)
if isarr6 == 0: sc = np.array([sc])
else: sc = np.array(sc)
islist = not (isnlist1 | isnlist2 | isnlist3 | isnlist4 | isnlist5
| isnlist6)
isarr = isarr1 + isarr2 + isarr3 + isarr4 + isarr5 + isarr6
shapes = [
np.shape(yr),
np.shape(mo),
np.shape(dy),
np.shape(hr),
np.shape(mi),
np.shape(sc)
]
nyr = np.size(yr)
nmo = np.size(mo)
ndy = np.size(dy)
nhr = np.size(hr)
nmi = np.size(mi)
nsc = np.size(sc)
sizes = [nyr, nmo, ndy, nhr, nmi, nsc]
nmax = np.amax(sizes)
ii = np.argmax(sizes)
outshape = shapes[ii]
if (islist & (np.size(outshape) > 2)):
raise ValueError(
"date2dec error: input is list > 2D; Use array input.")
if nyr < nmax:
if nyr == 1: yr = np.ones(outshape, ) * yr
else:
raise ValueError(
"date2dec error: size of yr != max input or 1.")
if nmo < nmax:
if nmo == 1: mo = np.ones(outshape, dtype=np.int) * mo
else:
raise ValueError(
"date2dec error: size of mo != max input or 1.")
if ndy < nmax:
if ndy == 1: dy = np.ones(outshape) * dy
else:
raise ValueError(
"date2dec error: size of dy != max input or 1.")
if nhr < nmax:
if nhr == 1: hr = np.ones(outshape) * hr
else:
raise ValueError(
"date2dec error: size of hr != max input or 1.")
if nmi < nmax:
if nmi == 1: mi = np.ones(outshape) * mi
else:
raise ValueError(
"date2dec error: size of mi != max input or 1.")
if nsc < nmax:
if nsc == 1: sc = np.ones(outshape) * sc
else:
raise ValueError(
"date2dec error: size of sc != max input or 1.")
indate = [yr, mo, dy, hr, mi, sc]
insize = [np.size(i) for i in indate]
inshape = [np.shape(i) for i in indate]
dimnum = [len(i) for i in inshape]
# calculation of maximum input size and maximum number of axis for
# reshaping the output
indate = [i.flatten() for i in indate]
outsize = max(insize)
timeobj = np.zeros(outsize, dtype=object)
# datetime object is constructed
for i in range(outsize):
iyr = int(indate[0][i])
imo = int(indate[1][i])
idy = int(indate[2][i])
ihr = int(indate[3][i])
imi = int(indate[4][i])
isc = int(indate[5][i])
if ((iyr == 1900) & (imo == 2) & (idy == 29)):
timeobj[i] = nt.datetime(iyr, 3, 1, ihr, imi, isc)
else:
timeobj[i] = nt.datetime(iyr, imo, idy, ihr, imi, isc)
# depending on chosen calendar and optional set of the time units
# decimal date is calculated
output = np.zeros(outsize)
t0 = nt.datetime(1582, 10, 5, 0, 0, 0)
t1 = nt.datetime(1582, 10, 15, 0, 0, 0)
is121 = True if (min(timeobj) < t0) and (max(timeobj) >= t1) else False
if (calendar == 'standard') or (calendar == 'gregorian'):
if units is None:
units = 'days since 0001-01-01 12:00:00'
dec0 = 1721424
else:
dec0 = 0
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt, units,
calendar='gregorian') + dec0
else:
output = nt.date2num(timeobj, units, calendar='gregorian') + dec0
elif calendar == 'julian':
if units is None:
units = 'days since 0001-01-01 12:00:00'
dec0 = 1721424
else:
dec0 = 0
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt, units, calendar='julian') + dec0
else:
output = nt.date2num(timeobj, units, calendar='julian') + dec0
elif calendar == 'proleptic_gregorian':
if units is None: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt,
units,
calendar='proleptic_gregorian')
else:
output = nt.date2num(timeobj,
units,
calendar='proleptic_gregorian')
elif calendar == 'excel1900':
doerr = False
if units is None:
units = 'days since 1899-12-31 00:00:00'
if excelerr: doerr = True
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt, units, calendar='gregorian')
else:
output = nt.date2num(timeobj, units, calendar='gregorian')
if doerr:
output = np.where(output >= 60., output + 1., output)
# date2num treats 29.02.1900 as 01.03.1990, i.e. is the same decimal number
if np.any((output >= 61.) & (output < 62.)):
for i in range(outsize):
# if (timeobj[i].year==1900) & (timeobj[i].month==2) & (timeobj[i].day==29):
# output[i] -= 1.
if (yr[i] == 1900) & (mo[i] == 2) & (dy[i] == 29):
output[i] -= 1.
elif calendar == 'excel1904':
if units is None: units = 'days since 1903-12-31 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt, units, calendar='gregorian')
else:
output = nt.date2num(timeobj, units, calendar='gregorian')
elif (calendar == '365_day') or (calendar == 'noleap'):
if units is None: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt, units, calendar='365_day')
else:
output = nt.date2num(timeobj, units, calendar='365_day')
elif (calendar == '366_day') or (calendar == 'all_leap'):
if units is None: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt, units, calendar='366_day')
else:
output = nt.date2num(timeobj, units, calendar='366_day')
elif calendar == '360_day':
if units is None: units = 'days since 0001-01-01 00:00:00'
if is121 and (nt.__version__ < '1.2.2'):
for ii, tt in enumerate(timeobj):
output[ii] = nt.date2num(tt, units, calendar='360_day')
else:
output = nt.date2num(timeobj, units, calendar='360_day')
elif calendar == 'decimal':
ntime = np.size(yr)
leap = np.array((((yr % 4) == 0) & ((yr % 100) != 0))
| ((yr % 400) == 0)).astype(np.int)
tdy = np.array(dy, dtype=np.float)
diy = np.array(
[[-9, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365],
[-9, 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366]])
for i in range(ntime):
tdy[i] = tdy[i] + np.array(diy[leap[i], mo[i]], dtype=np.float)
days_year = 365.
output = (np.array(yr, dtype=np.float) +
((tdy - 1.) * 24. + np.array(hr, dtype=np.float) +
np.array(mi, dtype=np.float) / 60. +
np.array(sc, dtype=np.float) / 3600.) /
((days_year + np.array(leap, dtype=np.float)) * 24.))
# for numerical stability, i.e. back and forth transforms
output += 1e-08 # 1/3 sec
elif calendar == 'decimal360':
ntime = np.size(yr)
tdy = np.array(dy, dtype=np.float)
diy = np.array(
[-9, 0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360])
for i in range(ntime):
tdy[i] = tdy[i] + np.array(diy[mo[i]], dtype=np.float)
days_year = 360.
output = (np.array(yr, dtype=np.float) +
((tdy - 1.) * 24. + np.array(hr, dtype=np.float) +
np.array(mi, dtype=np.float) / 60. +
np.array(sc, dtype=np.float) / 3600.) / (days_year * 24.))
# for numerical stability, i.e. back and forth transforms
output += 1e-08 # 1/3 sec
else:
raise ValueError(
"date2dec error: calendar not implemented; should have been catched before."
)
# return of reshaped output
output = np.reshape(output, outshape)
if isarr == 0:
output = np.float(output)
else:
if islist:
ns = np.size(outshape)
if ns == 1:
output = [i for i in output]
else:
loutput = [i for i in output[:, 0]]
for i in range(np.size(output[:, 0])):
loutput[i] = list(np.squeeze(output[i, :]))
output = loutput
return output
import calendar
import grd
import barotropic
import IOinitial
import IOsubset
import datetimeFunctions
try:
import ESMF
except ImportError:
print "Could not find module ESMF"
pass
__author__ = 'Trond Kristiansen'
__email__ = '*****@*****.**'
__created__ = datetime(2008, 8, 15)
__modified__ = datetime(2014, 12, 1)
__version__ = "1.5"
__status__ = "Development, modified on 15.08.2008,01.10.2009,07.01.2010, 15.07.2014, 01.12.2014, 07.08.2015"
def VerticalInterpolation(myvar, array1, array2, grdROMS, grdMODEL):
outINDEX_ST = (grdROMS.Nlevels, grdROMS.eta_rho, grdROMS.xi_rho)
outINDEX_U = (grdROMS.Nlevels, grdROMS.eta_u, grdROMS.xi_u)
outINDEX_UBAR = (grdROMS.eta_u, grdROMS.xi_u)
outINDEX_V = (grdROMS.Nlevels, grdROMS.eta_v, grdROMS.xi_v)
outINDEX_VBAR = (grdROMS.eta_v, grdROMS.xi_v)
if myvar in ['salinity', 'temperature']:
print 'Start vertical interpolation for %s (dimensions=%s x %s)' % (myvar, grdROMS.xi_rho, grdROMS.eta_rho)
outdata = np.empty((outINDEX_ST), dtype=np.float64, order='Fortran')
def getTime(dataPath, indatatype, grdROMS, grdMODEL, year, month, day, mytime, firstRun):
"""
Create a date object to keep track of Julian dates etc.
Also create a reference date starting at 1948/01/01.
Go here to check results:http://lena.gsfc.nasa.gov/lenaDEV/html/doy_conv.html
"""
if indatatype == 'SODA':
filename = getSODAfilename(year, month, day, None, dataPath)
if indatatype == 'SODAMONTHLY':
filename = getSODAMONTHLYfilename(year, month, day, None, dataPath)
if indatatype == 'GLORYS':
filename = getGLORYSfilename(year, month, day, "S", dataPath)
if indatatype == 'WOAMONTHLY':
filename = getWOAMONTHLYfilename(year, month, day, "temperature", dataPath)
if indatatype == 'NORESM':
filename = getNORESMfilename(year, month, day, "saln", dataPath)
if indatatype == 'NS8KM':
filename = getNS8KMfilename(year, month, day, "salt", dataPath)
if indatatype == 'NS8KMZ':
filename, readFromOneFile = getNS8KMZfilename(year, month, day, "salt", dataPath)
# Now open the input file and get the time
cdf = Dataset(filename)
if (indatatype)=='NORESM':
jdref = date2num(datetime(1800,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar)
elif indatatype=='NS8KMZ':
jdref = date2num(datetime(1948,1,1),units="days since 1948-01-01 00:00:00",calendar="gregorian")
elif (indatatype)=='GLORYS':
jdref = date2num(datetime(1948,1,1),cdf.variables["time_counter"].units,calendar=cdf.variables["time_counter"].calendar)
elif indatatype=='NS8KM':
jdref = date2num(datetime(1948,1,1),cdf.variables["ocean_time"].units,calendar=cdf.variables["ocean_time"].calendar)
else:
jdref = date2num(datetime(1948,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar)
if indatatype == 'SODA':
# Find the day and month that the SODA file respresents based on the year and ID number.
# Each SODA file represents a 5 day average, therefore we let the date we find be the first day
# of those 5 days. Thats the reason we subtract 4 below for day of month.
import date
days = 0.0; month = 1; loop = True
while loop is True:
d = date.NumberDaysMonth(month, year)
if days + d < int(ID) * 5:
days = days + d
month += 1
else:
day = int(int(ID) * 5 - days)
loop = False
mycalendar = cdf.variables["time"].calendar
myunits = cdf.variables["time"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate, myunits, calendar=mycalendar)
if indatatype == 'SODAMONTHLY':
# Find the day and month that the SODAMONTHLY file respresents based on the year and ID number.
# Each SODA file represents a 1 month average.
mycalendar = cdf.variables["time"].calendar
myunits = cdf.variables["time"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate, myunits, calendar=mycalendar)
if indatatype == 'GLORYS':
# Find the day and month that the GLORYS file respresents based on the year and ID number.
# Each file represents a 1 month average.
mycalendar = cdf.variables["time_counter"].calendar
myunits = cdf.variables["time_counter"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate, myunits, calendar=mycalendar)
if indatatype == 'NS8KM':
# Find the day and month that the GLORYS file respresents based on the year and ID number.
# Each file represents a 1 month average.
mycalendar = cdf.variables["ocean_time"].calendar
myunits = cdf.variables["ocean_time"].units
currentdate = datetime(year, month, day)
jd = date2num(currentdate, myunits, calendar=mycalendar)
if indatatype == 'NS8KMZ':
# Find the day and month that the GLORYS file respresents based on the year and ID number.
# Each file represents a 1 month average.
mycalendar = "gregorian"
refdate = datetime(1948, 1, 1)
currentdate = datetime(year, month, day)
myunits = cdf.variables["time"].units
jd = date2num(currentdate, myunits, calendar="gregorian")
print "Days:", jd, currentdate," year month day ",year, month, day
if indatatype == 'NORESM':
# Find the day and month that the NORESM file. We need to use the time modules from
# netcdf4 for python as they handle calendars that are no_leap.
# http://www.esrl.noaa.gov/psd/people/jeffrey.s.whitaker/python/netcdftime.html#datetime
mydays = cdf.variables["time"][0]
mycalendar = cdf.variables["time"].calendar
myunits = cdf.variables["time"].units
# Fake the start date first time around
# if (firstRun):
# currentdate = datetime(2006,1,1)
# print "NOTICE!\n First datestamp in result files are hardcoded to %s"%(currentdate)
# else:
currentdate = num2date(mydays, units=myunits, calendar=mycalendar)
jd = date2num(currentdate, myunits, calendar='noleap')
grdROMS.time = (jd - jdref)
grdROMS.reftime = jdref
grdROMS.timeunits=myunits
cdf.close()
print '\nCurrent time of %s file : %s' % (indatatype, currentdate)
def main():
start_year = 1979
end_year = 1981
HL_LABEL = "CRCM5_HL"
NEMO_LABEL = "CRCM5_NEMO"
file_prefix = "dm"
level = 1
level_type = level_kinds.HYBRID
wind_comp_names = ["UU", "VV"]
sim_label_to_path = OrderedDict(
[(HL_LABEL, "/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples"),
(NEMO_LABEL, "/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples")]
)
# get a coord file ...
coord_file = ""
found_coord_file = False
for mdir in os.listdir(sim_label_to_path[HL_LABEL]):
mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir)
if not os.path.isdir(mdir_path):
continue
for fn in os.listdir(mdir_path):
print(fn)
if fn[:2] not in ["pm", "dm", "pp", "dp"]:
continue
coord_file = os.path.join(mdir_path, fn)
found_coord_file = True
if found_coord_file:
break
bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file(path=coord_file)
xx, yy = bmp(lons, lats)
lons[lons > 180] -= 360
# loop through all files rotate vaectors and save to netcdf
for sim_label, samples_dir in sim_label_to_path.items():
samples = Path(samples_dir)
po = samples.parent
monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map(file_prefix=file_prefix,
start_year=start_year, end_year=end_year,
samples_dir_path=samples)
# Netcdf output file to put rotated winds
po /= "rotated_wind_{}.nc".format(sim_label)
with Dataset(str(po), "w") as ds:
ds.createDimension("time", None)
ds.createDimension("lon", lons.shape[0])
ds.createDimension("lat", lons.shape[1])
# create the schema of the output file
vname_to_ncvar = {}
for vname in wind_comp_names:
vname_to_ncvar[vname] = ds.createVariable(vname, "f4", dimensions=("time", "lon", "lat"))
vname_to_ncvar[vname].units = "knots"
lons_var = ds.createVariable("lon", "f4", dimensions=("lon", "lat"))
lats_var = ds.createVariable("lat", "f4", dimensions=("lon", "lat"))
time_var = ds.createVariable("time", "i8", dimensions=("time",))
time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format(datetime(start_year, 1, 1))
lons_var[:] = lons
lats_var[:] = lats
# use sorted dates
record_count = 0
for month_date in sorted(monthdate_to_path_list):
# select only dm files
mr = MultiRPN(path=monthdate_to_path_list[month_date])
vname_to_fields = {}
for vname in wind_comp_names:
vname_to_fields[vname] = mr.get_all_time_records_for_name_and_level(varname=vname, level=level, level_kind=level_type)
for ti, t in enumerate(sorted(vname_to_fields[wind_comp_names[0]])):
time_var[record_count] = date2num(t, time_var.units)
uu = vname_to_fields[wind_comp_names[0]][t]
vv = vname_to_fields[wind_comp_names[1]][t]
uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu, vv, lons, lats, bmp=bmp)
# in knots not in m/s
vname_to_ncvar[wind_comp_names[0]][record_count, :, :] = uu_rot
vname_to_ncvar[wind_comp_names[1]][record_count, :, :] = vv_rot
record_count += 1
import interpolation as interp
import IOwrite
import os
import barotropic
import IOinitial
import IOsubset
import datetimeFunctions
try:
import ESMF
except ImportError:
print("Could not find module ESMF")
pass
__author__ = 'Trond Kristiansen'
__email__ = '*****@*****.**'
__created__ = datetime(2008, 8, 15)
__modified__ = datetime(2014, 12, 1)
__version__ = "1.5"
__status__ = "Development, modified on 15.08.2008,01.10.2009,07.01.2010, 15.07.2014, 01.12.2014, 07.08.2015"
def verticalinterpolation(myvar, array1, array2, grdROMS, grdMODEL):
outINDEX_ST = (grdROMS.nlevels, grdROMS.eta_rho, grdROMS.xi_rho)
outINDEX_U = (grdROMS.nlevels, grdROMS.eta_u, grdROMS.xi_u)
outINDEX_UBAR = (grdROMS.eta_u, grdROMS.xi_u)
outINDEX_V = (grdROMS.nlevels, grdROMS.eta_v, grdROMS.xi_v)
outINDEX_VBAR = (grdROMS.eta_v, grdROMS.xi_v)
if myvar in ['salinity', 'temperature']:
print('Start vertical interpolation for %s (dimensions=%s x %s)' %
(myvar, grdROMS.xi_rho, grdROMS.eta_rho))
def main():
start_year = 1979
end_year = 1981
HL_LABEL = "CRCM5_HL"
NEMO_LABEL = "CRCM5_NEMO"
file_prefix = "dm"
level = 1
level_type = level_kinds.HYBRID
wind_comp_names = ["UU", "VV"]
sim_label_to_path = OrderedDict([
(HL_LABEL,
"/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples"
),
(NEMO_LABEL,
"/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples"
)
])
# get a coord file ...
coord_file = ""
found_coord_file = False
for mdir in os.listdir(sim_label_to_path[HL_LABEL]):
mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir)
if not os.path.isdir(mdir_path):
continue
for fn in os.listdir(mdir_path):
print(fn)
if fn[:2] not in ["pm", "dm", "pp", "dp"]:
continue
coord_file = os.path.join(mdir_path, fn)
found_coord_file = True
if found_coord_file:
break
bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file(
path=coord_file)
xx, yy = bmp(lons, lats)
lons[lons > 180] -= 360
# loop through all files rotate vaectors and save to netcdf
for sim_label, samples_dir in sim_label_to_path.items():
samples = Path(samples_dir)
po = samples.parent
monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map(
file_prefix=file_prefix,
start_year=start_year,
end_year=end_year,
samples_dir_path=samples)
# Netcdf output file to put rotated winds
po /= "rotated_wind_{}.nc".format(sim_label)
with Dataset(str(po), "w") as ds:
ds.createDimension("time", None)
ds.createDimension("lon", lons.shape[0])
ds.createDimension("lat", lons.shape[1])
# create the schema of the output file
vname_to_ncvar = {}
for vname in wind_comp_names:
vname_to_ncvar[vname] = ds.createVariable(vname,
"f4",
dimensions=("time",
"lon",
"lat"))
vname_to_ncvar[vname].units = "knots"
lons_var = ds.createVariable("lon",
"f4",
dimensions=("lon", "lat"))
lats_var = ds.createVariable("lat",
"f4",
dimensions=("lon", "lat"))
time_var = ds.createVariable("time", "i8", dimensions=("time", ))
time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format(
datetime(start_year, 1, 1))
lons_var[:] = lons
lats_var[:] = lats
# use sorted dates
record_count = 0
for month_date in sorted(monthdate_to_path_list):
# select only dm files
mr = MultiRPN(path=monthdate_to_path_list[month_date])
vname_to_fields = {}
for vname in wind_comp_names:
vname_to_fields[
vname] = mr.get_all_time_records_for_name_and_level(
varname=vname, level=level, level_kind=level_type)
for ti, t in enumerate(
sorted(vname_to_fields[wind_comp_names[0]])):
time_var[record_count] = date2num(t, time_var.units)
uu = vname_to_fields[wind_comp_names[0]][t]
vv = vname_to_fields[wind_comp_names[1]][t]
uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu,
vv,
lons,
lats,
bmp=bmp)
# in knots not in m/s
vname_to_ncvar[wind_comp_names[0]][
record_count, :, :] = uu_rot
vname_to_ncvar[wind_comp_names[1]][
record_count, :, :] = vv_rot
record_count += 1
def get3ddata(confM2R, myvar, year, month, day):
if myvar == 'temperature':
varN = 0
if myvar == 'salinity':
varN = 1
if myvar == 'uvel':
varN = 3
if myvar == 'vvel':
varN = 4
# The variable splitExtract is defined in IOsubset.py and depends on the orientation
# and indatatype of grid (-180-180 or 0-360). Assumes regular grid.
if confM2R.useesmf:
if confM2R.indatatype == "SODA":
filename = getSODAfilename(confM2R, year, month, None)
cdf = Dataset(filename)
data = cdf.variables[confM2R.inputdatavarnames[varN]][0, :, :, :]
if confM2R.indatatype == "SODA3":
filename = getSODA3filename(confM2R, year, month,
confM2R.inputdatavarnames[varN])
cdf = Dataset(filename)
print("=>Extracting data for month %s from SODA3 %s " %
(month - 1, filename))
data = cdf.variables[confM2R.inputdatavarnames[varN]][month -
1, :, :, :]
if confM2R.indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(confM2R, year, month,
confM2R.inputdatavarnames[varN])
cdf = Dataset(filename)
data = cdf.variables[str(confM2R.inputdatavarnames[varN])][:, :, :]
if confM2R.indatatype == "WOAMONTHLY":
filename = getWOAMONTHLYfilename(confM2R, year, month,
confM2R.inputdatavarnames[varN])
cdf = Dataset(filename)
data = cdf.variables[str(
confM2R.inputdatavarnames[varN])][month - 1, :, :, :]
if confM2R.indatatype == "NORESM":
cdf = Dataset(
getNORESMfilename(confM2R, year, month,
confM2R.inputdatavarnames[varN]))
myunits = cdf.variables[str(confM2R.inputdatavarnames[varN])].units
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][0, :, :, :])
data = np.where(data.mask, confM2R.grdROMS.fillval, data)
print("Data range", np.min(data), np.max(data))
if confM2R.indatatype == "NS8KMZ":
filename, readFromOneFile = getNS8KMZfilename(
confM2R, year, month, confM2R.inputdatavarnames[varN])
cdf = Dataset(filename)
print("Reading from one file %s" % (readFromOneFile))
myunits = cdf.variables[str(confM2R.inputdatavarnames[varN])].units
if (readFromOneFile):
currentdate = datetime(year, month, day, 12)
jd = date2num(currentdate,
cdf.variables["time"].units,
calendar="gregorian")
print("Days:", jd, currentdate)
timesteps = (cdf.variables["time"][:]).tolist()
timeindex = timesteps.index(jd)
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][timeindex, :, :, :])
else:
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][0, :, :, :])
print("Range of data", varN, np.min(data), np.max(data))
data = np.where(data.mask, confM2R.grdROMS.fillval, data)
if confM2R.indatatype == "GLORYS":
cdf = Dataset(
getGLORYSfilename(confM2R, year, month,
confM2R.inputdatavarnames[varN]))
myunits = cdf.variables[str(confM2R.inputdatavarnames[varN])].units
data = np.squeeze(cdf.variables[str(
confM2R.inputdatavarnames[varN])][0, :, :, :])
data = np.where(data.mask, confM2R.grdROMS.fillval, data)
cdf.close()
if myvar == 'temperature' and confM2R.indatatype in [
"NS8KMZ", "GLORYS", "NORESM"
]:
if myunits == "degree_Kelvin" or myunits == "K":
if confM2R.indatatype in ["GLORYS"]:
data = np.where(data <= -32.767, confM2R.grdROMS.fillval, data)
data = data - 273.15
if confM2R.indatatype == "GLORYS":
data = np.where(data <= -32.767, confM2R.grdROMS.fillval, data)
data = np.ma.masked_where(data <= confM2R.grdROMS.fillval, data)
if __debug__:
print(
"Data range of %s just after extracting from netcdf file: %s - %s"
% (str(confM2R.inputdatavarnames[varN]), data.min(), data.max()))
return data
def main(in_dir="/RESCUE/skynet3_rech1/huziy/anusplin_links",
out_dir="/HOME/huziy/skynet3_rech1/hail/anusplin_ts"):
out_dir_p = Path(out_dir)
in_dir_p = Path(in_dir)
lon0 = -114.0708
lat0 = 51.0486
vname = "daily_precipitation_accumulation"
vname_alternatives = ["daily_accumulation_precipitation"]
vname_alternatives.append(vname)
var_list = [vname]
fname_hint = "pcp"
spatial_ind = None
varname_to_list_of_frames = {vname: [] for vname in var_list}
for fin in in_dir_p.iterdir():
if fin.name.lower().endswith("ret"):
continue
if fin.name.lower().endswith("verif"):
continue
if fname_hint not in fin.name.lower():
continue
if not fin.name.endswith(".nc"):
continue
print(fin)
year, month = get_ym_from_path(fin)
with Dataset(str(fin)) as ds:
if spatial_ind is None:
lons, lats = ds.variables["lon"][:], ds.variables["lat"][:]
x, y, z = lat_lon.lon_lat_to_cartesian(lons.flatten(),
lats.flatten())
ktree = KDTree(list(zip(x, y, z)))
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon0, lat0)
dist, spatial_ind = ktree.query((x0, y0, z0))
for vname_alt in vname_alternatives:
try:
values = ds[vname_alt][:]
values = [field.flatten()[spatial_ind] for field in values]
break
except IndexError as ierr:
pass
dates = [datetime(year, month, int(d)) for d in ds["time"][:]]
varname_to_list_of_frames[vname].append(
pd.DataFrame(index=dates, data=values))
for vname in var_list:
df = pd.concat(varname_to_list_of_frames[vname])
assert isinstance(df, pd.DataFrame)
df.sort_index(inplace=True)
df.to_csv(str(out_dir_p.joinpath("{}.csv".format(vname))),
float_format="%.3f",
index_label="Time")
def __init__(self):
print('\n--------------------------\n')
print('Started ' + time.ctime(time.time()))
# EDIT ===================================================================
# Set show_progress to "False" if you do not want to see the progress
# indicator for horizontal interpolation.
self.showprogress = True
# Set compileAll to True if you want automatic re-compilation of all the
# fortran files necessary to run model2roms. Options are "gfortran" or "ifort". Edit
# compile.py to add other Fortran compilers.
self.compileall = False
# Extract time-series of data for given longitude/latitude
self.extractstations = False
# Define a set of longitude/latitude positions with names to extract into
# station files (using extractStations)
if self.extractstations:
# stationNames = ['NorthSea', 'Iceland', 'EastandWestGreenland', 'Lofoten', 'Georges Bank']
# lonlist = [2.4301, -22.6001, -47.0801, 13.3801, -67.2001]
# latlist = [54.5601, 63.7010, 60.4201, 67.5001, 41.6423]
self.stationnames = ["Ytre Utsira", "Indre Utsira", "Lista"]
self.latlist = [59.316667, 59.316667, 58.016667]
self.lonliost = [4.800000, 4.983333, 6.533333]
# Create the bry, init, and clim files for a given grid and input data
self.createoceanforcing = True
# Create atmospheric forcing for the given grid
self.createatmosforcing = False # currently in beta stages and unavailable
# Create a smaller resolution grid based on your original. Decimates every second for
# each time run
self.decimategridfile = False
# Write ice values to file (for Arctic regions)
self.writeice = True
# Use ESMF for the interpolation. This requires that you have ESMF and ESMPy installed (import ESMF)
self.useesmf = True
# Apply filter to smooth the 2D fields after interpolation (time consuming but enhances results)
self.usefilter = True
# Format to write the ouput to: 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', or 'NETCDF3_CLASSIC'
# Using NETCDF4 automatically turns on compression of files (ZLIB)
self.myformat = 'NETCDF4'
self.myzlib = True
# Frequency of the input data: usually monthly
self.timefrequencyofinputdata = "month" # , "month", "hour"
# IN GRIDTYPES ------------------------------------------------------------------------------
# Define what grid type you wnat to interpolate from (input MODEL data)
# Options:
# 1. SODA, 2. SODAMONTHLY, 3.WOAMONTHLY, 4. NORESM, 4. GLORYS, 5. SODA3
self.indatatype = 'SODA3'
# Define what grid type you wnat to interpolate from: Can be Z for SIGMA for ROMS
# vertical coordinate system or ZLEVEL. also define the name of the dimensions in the input files.
# Options:
# 1. SIGMA (not prpoerly implemented yet), 2. ZLEVEL
self.ingridtype = "SIGMA"
# Define the names of ythe geographical variables in the input files
self.grdType = 'regular'
self.lonName = "longitude"
self.latName = "latitude"
self.depthName = "depth"
self.fill_value = -1.e+20
# OUT GRIDTYPES ------------------------------------------------------------------------------
# Define what grid type you wnat to interpolate to
# Options: This is just the name of your grid used to identify your selection later
self.outgrid = "ROHO800"
self.outgridtype = "ROMS"
# Subset input data. If you have global data you may want to seubset these to speed up reading. Make
# sure that your input data are cartesian (0-360 or -180:180, -90:90)
self.subsetindata = False
if self.subsetindata:
self.subset = self.definesubsetforindata()
# Define nmber of output depth levels
self.nlevels = 40
# Define the grid stretching properties (leave default if uncertain what to pick)
self.vstretching = 4
self.vtransform = 2
self.theta_s = 7.0
self.theta_b = 0.1
self.tcline = 250.0
self.hc = 250
# PATH TO FORCINGDATA --------------------------------------------------------------------
# Define the path to the input data
self.modelpath = self.defineforcingdatapath()
print("model path ", self.modelpath)
# PATH TO GRID -----------------------------------------------------------------------------
# Define the path to the grid file
self.romsgridpath = self.defineromsgridpath()
# Climatology is only monthly and model2roms needs to know this
self.isclimatology = True if self.indatatype == 'WOAMONTHLY' else False
# DATE AND TIME DETAILS ---------------------------------------------------------
# Define the period to create forcing for
self.start_year = 2013
self.end_year = 2013
self.start_month = 1
self.end_month = 12
self.start_day = 1
self.end_day = 1
if int(calendar.monthrange(self.start_year,
self.start_month)[1]) < self.start_day:
self.start_day = int(
calendar.monthrange(self.start_year, self.start_month)[1])
if int(calendar.monthrange(self.end_year,
self.end_month)[1]) < self.end_day:
self.end_day = int(
calendar.monthrange(self.end_year, self.end_month)[1])
print "hello, startdate here"
self.startdate = datetime(self.start_year, self.start_month,
self.start_day)
self.enddate = datetime(self.end_year, self.end_month, self.end_day)
self.years = [
self.start_year + year
for year in range(self.end_year + 1 - self.start_year)
]
# DEFINE VARIABLE NAMES ---------------------------------------------------------
# Define what and name of variables to include in the forcing files
# -> myvars is the name model2roms uses to identify variables
# -> varNames is the name of the variable found in the NetCDF input files
self.globalvarnames = self.defineglobalvarnames()
self.inputdatavarnames = self.defineinputdatavarnames()
# NO EDIT BELOW ====================================================================================================
if self.compileall is True:
import compile
compile.compileallgfortran()
if self.createatmosforcing or self.createoceanforcing:
self.abbreviation = self.defineabbreviation()
self.climname, self.initname, self.bryname = self.defineoutputfilenames(
)
if self.isclimatology is True:
self.climname = self.abbreviation + '_' + str(
self.indatatype) + '_climatology.nc'
self.showinfo()
if self.useesmf:
import ESMF
print("Starting logfile for ESMF")
manager = ESMF.Manager(debug=True)
# Create the grid object for the output grid
self.grdROMS = grd.Grd("ROMS", self.outgridtype, self.useesmf,
'ocean', self.outgrid)
self.grdROMS.Nlevels = self.nlevels
self.grdROMS.vstretching = self.vstretching
self.grdROMS.vtransform = self.vtransform
self.grdROMS.theta_s = self.theta_s
self.grdROMS.theta_b = self.theta_b
self.grdROMS.Tcline = self.tcline
self.grdROMS.hc = self.hc
self.grdROMS.vars = self.inputdatavarnames
self.grdROMS.varNames = self.globalvarnames
self.grdROMS.lonName = 'lon_rho'
self.grdROMS.latName = 'lat_rho'
self.grdROMS.openNetCDF(self.romsgridpath)
self.grdROMS.createObject()
self.grdROMS.getDims()
# Create the grid object for the input grid
self.grdMODEL = grd.Grd("FORCINGDATA", self.outgridtype,
self.useesmf, 'ocean', self.outgrid)
self.grdMODEL.grdType = self.grdType
self.grdMODEL.lonName = self.lonName
self.grdMODEL.latName = self.latName
self.grdMODEL.depthName = self.depthName
self.grdMODEL.fill_value = self.fill_value
#if self.createoceanforcing:
# model2roms.convertMODEL2ROMS()
# clim2bry.writeBry(grdROMS, start_year, bryName, climname, writeice, indatatype, myformat)
#if self.createAtmosForcing:
# atmosForcing.createAtmosFileUV(grdROMS, modelpath, atmospath, startdate, enddate, useESMF,
# myformat, abbreviation, indatatype, gridtype, show_progress)
if self.decimategridfile:
decimateGrid.createGrid(
grdROMS,
"/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015.nc",
"/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015v2.nc",
2)
if self.extractstations:
print(
"Running in station mode and extracting pre-defined station locations"
)
IOstation.getStationData(years, IDS, modelpath, latlist, lonlist,
stationNames)
print('Finished ' + time.ctime(time.time()))
def selseasyear(year, seas, t_coord, t_units, t_cal):
if seas == ['ANN']:
ti = netcdf.date2num(netcdf.datetime(year, 1, 1), t_units, t_cal)
if t_cal == '360_day':
tf = netcdf.date2num(netcdf.datetime(year, 12, 30), t_units, t_cal)
else:
tf = netcdf.date2num(netcdf.datetime(year, 12, 31), t_units, t_cal)
if seas == ['DJF']:
ti = netcdf.date2num(netcdf.datetime(year - 1, 12, 1), t_units, t_cal)
if t_cal == '360_day':
tf = netcdf.date2num(netcdf.datetime(year, 2, 30), t_units, t_cal)
else:
tf = netcdf.date2num(netcdf.datetime(year, 2, 28), t_units, t_cal)
if seas == ['MAM']:
ti = netcdf.date2num(netcdf.datetime(year, 3, 1), t_units, t_cal)
if t_cal == '360_day':
tf = netcdf.date2num(netcdf.datetime(year, 5, 30), t_units, t_cal)
else:
tf = netcdf.date2num(netcdf.datetime(year, 5, 31), t_units, t_cal)
if seas == ['JJA']:
ti = netcdf.date2num(netcdf.datetime(year, 6, 1), t_units, t_cal)
if t_cal == '360_day':
tf = netcdf.date2num(netcdf.datetime(year, 8, 30), t_units, t_cal)
else:
tf = netcdf.date2num(netcdf.datetime(year, 8, 31), t_units, t_cal)
if seas == ['SON']:
ti = netcdf.date2num(netcdf.datetime(year, 9, 1), t_units, t_cal)
tf = netcdf.date2num(netcdf.datetime(year, 11, 30), t_units, t_cal)
ind_t = np.where(np.bitwise_and(t_coord >= ti, t_coord <= tf + 1))
return ind_t
import time, calendar
from netCDF4 import Dataset, datetime, date2num, num2date
import model2roms
import IOstation
import clim2bry
import decimateGrid
import grd
import numpy as np
import atmosForcing
__author__ = 'Trond Kristiansen'
__email__ = '*****@*****.**'
__created__ = datetime(2009, 1, 30)
__modified__ = datetime(2016, 4, 21)
__version__ = "1.5"
__status__ = "Development"
def myhelp():
"""
This program is run by typing: python main.py in the command window.
"""
def defineSubsetForIndata():
# Subset the input data. The more you subset the less memory is needed for calculations
# and the faster the process is performed. The subset is initially performed in IOsubset.py
if gridtype == "NS8KM":
abbreviation = "nordsjoen_8km"
minLat = 40
maxLat = 70
'/dmf2/scenario/external_data/cccma/CanESM2_large_ensemble/historical-'
+ f + '/day/seaIce/sic/' + m + 'i1p1/sic_day_CanESM2_historical-' +
f + '_' + m + 'i1p1_' + yi2 + '0101-' + yf2 + '1231.nc', 'r')
# concatenating on time axis, selecting data for 20-year periods and plotting maps
time1 = nc1.variables['time']
time2 = nc2.variables['time']
lat = nc1.variables['lat'][32:]
lon = nc1.variables['lon'][:]
sic1 = nc1.variables['sic'][:, 32:, :]
sic2 = nc2.variables['sic'][:, 32:, :]
fig1 = plt.figure(figsize=(7.2, 5))
for i in range(len(yi)):
ti = netcdf.date2num(netcdf.datetime(yi[i], 1, 1), time1.units,
time1.calendar)
tf = netcdf.date2num(netcdf.datetime(yf[i], 1, 1), time1.units,
time1.calendar)
if i <= 1:
ind_t = np.where(
np.bitwise_and(time1[:] >= ti, time1[:] <= tf + 1))
sie_tmp = sic1[ind_t[0], :, :]
sie = np.mean(sie_tmp, 0)
else:
ind_t = np.where(
np.bitwise_and(time2[:] >= ti, time2[:] <= tf + 1))
sie_tmp = sic2[ind_t[0], :, :]
sie = np.mean(sie_tmp, 0)
im = fct_pt.map_contourfAR(fig1, subplt[i], sie, lon, lat,
from netCDF4 import Dataset, datetime, date2num,num2date
import os
import calendar
__author__ = 'Trond Kristiansen'
__email__ = '*****@*****.**'
__created__ = datetime(2015, 8, 11)
__modified__ = datetime(2015, 8, 11)
__version__ = "1.5"
__status__ = "Development, modified on 11.08.2015"
# Methods for returning list of months and days for the given time-step.
def createListOfMonths(currentyear,startdate,enddate,isClimatology):
print currentyear, startdate.year, enddate.year
if currentyear == startdate.year:
IDS=[startdate.month+m for m in xrange(13-startdate.month)]
# months from start month to end of first year
elif currentyear == enddate.year:
IDS=[1+m for m in xrange(int(enddate.month))]
# months from first month to last month of last year
elif startdate.year < currentyear < enddate.year:
# months from first month to last month of last year
IDS = [1+m for m in xrange(12)]
if startdate.year == enddate.year:
# months from first month to last month of last year
IDS = [startdate.month + m for m in xrange(enddate.month - startdate.month)]
def get_fields(path, var_key, wildcards='*', num_members=4, chunks=60):
fields_dict = {}
# create coordinates
domain = path.split('/')[-1]
coordn = loadmat(os.path.join(path, 'coords_' + domain + '.mat'))
grid_shape = coordn['lon'].shape
fields_dict['lon'] = coordn['lon']
fields_dict['lat'] = coordn['lat']
files = sorted(glob(os.path.join(path, wildcards)))
# Find unique dates for members ...
print('Creating model data: ', len(files), ' files found')
members = {}
for file in files:
data = loadmat(file)
for time, var in zip(data['dates'], data[var_key]):
if not members.has_key(time):
# create empty lists for unseen data..
members[time] = []
# append member to existing ones
members[time].append(var)
# create output arrays:
valid = 0
times = []
for time, values in members.items():
# excluding last value (is an initialization)
if len(values) > num_members:
check_values = not np.isnan(values).any() and np.ptp(values) < 1e3
if check_values:
members[time] = np.array(values[-num_members - 1:-1])
print('processed: ', time, 'shape: ', members[time].shape)
# create times:
year, month, days_hours = time.split('-')
day, hour = days_hours.split('_')
time_args = (int(year), int(month), int(day),
int(hour.split('.')[0]))
times.append(datetime(*time_args))
else:
print('discarded: ', time, 'var contains nans: ',
np.max(members.pop(time)))
else:
print('discarded: ', time, 'shape: ', np.shape(members.pop(time)))
# store dates
dates = date2num(times, time_units)
# index for ordered dates
ind = np.argsort(dates)
# stack arrays and send members to last dimension (as channels)
dates = dates[ind]
values = np.stack(members.values()).transpose([0, 2, 3, 1])[ind]
if var_key is 'rain':
# create actual 3h acumm variables
time_s = dates[0]
value_s = values[0]
fields_dict['time'] = []
fields_dict['values'] = []
for time, value in zip(dates[1:], values[1:]):
if time - time_s == 10800: # 3 h
print('calculate acummulated rain from: ', time_s, 'to: ',
time)
fields_dict['time'].append(time)
fields_dict['values'].append((value - value_s).clip(min=0.0))
time_s = time
value_s = value
fields_dict['time'] = np.array(fields_dict['time'])
fields_dict['values'] = np.stack(fields_dict['values'])
else:
fields_dict['time'] = dates
fields_dict['values'] = values
return fields_dict
import time, calendar
from netCDF4 import Dataset, datetime, date2num,num2date
import model2roms
import IOstation
import clim2bry
import decimateGrid
import grd
import numpy as np
import atmosForcing
__author__ = 'Trond Kristiansen'
__email__ = '*****@*****.**'
__created__ = datetime(2009, 1, 30)
__modified__ = datetime(2015, 8, 7)
__version__ = "1.5"
__status__ = "Development"
def myhelp():
"""
This program is run by typing: python main.py in the command window.
"""
def defineSubsetForIndata():
# Subset the input data. The more you subset the less memory is needed for calculations
# and the faster the process is performed. The subset is initially performed in IOsubset.py
if gridtype == "NS8KM":
abbreviation = "nordsjoen_8km"
minLat = 40
maxLat = 70
minLon = -30
def get3Ddata(grdROMS, grdMODEL, myvar, indatatype, year, month, day, varNames, dataPath):
if myvar == 'temperature': varN = 0;
if myvar == 'salinity': varN = 1;
if myvar == 'uvel': varN = 3;
if myvar == 'vvel': varN = 4;
# The variable splitExtract is defined in IOsubset.py and depends on the orientation
# and indatatype of grid (-180-180 or 0-360). Assumes regular grid.
if grdMODEL.useESMF:
if indatatype == "SODA":
filename = getSODAfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[varNames[varN]][0,:,:,:]
if indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][:,:,:]
if indatatype == "WOAMONTHLY":
filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][month - 1, :,:,:]
if indatatype == "NORESM":
cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath))
myunits = cdf.variables[str(varNames[varN])].units
data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:,:])
data=np.where(data.mask,grdROMS.fill_value,data)
# data = np.where(abs(data)>=32768 , grdROMS.fill_value, data)
print "Data range", np.min(data),np.max(data)
if indatatype == "NS8KMZ":
filename, readFromOneFile = getNS8KMZfilename(year, month, day, varNames[varN], dataPath)
cdf = Dataset(filename)
print "Reading from one file %s"%(readFromOneFile)
myunits = cdf.variables[str(varNames[varN])].units
if (readFromOneFile):
jdref = date2num(datetime(1948,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar)
currentdate = datetime(year, month, day, 12)
jd = date2num(currentdate, cdf.variables["time"].units, calendar="gregorian")
print "Days:", jd, currentdate
timesteps = (cdf.variables["time"][:]).tolist()
timeindex = timesteps.index(jd)
data = np.squeeze(cdf.variables[str(varNames[varN])][timeindex,:,:,:])
else:
data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:,:])
print "Range of data",varN, np.min(data),np.max(data)
data=np.where(data.mask,grdROMS.fill_value,data)
if indatatype == "GLORYS":
cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath))
myunits = cdf.variables[str(varNames[varN])].units
data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:,:])
data=np.where(data.mask,grdROMS.fill_value,data)
cdf.close()
else:
if grdMODEL.splitExtract is True:
if indatatype == "SODA":
filename = getSODAfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data1 = cdf.variables[varNames[varN]][0, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
data2 = cdf.variables[varNames[varN]][0, :,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]
if indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data1 = cdf.variables[str(varNames[varN])][:,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
data2 = cdf.variables[str(varNames[varN])][:,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]
if indatatype in ["WOAMONTHLY"]:
filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath)
cdf = Dataset(filename)
data1 = cdf.variables[str(varNames[varN])][month - 1, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
data2 = cdf.variables[str(varNames[varN])][month - 1, :,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]
if indatatype == "GLORYS":
cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath))
myunits = cdf.variables[str(varNames[varN])].units
data1 = np.squeeze(cdf.variables[str(varNames[varN])][0, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
data2 = np.squeeze(cdf.variables[str(varNames[varN])][0, :,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])])
if indatatype == "NORESM":
cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath))
myunits = cdf.variables[str(varNames[varN])].units
data1 = np.squeeze(cdf.variables[str(varNames[varN])][0, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
data2 = np.squeeze(cdf.variables[str(varNames[varN])][0, :,
int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]),
int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])])
cdf.close()
data = np.concatenate((data1, data2), axis=2)
else:
if indatatype == "SODA":
filename = getSODAfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][0, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
if indatatype == "SODAMONTHLY":
filename = getSODAMONTHLYfilename(year, month, day, None, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][:,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
if indatatype == "WOAMONTHLY":
filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath)
cdf = Dataset(filename)
data = cdf.variables[str(varNames[varN])][month, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]
if indatatype == "GLORYS":
if myvar == 'temperature':
cdf = Dataset(getGLORYSfilename(year, month, day, 'T', dataPath))
myunits = cdf.variables[str(varNames[varN])].units
if myvar == 'salinity': cdf = Dataset(getGLORYSfilename(year, month, day, 'S', dataPath))
if myvar == 'uvel': cdf = Dataset(getGLORYSfilename(year, month, day, 'U', dataPath))
if myvar == 'vvel': cdf = Dataset(getGLORYSfilename(year, month, day, 'V', dataPath))
data = np.squeeze(cdf.variables[str(varNames[varN])][0, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
if indatatype == "NORESM":
cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath))
myunits = cdf.variables[str(varNames[varN])].units
data = np.squeeze(cdf.variables[str(varNames[varN])][0, :,
int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]),
int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])])
cdf.close()
if myvar == 'temperature' and indatatype in ["NS8KMZ", "GLORYS", "NORESM"]:
if myunits == "degree_Kelvin" or myunits == "K":
if indatatype in ["GLORYS"]:
data = np.where(data <= -32.767, grdROMS.fill_value, data)
data = data - 273.15
# if time == 0 and myvar == myvars[0]:
# tmp = np.squeeze(data[0, :, :])
#grdMODEL.mask = np.zeros(grdMODEL.lon.shape, dtype=np.float64)
#grdMODEL.mask[:, :] = np.where(tmp == grdROMS.fill_value, 1, 0)
if indatatype == "GLORYS":
data = np.where(data <= -32.767, grdROMS.fill_value, data)
data = np.ma.masked_where(data <= grdROMS.fill_value, data)
if __debug__:
print "Data range of %s just after extracting from netcdf file: %s - %s" % (str(varNames[varN]),
data.min(), data.max())
return data
import time, calendar
from netCDF4 import Dataset, datetime, date2num, num2date
import model2roms
import IOstation
import clim2bry
import decimateGrid
import grd
import numpy as np
import atmosForcing
import sys
__author__ = 'Trond Kristiansen'
__email__ = '*****@*****.**'
__created__ = datetime(2009, 1, 30)
__modified__ = datetime(2018, 4, 5)
__version__ = "1.5"
__status__ = "Development"
class Model2romsConfig(object):
def definesubsetforindata(self):
# Subset the input data. The more you subset the less memory is needed for calculations
# and the faster the process is performed. The subset is initially performed in IOsubset.py
subset = np.zeros(4)
if self.outgrid == "NS8KM":
subset[1] = 40
subset[1] = 70
subset[2] = -30
subset[3] = 40
def main():
print '\n--------------------------\n'
print 'Started ' + time.ctime(time.time())
# EDIT ===================================================================
# Set show_progress to "False" if you do not want to see the progress
# indicator for horizontal interpolation.
show_progress = True
# Set compileAll to True if you want automatic re-compilation of all the
# fortran files necessary to run model2roms. Options are "gfortran" or "ifort". Edit
# compile.py to add other Fortran compilers.
compileAll = False
if compileAll is True:
import compile; compile.compileAll("ifort")
# Extract time-series of data for given longitude/latitude
extractStations = False
# Define a set of longitude/latitude positions with names to extract into
# station files (using extractStations)
if (extractStations):
stationNames = ['NorthSea', 'Iceland', 'EastandWestGreenland', 'Lofoten', 'Georges Bank']
lonlist = [2.4301, -22.6001, -47.0801, 13.3801, -67.2001]
latlist = [54.5601, 63.7010, 60.4201, 67.5001, 41.6423]
# Create the bry, init, and clim files for a given grid and input data
createOceanForcing = True
# Create atmospheric forcing for the given grid
createAtmosForcing = False
# Create a smaller resolution grid based on your original. Decimates every second for
# each time run
decimateGridfile = False
# Write ice values to file (for Arctic regions)
writeIce = False
# Use ESMF for the interpolation. This requires that you have ESMF and ESMPy installed (import ESMF)
useESMF = True
# Apply filter to smooth the 2D fields after interpolation (time consuming)
useFilter = True
# Format to write the ouput to: 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', or 'NETCDF3_CLASSIC'
# Using NETCDF4 automatically turns on compression of files (ZLIB)
myformat='NETCDF4'
# Frequency of the input data: usually monthly
timeFrequencyOfInputData = "day" #, "month", "hour"
# Subset input data. If you have global data you may want to seubset these to speed up reading. Make
# sure that your input data are cartesian (0-360 or -180:180, -90:90)
subsetIndata = False
if subsetIndata:
subset = defineSubsetForIndata()
# Set the input data MODEL indatatype
indatatype = 'SODA'
indatatype = 'SODAMONTHLY'
indatatype = 'WOAMONTHLY'
indatatype = 'NORESM'
indatatype = 'GLORYS'
#indatatype = 'NS8KM'
#indatatype = 'NS8KMZ'
# GRIDTYPES ------------------------------------------------------------------------------
# Define what grid type you wnat to interpolate to
outgrid = "NS8KM"
#outgrid = "REGSCEN"
#outgrid = "KINO"
# Define what grid type you wnat to interpolate from: Can be Z for SIGMA for ROMS
# vertical coordinate system or ZLEVEL
ingridtype = "SIGMA"
ingridtype = "ZLEVEL"
outgridtype="ROMS"
# PATH TO DATA ----------------------------------------------------------------------------
# Define the path to the input data
if indatatype == 'SODA':
modelpath = "/Volumes/MacintoshHD2/Datasets/SODA/"
if indatatype == 'SODAMONTHLY':
modelpath = "/Volumes/MacintoshHD2/Datasets/SODAMonthly/"
if indatatype == 'GLORYS':
modelpath = "/Volumes/MacintoshHD2/Datasets/GLOBAL_REANALYSIS_PHYS_001_009/"
modelpath = "/Users/trondkr/Projects/is4dvar/GLORYS2V3/"
modelpath = "/work/shared/imr/NS8KM/FORCING/GLORYS2V3/ftp.myocean.mercator-ocean.fr/Core/GLOBAL_REANALYSIS_PHYS_001_009/"
if indatatype == 'NORESM':
modelpath = "/Users/trondkr/Projects/RegScen/NRCP45AERCN_f19_g16_CLE_01/"
#modelpath = "/work/users/trondk/REGSCEN/NRCP45AERCN_f19_g16_CLE_01/"
#modelpath = "/work/users/trondk/REGSCEN/N20TRAERCN/"
if createAtmosForcing:
atmospath = "/Users/trondkr/Projects/RegScen/model2roms/TESTFILES/"
if indatatype == 'NS8KM':
modelpath = "/Users/trondkr/Projects/is4dvar/grid2lonlat/RESULTS/"
if indatatype == 'NS8KMZ':
modelpath = "/Users/trondkr/Dropbox/deliveryFOFINAL/"
modelpath = "/work/shared/imr/NS8KM/deliveryFO/FINAL/"
if indatatype == 'WOAMONTHLY':
modelpath = "/Users/trondkr/Projects/is4dvar/createSSS/"
# PATH TO GRID -----------------------------------------------------------------------------
# Define the path to the grid file
if outgrid == "NS8KM":
romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_smoothed02022015.nc"
romsgridpath = "/work/users/trondk/NS8km/FORCING/GRID/nordsjoen_8km_grid_hmax20m_v3.nc"
if outgrid == "KINO":
romsgridpath = "/work/users/trondk/KINO/GRID/kino_1600m_07082015_vf20.nc"
#romsgridpath = "/Users/trondkr/Projects/KINO/GRID/kino_1600m_07082015_vf20.nc"
if outgrid == "REGSCEN":
romsgridpath = "/Users/trondkr/Projects/RegScen/Grid/AA_10km_grid_noest.nc"
#romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_grid_hmax20m_v3.nc"
#romsgridpath = "/work/users/trondk/REGSCEN/GRID/AA_10km_grid_noest.nc"
if outgrid == "GREENLAND":
romsgridpath="/Users/trondkr/Projects/RegScen/Grid/Sermilik_grid_4000m.nc"
romsgridpath="/Users/trondkr/Projects/RegScen/model2roms/polarlowr_grid.nc"
if indatatype == 'WOAMONTHLY': isClimatology = True
else: isClimatology = False
# DETAILS -----------------------------------------------------------------------------------
# Define the period to create forcing for
start_year = 2012
end_year = 2013
start_month = 11
end_month = 12
start_day = 15
end_day = 15
if (int(calendar.monthrange(start_year, start_month)[1]) < start_day):
start_day = int(calendar.monthrange(start_year, start_month)[1])
if (int(calendar.monthrange(end_year, end_month)[1]) < end_day):
end_day = int(calendar.monthrange(end_year, end_month)[1])
startdate = datetime(start_year, start_month, start_day)
enddate = datetime(end_year, end_month, end_day)
years = [start_year+year for year in xrange(end_year+1-start_year)]
# Define what and name of variables to include in the forcing files
# -> myvars is the name model2roms uses to identify variables
# -> varNames is the name of the variable found in the NetCDF input files
if indatatype == 'SODA':
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = ['TEMP', 'SALT', 'SSH', 'U', 'V']
if indatatype == 'SODAMONTHLY':
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = ['temp', 'salt', 'ssh', 'u', 'v']
if indatatype == 'NS8KM':
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty']
if indatatype == 'NS8KMZ':
fileNameIn, readFromOneFile = model2roms.getNS8KMZfilename(startdate.year, startdate.month, startdate.day, "S", modelpath)
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
if (readFromOneFile):
varNames = ['temp', 'salt', 'zeta', 'u_eastward', 'v_northward']
else:
varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty']
if indatatype == 'GLORYS':
if (writeIce):
myvars = ['temperature','salinity', 'ssh', 'uvel', 'vvel','uice','vice','aice','hice']
varNames = ['votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty','iicevelu', 'iicevelv', 'ileadfra', 'iicethic']
else:
myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel']
varNames = ['votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty']
if indatatype == 'WOAMONTHLY':
myvars = ['temperature','salinity']
varNames = ['t_an', 's_an']
if indatatype == 'NORESM':
myvars = ['temperature','salinity', 'ssh', 'uvel', 'vvel','ageice','uice','vice','aice','hice','snow_thick']
varNames = ['templvl','salnlvl','sealv', 'uvellvl', 'vvellvl','iage', 'uvel', 'vvel', 'aice', 'hi', 'hs']
# NO EDIT BELOW ====================================================================================================
abbreviation = defineAbbreviation(outgrid)
climName,initName,bryName = defineOutputFilenames(abbreviation,start_year,end_year,start_month,end_month,start_day,end_day,indatatype)
if isClimatology is True:
climName=abbreviation + '_' + str(indatatype) + '_climatology.nc'
# Create the grid object for the output grid
grdROMS = grd.grdClass(romsgridpath, "ROMS", outgridtype, useESMF,'ocean', outgrid)
grdROMS.vars=myvars
if (useESMF):
import ESMF
manager = ESMF.Manager(logkind = ESMF.LogKind.MULTI, debug = True)
if createOceanForcing:
showInfo(myvars, romsgridpath, climName, initName, bryName, start_year, end_year, isClimatology, useESMF, myformat)
model2roms.convertMODEL2ROMS(years, startdate, enddate, timeFrequencyOfInputData, climName, initName, modelpath, romsgridpath, myvars, varNames, show_progress,
indatatype, outgridtype, isClimatology, writeIce, useESMF, useFilter, myformat, subsetIndata, outgrid, subset=None)
clim2bry.writeBry(grdROMS, start_year, bryName, climName, writeIce, indatatype, myformat)
if createAtmosForcing:
atmosForcing.createAtmosFileUV(grdROMS,modelpath,atmospath,startdate,enddate,useESMF,
myformat,abbreviation,indatatype,gridtype,show_progress)
#if decimateGridfile:
#decimateGrid.createGrid(grdROMS, "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015.nc", "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015v2.nc", 2)
if extractStations:
print "Running in station mode and extracting pre-defined station locations"
IOstation.getStationData(years, IDS, modelpath, latlist, lonlist, stationNames)
print 'Finished ' + time.ctime(time.time())
ncfile = Dataset('/ninod/baird/cmip5/obs/ERSSTv4/sst.mnmean.v4_invertlat_72x144regrid.nc', 'r', format='NETCDF4')
sst_data_orig = ncfile.variables['sst'][:]
sst_lat = ncfile.variables['lat'][:]
sst_lon = ncfile.variables['lon'][:]
global_nlat, global_nlon = sst_data_orig.shape[1:3]
global_lat_vals = sst_lat[:]
global_lon_vals = sst_lon[:]
##########################################################################################
##########################################################################################
##########################################################################################
# (YYYY,MM,DD)
hist_start = netCDF4.datetime(1969,12,1)
hist_end = netCDF4.datetime(2000,2,28)
sst_histmonths_list = []
sst_histclim_list = []
hist_stdevs_list = []
# OPEN HISTORICAL PERIOD PR DATA
file_name = '/ninod/baird/cmip5/obs/ERSSTv4/sst.mnmean.v4_invertlat_72x144regrid.nc'
ncfile = Dataset(file_name, 'r', format='NETCDF4')
sst_hist_data = ncfile.variables['sst'][:,:,:]
time_variable = ncfile.variables['time']
date_start = netCDF4.date2num(hist_start, time_variable.units, time_variable.calendar)
date_end = netCDF4.date2num(hist_end, time_variable.units, time_variable.calendar)
season_names = ['djf','mam','jja','son','ondjfm']
season='djf'
#season='jja'
#season='annual'
##########################################################################################
##########################################################################################
##########################################################################################
# (YYYY,MM,DD)
#hist_start = netCDF4.datetime(1979,1,1)
#hist_end = netCDF4.datetime(2009,12,31)
hist_start = netCDF4.datetime(1970,1,1)
hist_end = netCDF4.datetime(2000,12,31)
u200_histmonths_list = []
u200_histclim_list = []
hist_stdevs_list = []
#ncfile = Dataset('/Users/baird/google_drive/_data_original/era_interim/u200_MERRA_monthly_197901-201412_invertlat_96x144regrid.nc', 'r', format='NETCDF4')
ncfile = Dataset('/ninod/baird/cmip5/obs/MERRA/u200/MERRA_197901-201412_u200_monthly_2.5x2.5regrid.nc', 'r', format='NETCDF4')
u200_hist_data = ncfile.variables['u200'][:]
u200_lat = ncfile.variables['lat'][:]
u200_lon = ncfile.variables['lon'][:]
global_nlat, global_nlon = u200_hist_data.shape[1:3]
global_lat_vals = u200_lat[:]
global_lon_vals = u200_lon[:]
def selmonthyear(year, month, t_coord, t_units, t_cal):
ti = netcdf.date2num(netcdf.datetime(year, month, 1), t_units, t_cal)
tf = netcdf.date2num(netcdf.datetime(year, month + 1, 1), t_units, t_cal)
ind_t = np.where(np.bitwise_and(t_coord >= ti, t_coord <= tf))
return ind_t
from netCDF4 import Dataset, datetime, date2num,num2date import numpy as np import os from subprocess import call __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2017, 2, 1) __modified__ = datetime(2017, 2, 1) __version__ = "1.0" __status__ = "Development, modified on 01.02.2015" myprefix='ws4km' myvars=['Pair','Uwind','Vwind','cloud','Tair','Qair','rain'] years=[year for year in xrange(1989,2015,1)] command = "export SKIP_SAME_TIME=1" call(command,shell=True) for myvar in myvars: newfilename="%s_%s_%s_%s.nc"%(myprefix,myvar,years[0],years[-1]) mylistoffiles=[] if os.path.exists(newfilename): os.remove(newfilename) for year in years: filename="%s_%s_%s.nc"%(myprefix,myvar,year) mylistoffiles.append(filename) if len(mylistoffiles)>0: options="" for f in mylistoffiles:
