def plot_histograms(self, save=False, save_dir=None, fancy=True): '''Create an timeseries plot of specific variables. Parameters:: ---------- save = boolean True to save an output plot, False displays on screen. fancy = boolean If set to true uses ggplot style, otherwise default is used. ''' if fancy: matplotlib.style.use('ggplot') # Set the time string format fmt = '%H:%M:%S' # Create figure/axes instance fig, axs = plt.subplots(nrows=4, ncols=3, figsize=(8, 11), \ sharey="row") for ax, fills in zip(axs.flat, self.dts): # Calculate start and end times stime = netCDF4.datetime(fills[0][0], fills[0][1], fills[0][2],\ fills[0][3], fills[0][4], fills[0][5]) etime = netCDF4.datetime(fills[1][0], fills[1][1], fills[1][2],\ fills[1][3], fills[1][4], fills[1][5]) # Make title from these times figtitle = stime.strftime(fmt) + ' - ' + etime.strftime(fmt) ##if self.pcasp[ # Calculate Poisson error from Cai et al. AMT (2013), Equation A4 yerr = self.pcaspf.between_time(stime, etime).sum(axis=0).divide(self.flow3s.between_time(stime, etime).sum(axis=0)**2).pow(0.5).mul(1./self.dlog10d) # Calculate the dNd(log10{D}) value for this time period, # sum at each diameter (bin), and replace zero values with NaN dndlogp = self.pcaspf.between_time(stime, etime).sum(axis=0).divide(\ (self.flow3s.between_time(stime, etime).sum(axis=0) * self.dlog10d)) # Create a pandas Series for plotting dndlogpf = pd.Series(data=dndlogp.replace(to_replace=0., value=np.nan), \ index=self.pcasp_bin_mid, name=figtitle) xlabels = self.pcasp_bin_mid.astype('|S5') dndlogpf.plot(ax=ax, logx=True, logy=True, yerr=yerr, legend=False, x=xlabels) ax.set_title(figtitle) ax.set_xlim(left=0.1, right=10.) ax.set_ylim(bottom=0.01, top=10000.) ax.xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter()) ax.yaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter()) ax.set_xlabel(r'D ($\mu$m)') ax.set_ylabel(r"dN/dlog$_{10}$D (cm$^{-3}$)") fig.tight_layout() if save: # Set the output file name fname = "pcasp_timehist_" + os.path.splitext(self.filename)[0] + ".png" outfile = os.sep.join([save_dir, fname]) plt.savefig(outfile, format='png') print "Creating image: " + outfile else: plt.show()
def _draw_time_region(self, ax): '''A function to highlight time regions for further analysis. Parameters:: ---------- ax = Matplotlib axis instance Used for drawing polygons on that axis. ''' # Get the y-limits to match the polycollection ylim = ax.get_ylim() # Loop through the times and create for fills in self.dts: stime = netCDF4.datetime(fills[0][0], fills[0][1], fills[0][2],\ fills[0][3], fills[0][4], fills[0][5]) etime = netCDF4.datetime(fills[1][0], fills[1][1], fills[1][2],\ fills[1][3], fills[1][4], fills[1][5]) ax.fill_between(self.time, ylim[0], ylim[1], \ where=(self.time>=stime)&(self.time<=etime), alpha=0.3)
def shift_time(infilename, outfilename, amount, shift_units): """ Do the actual time shift :param infilename: the name of the input file (full or relative path) :param outfilename: the name of the output file (full or relative path) :param amount: amount to shift the time :type amount: floating point number, of a string version :param shift_units: units of the time shift :type shift_units: string """ units = str(shift_units).strip() # make a copy of the file if it's different if not infilename == outfilename: print "creating new file:", outfilename shutil.copyfile(infilename, outfilename) # pull the time units out of the source copy: nc = netCDF4.Dataset(infilename, mode='r') time_var = nc.variables['time'] time_units = time_var.units print "original time units", time_units nc.close() units, epoch = time_units.split('since') # iso datetimes can be separated with whitesapce or "T" try: date, time = epoch.strip().split() except ValueError: date, time = epoch.strip().split("T") year, month, day = [int(i) for i in date.split('-')] hour, minute, second = [int(i) for i in time.split(':')] dt = netCDF4.datetime(year, month, day, hour, minute, second) try: shift = netCDF4.timedelta(**{shift_units: float(amount)}) except TypeError: raise ValueError( "Time shift must units options are: days, seconds, minutes, hours, or weeks" ) dt += shift new_units = "%s since %s" % (units.strip(), dt.isoformat()) print "new time units:", new_units ## write out to the new file: nc = netCDF4.Dataset(outfilename, mode='r+') time_var = nc.variables['time'] time_var.units = new_units nc.close() print "Done"
def sel_season(var, dates, season): #---------------------------------------------------------------------------------------- #print('____________________________________________________________________________________________________________________') #print('Selecting only {0} data'.format(season)) #---------------------------------------------------------------------------------------- dates_pdh = pd.to_datetime(dates) if season == 'DJF': #ONLY DEC-JAN-FEB m = [12, 1, 2] mask = (dates_pdh.month == 12) | (dates_pdh.month == 1) | (dates_pdh.month == 2) elif season == 'DJFM': #ONLY DEC-JAN-FEB-MAR m = [12, 1, 2, 3] mask = (dates_pdh.month == 12) | (dates_pdh.month == 1) | ( dates_pdh.month == 2) | (dates_pdh.month == 3) elif season == 'NDJFM': #ONLY NOV-DEC-JAN-FEB-MAR m = [11, 12, 1, 2, 3] mask = (dates_pdh.month == 11) | (dates_pdh.month == 12) | ( dates_pdh.month == 1) | (dates_pdh.month == 2) | (dates_pdh.month == 3) elif season == 'JJA': #ONLY JUN-JUL-AUG m = [6, 7, 8] mask = (dates_pdh.month == 6) | (dates_pdh.month == 7) | (dates_pdh.month == 8) else: print('season is not one of the following: DJF, DJFM, NDJFM, JJA') var_season = var[mask, :, :] dates_season = dates[mask] if (12 in m) or (1 in m): #REMOVING THE FIRST MONTHS (for the first year) because there is no previuos december start = int( np.where(dates_season == datetime(dates_pdh.year[0], m[0], dates_pdh.day[0], dates_pdh.hour[0], dates_pdh.minute[0]))[0]) #REMOVING THE LAST MONTHS (for the last year) because there is no following january end = int( np.where(dates_season == datetime(dates_pdh.year[-1], m[0], dates_pdh.day[0], dates_pdh.hour[0], dates_pdh.minute[0]))[0]) var_season = var_season[start:end, :, :] dates_season = dates_season[start:end] return var_season, dates_season
def parseDate(dat): dateRe = r'^(\d{4})(?:-|\.|\/)?(0[1-9]|1[0-2])(?:-|\.|\/)?(0[1-9]|[1-2]\d|3[01])(?:T?([01]\d|2[0-3])(?::?([0-5]\d)(?::?([0-5]\d))?)?)?$' parDat = re.match(dateRe, dat) if parDat: parDat = [int(e) for e in parDat.groups(default=0)] #If it is efectively a date return datetime(*parDat) else: return None
def shift_time(infilename, outfilename, amount, shift_units): """ Do the actual time shift :param infilename: the name of the input file (full or relative path) :param outfilename: the name of the output file (full or relative path) :param amount: amount to shift the time :type amount: floating point number, of a string version :param shift_units: units of the time shift :type shift_units: string """ units = str(shift_units).strip() # make a copy of the file if it's different if not infilename == outfilename: print("creating new file:", outfilename) shutil.copyfile(infilename, outfilename) # pull the time units out of the source copy: nc = netCDF4.Dataset(infilename, mode='r') time_var = nc.variables['time'] time_units = time_var.units print("original time units", time_units) nc.close() units, epoch = time_units.split('since') # iso datetimes can be separated with whitesapce or "T" try: date, time = epoch.strip().split() except ValueError: date, time = epoch.strip().split("T") year, month, day = [int(i) for i in date.split('-')] hour, minute, second = [int(i) for i in time.split(':')] dt = netCDF4.datetime(year, month, day, hour, minute, second) try: shift = netCDF4.timedelta(**{shift_units:float(amount)}) except TypeError: raise ValueError("Time shift must units options are: days, seconds, minutes, hours, or weeks") dt+=shift new_units = "%s since %s"%(units.strip(), dt.isoformat()) print("new time units:", new_units) ## write out to the new file: nc = netCDF4.Dataset(outfilename, mode='r+') time_var = nc.variables['time'] time_var.units = new_units nc.close() print("Done")
def seltimeframe(scenario, t_coord, t_units, t_cal): if scenario == 'historical': yi = 1860 yf = 2005 elif (scenario == 'rcp45') or (scenario == 'rcp85'): yi = 2005 yf = 2100 elif scenario == '1pctCO2': yi_tmp = t_units[ 11:15] # time axis for 1pctCO2 varies from one model to another if yi_tmp == "1-1-": yi = 1 else: yi = int(yi_tmp) yf = yi + 139 ti = netcdf.date2num(netcdf.datetime(yi, 1, 1), t_units, t_cal) if t_cal == '360_day': tf = netcdf.date2num(netcdf.datetime(yf, 12, 30), t_units, t_cal) else: tf = netcdf.date2num(netcdf.datetime(yf, 12, 31), t_units, t_cal) ind_t = np.where(np.bitwise_and(t_coord >= ti, t_coord <= tf + 1)) return ind_t
def fetch_radar_time_profile(sonde_dset, radar, time_key='time', height_key='height', nvars=None): """ Extract the correct profile from a interpolated sonde. This is an ARM specific method which extract the correct profile out of netCDF Variables from a Interpolated Sonde VAP for the volume start time of a radar object. Parameters ---------- sonde_dset : Dataset Interpolate sonde Dataset. radar : Radar Radar object from which the nearest profile will be found. time_key : string, optional Key to find a CF startard time variable height_key : string, optional Key to find profile height data nvars : list, optional NetCDF variable to generated profiles for. If None (the default) all variables with dimension of time, height will be found in ncvars. Returns ------- return_dic : dict Profiles at the start time of the radar """ ncvars = sonde_dset.variables if nvars is None: time_height_shape = (len(ncvars[time_key]), len(ncvars[height_key])) nvars = [k for k, v in ncvars.items() if v.shape == time_height_shape] radar_start = netCDF4.num2date(radar.time['data'][0], radar.time['units']) radar_day_start = netCDF4.datetime(radar_start.year, radar_start.month, radar_start.day) seconds_since_start_of_day = (radar_start - radar_day_start).seconds time_index = abs(ncvars[time_key][:] - seconds_since_start_of_day).argmin() return_dic = dict([(key, ncvars[key][time_index, :]) for key in nvars]) return_dic[height_key] = ncvars[height_key][:] return return_dic
def fetch_radar_time_profile(sonde_dset, radar, time_key='time', height_key='height', nvars=None): """ Extract the correct profile from a interpolated sonde. This is an ARM specific method which extract the correct profile out of netCDF Variables from a Interpolated Sonde VAP for the volume start time of a radar object. Parameters ---------- sonde_dset : Dataset Interpolate sonde Dataset. radar : Radar Radar object from which the nearest profile will be found. time_key : string, optional Key to find a CF startard time variable height_key : string, optional Key to find profile height data nvars : list, optional NetCDF variable to generated profiles for. If None (the default) all variables with dimension of time, height will be found in ncvars. Returns ------- return_dic : dict Profiles at the start time of the radar """ ncvars = sonde_dset.variables if nvars is None: time_height_shape = (len(ncvars[time_key]), len(ncvars[height_key])) nvars = [k for k, v in ncvars.items() if v.shape == time_height_shape] radar_start = num2date(radar.time['data'][0], radar.time['units']) radar_day_start = datetime(radar_start.year, radar_start.month, radar_start.day) seconds_since_start_of_day = (radar_start - radar_day_start).seconds time_index = abs(ncvars[time_key][:] - seconds_since_start_of_day).argmin() return_dic = dict([(key, ncvars[key][time_index, :]) for key in nvars]) return_dic[height_key] = ncvars[height_key][:] return return_dic
########################################################################################## ########################################################################################## season_names = ['djf', 'mam', 'jja', 'son', 'ondjfm'] season = 'djf' #season='jja' #season='annual' ########################################################################################## ########################################################################################## ########################################################################################## # (YYYY,MM,DD) hist_start = netCDF4.datetime(1979, 12, 1) hist_end = netCDF4.datetime(2010, 2, 28) sst_histmonths_list = [] sst_histclim_list = [] hist_stdevs_list = [] #ncfile = Dataset('/Users/baird/google_drive/_data_original/era_interim/u200_MERRA_monthly_197901-201412_invertlat_96x144regrid.nc', 'r', format='NETCDF4') ncfile = netCDF4.Dataset( '/ninod/baird/cmip5/obs/MERRA/u200/MERRA_197901-201412_u200_monthly_2.5x2.5regrid.nc', 'r', format='NETCDF4') u200_hist_data = ncfile.variables['u200'][:] u200_lat = ncfile.variables['lat'][:] u200_lon = ncfile.variables['lon'][:]
def get2Ddata(grdROMS, grdMODEL, myvar, indatatype, year, month, day, varNames, dataPath): indexROMS_SSH = (grdROMS.eta_rho, grdROMS.xi_rho) if indatatype == "NORESM": if myvar == 'ageice': varN = 5; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'uice': varN = 6; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'vice': varN = 7; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'aice': varN = 8; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'hice': varN = 9; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'snow_thick': varN = 10; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if indatatype == "GLORYS": if myvar == 'uice': varN = 5; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'vice': varN = 6; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'aice': varN = 7; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'hice': varN = 8; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'ssh': varN = 2; SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if grdMODEL.useESMF: if indatatype == "SODA": filename = getSODAfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[varNames[varN]][0,:,:] if indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][:,:] if indatatype == "WOAMONTHLY": filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][month - 1, :,:] if indatatype == "NORESM": cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath)) #myunits = cdf.variables[str(varNames[varN])].units # For NORESM data are 12 months of data stored in ice files. Use ID as month indicator to get data. if myvar in ['ageice','uice','vice','aice','hice','snow_thick']: data = np.squeeze(cdf.variables[str(varNames[varN])][month-1, :,:]) else: data = np.squeeze(cdf.variables[str(varNames[varN])][0, :,:]) data=np.where(data.mask,grdROMS.fill_value,data) if indatatype == "GLORYS": cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath)) data = np.squeeze(cdf.variables[str(varNames[varN])][0, :,:]) data=np.where(data.mask,grdROMS.fill_value,data) if indatatype == "NS8KMZ": filename, readFromOneFile = getNS8KMZfilename(year, month, day, varNames[varN], dataPath) cdf = Dataset(filename) print "Reading from file", filename if (readFromOneFile): jdref = date2num(datetime(1948,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar) currentdate = datetime(year, month, day, 12) jd = date2num(currentdate, cdf.variables["time"].units, calendar="gregorian") timesteps = (cdf.variables["time"][:]).tolist() timeindex = timesteps.index(jd) data = np.squeeze(cdf.variables[str(varNames[varN])][timeindex,:,:]) else: data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:]) data=np.where(data.mask,grdROMS.fill_value,data) print "Extracted raw data: %s min: %s max: %s"%(myvar,np.min(data),np.max(data)) cdf.close() else: if grdMODEL.splitExtract is True: if indatatype == "SODA": filename = getSODAfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data1 = cdf.variables[varNames[varN]][0, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] data2 = cdf.variables[varNames[varN]][0, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])] if indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data1 = cdf.variables[str(varNames[varN])][ int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] data2 = cdf.variables[str(varNames[varN])][ int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])] if indatatype == "GLORYS": cdf = Dataset(getGLORYSfilename(year, month, day, '2D', dataPath)) data1 = np.squeeze(cdf.variables[str(varNames[varN])][0, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) data2 = np.squeeze(cdf.variables[str(varNames[varN])][0, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]) if indatatype == "NORESM": cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath)) if myvar in ['ageice','uice','vice','aice','hice','snow_thick']: data1 = np.squeeze(cdf.variables[str(varNames[varN])][month-1, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) data2 = np.squeeze(cdf.variables[str(varNames[varN])][month-1, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]) else: data1 = np.squeeze(cdf.variables[str(varNames[varN])][0, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) data2 = np.squeeze(cdf.variables[str(varNames[varN])][0, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]) cdf.close() data = np.concatenate((data1, data2), axis=1) else: if indatatype == "SODA": filename = getSODAfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][0, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] if indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][ int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] if indatatype == "GLORYS": cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath)) data = np.squeeze(cdf.variables[str(varNames[varN])][0, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) if indatatype == "NORESM": cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath)) if myvar in ['ageice','uice','vice','aice','hice','snow_thick']: data = np.squeeze(cdf.variables[str(varNames[varN])][month-1, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) else: data = np.squeeze(cdf.variables[str(varNames[varN])][0, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) cdf.close() # data = np.where(abs(data) == grdMODEL.fill_value, grdROMS.fill_value, data) #data = np.ma.where(abs(data) > 1000, grdROMS.fill_value, data) if indatatype == "GLORYS": data = np.where(data <= -32.767, grdROMS.fill_value, data) data = np.ma.masked_where(data <= grdROMS.fill_value, data) if __debug__: print "Data range of %s just after extracting from netcdf file: %s - %s" % (str(varNames[varN]), data.min(), data.max()) return data
def main(): print '\n--------------------------\n' print 'Started ' + time.ctime(time.time()) # EDIT =================================================================== # Set show_progress to "False" if you do not want to see the progress # indicator for horizontal interpolation. show_progress = True # Set compileAll to True if you want automatic re-compilation of all the # fortran files necessary to run model2roms. Options are "gfortran" or "ifort". Edit # compile.py to add other Fortran compilers. compileAll = False if compileAll is True: import compile compile.compileAll("ifort") # Extract time-series of data for given longitude/latitude extractStations = False # Define a set of longitude/latitude positions with names to extract into # station files (using extractStations) if (extractStations): stationNames = [ 'NorthSea', 'Iceland', 'EastandWestGreenland', 'Lofoten', 'Georges Bank' ] lonlist = [2.4301, -22.6001, -47.0801, 13.3801, -67.2001] latlist = [54.5601, 63.7010, 60.4201, 67.5001, 41.6423] # Create the bry, init, and clim files for a given grid and input data createOceanForcing = True # Create atmospheric forcing for the given grid createAtmosForcing = False # Create a smaller resolution grid based on your original. Decimates every second for # each time run decimateGridfile = False # Write ice values to file (for Arctic regions) writeIce = True # Use ESMF for the interpolation. This requires that you have ESMF and ESMPy installed (import ESMF) useESMF = True # Apply filter to smooth the 2D fields after interpolation (time consuming) useFilter = True # Format to write the ouput to: 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', or 'NETCDF3_CLASSIC' # Using NETCDF4 automatically turns on compression of files (ZLIB) myformat = 'NETCDF4' # Frequency of the input data: usually monthly timeFrequencyOfInputData = "month" #, "month", "hour" # Subset input data. If you have global data you may want to seubset these to speed up reading. Make # sure that your input data are cartesian (0-360 or -180:180, -90:90) subsetIndata = False if subsetIndata: subset = defineSubsetForIndata() # Set the input data MODEL indatatype indatatype = 'SODA' indatatype = 'SODAMONTHLY' indatatype = 'WOAMONTHLY' indatatype = 'NORESM' indatatype = 'GLORYS' #indatatype = 'NS8KM' #indatatype = 'NS8KMZ' # GRIDTYPES ------------------------------------------------------------------------------ # Define what grid type you wnat to interpolate to outgrid = "NS8KM" #outgrid = "REGSCEN" #outgrid = "KINO" # outgrid= "AA10KM" # Define what grid type you wnat to interpolate from: Can be Z for SIGMA for ROMS # vertical coordinate system or ZLEVEL ingridtype = "SIGMA" ingridtype = "ZLEVEL" outgridtype = "ROMS" # PATH TO DATA ---------------------------------------------------------------------------- # Define the path to the input data if indatatype == 'SODA': modelpath = "/Volumes/MacintoshHD2/Datasets/SODA/" if indatatype == 'SODAMONTHLY': modelpath = "/Volumes/MacintoshHD2/Datasets/SODAMonthly/" if indatatype == 'GLORYS': modelpath = "/Volumes/MacintoshHD2/Datasets/GLOBAL_REANALYSIS_PHYS_001_009/" modelpath = "/Users/trondkr/Projects/is4dvar/GLORYS2V3/" modelpath = "/work/shared/imr/NS8KM/FORCING/GLORYS2V3/ftp.myocean.mercator-ocean.fr/Core/GLOBAL_REANALYSIS_PHYS_001_009/" if indatatype == 'NORESM': modelpath = "/Users/trondkr/Projects/RegScen/NRCP45AERCN_f19_g16_CLE_01/" #modelpath = "/work/users/trondk/REGSCEN/NRCP45AERCN_f19_g16_CLE_01/" #modelpath = "/work/users/trondk/REGSCEN/N20TRAERCN/" if createAtmosForcing: atmospath = "/Users/trondkr/Projects/RegScen/model2roms/TESTFILES/" if indatatype == 'NS8KM': modelpath = "/Users/trondkr/Projects/is4dvar/grid2lonlat/RESULTS/" if indatatype == 'NS8KMZ': modelpath = "/Users/trondkr/Projects/NOWMAPS/delivery-COPERNICUS/" #modelpath = "/work/shared/imr/NS8KM/Z-LEVEL-ASSIMILATION2010-2013/" if indatatype == 'WOAMONTHLY': modelpath = "/Users/trondkr/Projects/is4dvar/createSSS/" # PATH TO GRID ----------------------------------------------------------------------------- # Define the path to the grid file if outgrid == "NS8KM": romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_smoothed02022015.nc" romsgridpath = "/work/shared/imr/NS8KM/FORCING/GRID/nordsjoen_8km_grid_hmax20m_v4.nc" if outgrid == "KINO": romsgridpath = "/work/users/trondk/KINO/GRID/kino_1600m_19022016_vf20.nc" #romsgridpath = "/Users/trondkr/Projects/KINO/GRID/kino_1600m_07082015_vf20.nc" if outgrid == "REGSCEN": romsgridpath = "/Users/trondkr/Projects/RegScen/Grid/AA_10km_grid_noest.nc" #romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_grid_hmax20m_v3.nc" #romsgridpath = "/work/users/trondk/REGSCEN/GRID/AA_10km_grid_noest.nc" if outgrid == "GREENLAND": romsgridpath = "/Users/trondkr/Projects/RegScen/Grid/Sermilik_grid_4000m.nc" romsgridpath = "/Users/trondkr/Projects/RegScen/model2roms/polarlowr_grid.nc" if outgrid == "AA10KM": romsgridpath = "/work-common/shared/imr/AA10KM/GRID/AA_10km_grid.nc" if indatatype == 'WOAMONTHLY': isClimatology = True else: isClimatology = False # DETAILS ----------------------------------------------------------------------------------- # Define the period to create forcing for start_year = 2013 end_year = 2014 start_month = 11 end_month = 12 start_day = 15 end_day = 15 if (int(calendar.monthrange(start_year, start_month)[1]) < start_day): start_day = int(calendar.monthrange(start_year, start_month)[1]) if (int(calendar.monthrange(end_year, end_month)[1]) < end_day): end_day = int(calendar.monthrange(end_year, end_month)[1]) startdate = datetime(start_year, start_month, start_day) enddate = datetime(end_year, end_month, end_day) years = [start_year + year for year in xrange(end_year + 1 - start_year)] # Define what and name of variables to include in the forcing files # -> myvars is the name model2roms uses to identify variables # -> varNames is the name of the variable found in the NetCDF input files if indatatype == 'SODA': myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = ['TEMP', 'SALT', 'SSH', 'U', 'V'] if indatatype == 'SODAMONTHLY': myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = ['temp', 'salt', 'ssh', 'u', 'v'] if indatatype == 'NS8KM': myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty'] if indatatype == 'NS8KMZ': fileNameIn, readFromOneFile = model2roms.getNS8KMZfilename( startdate.year, startdate.month, startdate.day, "S", modelpath) myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] if (readFromOneFile): varNames = ['temp', 'salt', 'zeta', 'u_eastward', 'v_northward'] else: varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty'] if indatatype == 'GLORYS': if (writeIce): myvars = [ 'temperature', 'salinity', 'ssh', 'uvel', 'vvel', 'uice', 'vice', 'aice', 'hice' ] varNames = [ 'votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty', 'iicevelu', 'iicevelv', 'ileadfra', 'iicethic' ] else: myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = [ 'votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty' ] if indatatype == 'WOAMONTHLY': myvars = ['temperature', 'salinity'] varNames = ['t_an', 's_an'] if indatatype == 'NORESM': myvars = [ 'temperature', 'salinity', 'ssh', 'uvel', 'vvel', 'ageice', 'uice', 'vice', 'aice', 'hice', 'snow_thick' ] varNames = [ 'templvl', 'salnlvl', 'sealv', 'uvellvl', 'vvellvl', 'iage', 'uvel', 'vvel', 'aice', 'hi', 'hs' ] # NO EDIT BELOW ==================================================================================================== abbreviation = defineAbbreviation(outgrid) climName, initName, bryName = defineOutputFilenames( abbreviation, start_year, end_year, start_month, end_month, start_day, end_day, indatatype) if isClimatology is True: climName = abbreviation + '_' + str(indatatype) + '_climatology.nc' # Create the grid object for the output grid grdROMS = grd.grdClass(romsgridpath, "ROMS", outgridtype, useESMF, 'ocean', outgrid) grdROMS.vars = myvars if (useESMF): import ESMF manager = ESMF.Manager(logkind=ESMF.LogKind.MULTI, debug=True) if createOceanForcing: showInfo(myvars, romsgridpath, climName, initName, bryName, start_year, end_year, isClimatology, useESMF, myformat) model2roms.convertMODEL2ROMS(years, startdate, enddate, timeFrequencyOfInputData, climName, initName, modelpath, romsgridpath, myvars, varNames, show_progress, indatatype, outgridtype, isClimatology, writeIce, useESMF, useFilter, myformat, subsetIndata, outgrid, subset=None) clim2bry.writeBry(grdROMS, start_year, bryName, climName, writeIce, indatatype, myformat) if createAtmosForcing: atmosForcing.createAtmosFileUV(grdROMS, modelpath, atmospath, startdate, enddate, useESMF, myformat, abbreviation, indatatype, gridtype, show_progress) #if decimateGridfile: #decimateGrid.createGrid(grdROMS, "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015.nc", "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015v2.nc", 2) if extractStations: print "Running in station mode and extracting pre-defined station locations" IOstation.getStationData(years, IDS, modelpath, latlist, lonlist, stationNames) print 'Finished ' + time.ctime(time.time())
def main(in_dir="/RESCUE/skynet3_rech1/huziy/anusplin_links", out_dir="/HOME/huziy/skynet3_rech1/hail/anusplin_ts"): out_dir_p = Path(out_dir) in_dir_p = Path(in_dir) lon0 = -114.0708 lat0 = 51.0486 vname = "daily_precipitation_accumulation" vname_alternatives = ["daily_accumulation_precipitation"] vname_alternatives.append(vname) var_list = [vname] fname_hint = "pcp" spatial_ind = None varname_to_list_of_frames = {vname: [] for vname in var_list} for fin in in_dir_p.iterdir(): if fin.name.lower().endswith("ret"): continue if fin.name.lower().endswith("verif"): continue if fname_hint not in fin.name.lower(): continue if not fin.name.endswith(".nc"): continue print(fin) year, month = get_ym_from_path(fin) with Dataset(str(fin)) as ds: if spatial_ind is None: lons, lats = ds.variables["lon"][:], ds.variables["lat"][:] x, y, z = lat_lon.lon_lat_to_cartesian(lons.flatten(), lats.flatten()) ktree = KDTree(list(zip(x, y, z))) x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon0, lat0) dist, spatial_ind = ktree.query((x0, y0, z0)) for vname_alt in vname_alternatives: try: values = ds[vname_alt][:] values = [field.flatten()[spatial_ind] for field in values] break except IndexError as ierr: pass dates = [datetime(year, month, int(d)) for d in ds["time"][:]] varname_to_list_of_frames[vname].append(pd.DataFrame(index=dates, data=values)) for vname in var_list: df = pd.concat(varname_to_list_of_frames[vname]) assert isinstance(df, pd.DataFrame) df.sort_index(inplace=True) df.to_csv(str(out_dir_p.joinpath("{}.csv".format(vname))), float_format="%.3f", index_label="Time")
def get2ddata(confM2R, myvar, year, month, day): indexROMS_SSH = (confM2R.grdROMS.eta_rho, confM2R.grdROMS.xi_rho) if confM2R.indatatype == "NORESM": if myvar == 'ageice': varN = 5 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'uice': varN = 6 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'vice': varN = 7 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'aice': varN = 8 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'hice': varN = 9 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'snow_thick': varN = 10 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if confM2R.indatatype == "GLORYS": if myvar == 'uice': varN = 5 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'vice': varN = 6 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'aice': varN = 7 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'hice': varN = 8 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) print("Current type %s and variable %s" % (confM2R.indatatype, myvar)) if confM2R.indatatype == "SODA3": if myvar == 'aice': varN = 5 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'hice': varN = 6 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'snow_thick': varN = 7 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if myvar == 'ssh': varN = 2 SSHdata = np.zeros((indexROMS_SSH), dtype=np.float64) if confM2R.useesmf: if confM2R.indatatype == "SODA": filename = getSODAfilename(confM2R, year, month, day, None) cdf = Dataset(filename) data = cdf.variables[confM2R.inputdatavarnames[varN]][0, :, :] if confM2R.indatatype == "SODA3": filename = getSODA3filename(confM2R, year, month, confM2R.inputdatavarnames[varN]) print("Trying to open file %s" % (filename)) cdf = Dataset(filename) if myvar == 'aice': # We only extract the first thickness concentration. Need to fix this so all 5 classes can be extracted. # http://www.atmos.umd.edu/~ocean/index_files/soda3_readme.htm # hi: sea ice thickness [m ice] # mi: sea ice mass [kg/m^2] # hs: snow thickness [m snow] # {cn1,cn2,cn3,cn4,cn5}: sea ice concentration [0:1] in five ice thickness classes data = cdf.variables[confM2R.inputdatavarnames[varN]][ int(month - 1), 0, :, :] else: data = cdf.variables[confM2R.inputdatavarnames[varN]][ int(month - 1), :, :] if confM2R.indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(confM2R, year, month, None) cdf = Dataset(filename) data = cdf.variables[str(confM2R.inputdatavarnames[varN])][:, :] if confM2R.indatatype == "WOAMONTHLY": filename = getWOAMONTHLYfilename(confM2R, year, month, myvar) cdf = Dataset(filename) data = cdf.variables[str(confM2R.inputdatavarnames[varN])][month - 1, :, :] if confM2R.indatatype == "NORESM": cdf = Dataset( getNORESMfilename(confM2R, year, month, confM2R.inputdatavarnames[varN])) # myunits = cdf.variables[str(grdROMS.varNames[varN])].units # For NORESM data are 12 months of data stored in ice files. Use ID as month indicator to get data. if myvar in [ 'ageice', 'uice', 'vice', 'aice', 'hice', 'snow_thick' ]: data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][month - 1, :, :]) else: data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][0, :, :]) data = np.where(data.mask, confM2R.grdROMS.fillval, data) if confM2R.indatatype == "GLORYS": cdf = Dataset( getGLORYSfilename(confM2R, year, month, confM2R.inputdatavarnames[varN])) data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][0, :, :]) data = np.where(data.mask, confM2R.grdROMS.fillval, data) if confM2R.indatatype == "NS8KMZ": filename, readFromOneFile = getNS8KMZfilename( confM2R, year, month, confM2R.inputdatavarnames[varN]) cdf = Dataset(filename) print("Reading from file", filename) if (readFromOneFile): jdref = date2num(datetime(1948, 1, 1), cdf.variables["time"].units, calendar=cdf.variables["time"].calendar) currentdate = datetime(year, month, day, 12) jd = date2num(currentdate, cdf.variables["time"].units, calendar="gregorian") timesteps = (cdf.variables["time"][:]).tolist() timeindex = timesteps.index(jd) data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][timeindex, :, :]) else: data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][0, :, :]) data = np.where(data.mask, confM2R.grdROMS.fillval, data) print("Extracted raw data: %s min: %s max: %s" % (myvar, np.min(data), np.max(data))) cdf.close() if __debug__: print( "Data range of %s just after extracting from netcdf file: %s - %s" % (str(confM2R.inputdatavarnames[varN]), data.min(), data.max())) return data
def getTime(confM2R, year, month, day): """ Create a date object to keep track of Julian dates etc. Also create a reference date starting at 1948/01/01. Go here to check results:http://lena.gsfc.nasa.gov/lenaDEV/html/doy_conv.html """ if confM2R.indatatype == 'SODA': filename = getSODAfilename(confM2R, year, month, None) if confM2R.indatatype == 'SODA3': filename = getSODA3filename(confM2R, year, month, None) if confM2R.indatatype == 'SODAMONTHLY': filename = getSODAMONTHLYfilename(confM2R, year, month, None) if confM2R.indatatype == 'GLORYS': filename = getGLORYSfilename(confM2R, year, month, "S") if confM2R.indatatype == 'WOAMONTHLY': filename = getWOAMONTHLYfilename(confM2R, year, month, "temperature") if confM2R.indatatype == 'NORESM': filename = getNORESMfilename(confM2R, year, month, "saln") if confM2R.indatatype == 'NS8KM': filename = getNS8KMfilename(confM2R, year, month, "salt") if confM2R.indatatype == 'NS8KMZ': filename, readFromOneFile = getNS8KMZfilename(confM2R, year, month, "salt") # Now open the input file and get the time cdf = Dataset(filename) if confM2R.indatatype == 'NORESM': jdref = date2num(datetime(1800, 1, 1), cdf.variables["time"].units, calendar=cdf.variables["time"].calendar) elif confM2R.indatatype == 'NS8KMZ': jdref = date2num(datetime(1948, 1, 1), units="days since 1948-01-01 00:00:00", calendar="standard") elif confM2R.indatatype == 'GLORYS': jdref = date2num(datetime(1948, 1, 1), cdf.variables["time_counter"].units, calendar=cdf.variables["time_counter"].calendar) elif confM2R.indatatype == 'NS8KM': jdref = date2num(datetime(1948, 1, 1), cdf.variables["ocean_time"].units, calendar=cdf.variables["ocean_time"].calendar) elif confM2R.indatatype == 'SODA3': jdref = date2num(datetime(1948, 1, 1), units="days since 1948-01-01 00:00:00", calendar="standard") else: jdref = date2num(datetime(1948, 1, 1), cdf.variables["time"].units, calendar=cdf.variables["time"].calendar) if confM2R.indatatype == 'SODAMONTHLY': # Find the day and month that the SODAMONTHLY file respresents based on the year and ID number. # Each SODA file represents a 1 month average. mycalendar = cdf.variables["time"].calendar myunits = cdf.variables["time"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, myunits, calendar=mycalendar) if confM2R.indatatype == 'SODA3': # Each SODA file represents 12 month averages. myunits = cdf.variables["time"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, units="days since 1948-01-01 00:00:00", calendar="standard") if confM2R.indatatype == 'GLORYS': # Find the day and month that the GLORYS file respresents based on the year and ID number. # Each file represents a 1 month average. mycalendar = cdf.variables["time_counter"].calendar myunits = cdf.variables["time_counter"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, myunits, calendar=mycalendar) if confM2R.indatatype == 'NS8KM': # Find the day and month that the GLORYS file respresents based on the year and ID number. # Each file represents a 1 month average. mycalendar = cdf.variables["ocean_time"].calendar myunits = cdf.variables["ocean_time"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, myunits, calendar=mycalendar) if confM2R.indatatype == 'NS8KMZ': # Find the day and month that the GLORYS file respresents based on the year and ID number. # Each file represents a 1 month average. currentdate = datetime(year, month, day) myunits = cdf.variables["time"].units jd = date2num(currentdate, myunits, calendar="gregorian") print("Days:", jd, currentdate, year, month, day) if confM2R.indatatype == 'NORESM': # Find the day and month that the NORESM file. We need to use the time modules from # netcdf4 for python as they handle calendars that are no_leap. # http://www.esrl.noaa.gov/psd/people/jeffrey.s.whitaker/python/netcdftime.html#datetime mydays = cdf.variables["time"][0] mycalendar = cdf.variables["time"].calendar myunits = cdf.variables["time"].units currentdate = num2date(mydays, units=myunits, calendar=mycalendar) jd = date2num(currentdate, myunits, calendar='noleap') confM2R.grdROMS.time = (jd - jdref) confM2R.grdROMS.reftime = jdref confM2R.grdROMS.timeunits = myunits cdf.close() print("-------------------------------") print('\nCurrent time of %s file : %s' % (confM2R.indatatype, currentdate)) print("-------------------------------")
def date2dec(calendar = 'standard', units=None, excelerr = True, yr=1, mo=1, dy=1, hr=0, mi=0, sc=0, ascii=None, en=None, eng=None): """ Convert scalar and array_like with calendar dates into decimal dates. Supported calendar formats are standard, gregorian, julian, proleptic_gregorian, excel1900, excel1904, 365_day, noleap, 366_day, all_leap, 360_day, decimal, or decimal360. Input is year, month day, hour, minute, second or a combination of them. Input as date string is possible. Output is decimal date with day as unit. Parameters ---------- yr : array_like, optional years (default: 1) mo : array_like, optional month (default: 1) dy : array_like, optional days (default: 1) hr : array_like, optional hours (default: 0) mi : array_like, optional minutes (default: 0) sc : array_like, optional seconds (default: 0) ascii : array_like, optional strings of the format 'dd.mm.yyyy hh:mm:ss'. Missing hour, minutes and/or seconds are set to their default values (0). `ascii` overwrites all other keywords. `ascii` and `eng` are mutually exclusive. en : array_like, optional strings of the format 'yyyy-mm-dd hh:mm:ss'. Missing hour, minutes and/or seconds are set to their default values (0). `en` overwrites all other keywords. `en` and `ascii` are mutually exclusive. eng : array_like, optional Same as en: obsolete. calendar : str, optional Calendar of output dates (default: 'standard'). Possible values are: 'standard', 'gregorian' = julian calendar from 01.01.-4712 12:00:00 (BC) until 05.03.1583 00:00:00 and gregorian calendar from 15.03.1583 00:00:00 until now. Missing 10 days do not exsist. 'julian' = julian calendar from 01.01.-4712 12:00:00 (BC) until now. 'proleptic_gregorian' = gregorian calendar from 01.01.0001 00:00:00 until now. 'excel1900' = Excel dates with origin at 01.01.1900 00:00:00. 'excel1904' = Excel 1904 (Lotus) format. Same as excel1904 but with origin at 01.01.1904 00:00:00. '365_day', 'noleap' = 365 days format, i.e. common years only (no leap years) with origin at 01.01.0001 00:00:00. '366_day', 'all_leap' = 366 days format, i.e. leap years only (no common years) with origin at 01.01.0001 00:00:00. '360_day' = 360 days format, i.e. years with only 360 days (30 days per month) with origin at 01.01.0001 00:00:00. 'decimal' = decimal year instead of decimal days. 'decimal360' = decimal year with a year of 360 days, i.e. 12 month with 30 days each. units : str, optional User set units of output dates. Must be a string in the format 'days since yyyy-mm-dd hh:mm:ss'. Default values are set automatically depending on `calendar`. excelerr : bool, optional In Excel, the year 1900 is normally considered a leap year, which it was not. By default, this error is taken into account if `calendar=='excel1900'` (default: True). 1900 is not considered a leap year if `excelerr==False`. Returns ------- array_like array_like with decimal dates. The type of output will be the same as the input type. Notes ----- Most versions of `datetime` do not support negative years, i.e. Julian days < 1721423.5 = 01.01.0001 00:00:00. There is an issue in `netcdftime` version < 0.9.5 in proleptic_gregorian for dates before year 301: dec2date(date2dec(ascii='01.01.0300 00:00:00', calendar='proleptic_gregorian'), calendar='proleptic_gregorian') [300, 1, 2, 0, 0, 0] dec2date(date2dec(ascii='01.01.0301 00:00:00', calendar='proleptic_gregorian'), calendar='proleptic_gregorian') [301, 1, 1, 0, 0, 0] List input is only supported up to 2 dimensions. Requires `netcdftime.py` from module `netcdftime` available at: http://netcdf4-python.googlecode.com Examples -------- # Some implementations of datetime do not support negative years >>> import datetime >>> if datetime.MINYEAR > 0: ... print('The minimum year in your datetime implementation is ', datetime.MINYEAR) ... print('i.e. it does not support negative years (BC).') The minimum year in your datetime implementation is 1 i.e. it does not support negative years (BC). >>> if datetime.MINYEAR > 0: ... year = np.array([2000, 1810, 1630, 1510, 1271, 619, 2, 1]) ... else: ... year = np.array([2000, 1810, 1630, 1510, 1271, 619, -1579, -4712]) >>> month = np.array([1, 4, 7, 9, 3, 8, 8, 1]) >>> day = np.array([5, 24, 15, 20, 18, 27, 23, 1]) >>> hour = np.array([12, 16, 10, 14, 19, 11, 20, 12]) >>> minute = np.array([30, 15, 20, 35, 41, 8, 3, 0]) >>> second = np.array([15, 10, 40, 50, 34, 37, 41, 0]) >>> decimal = date2dec(calendar = 'standard', yr=year, mo=month, dy=day, hr=hour, mi=minute, sc=second) >>> nn = year.size >>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2])) 2.45154902100694e+06 2.38226217719907e+06 2.31660093101852e+06 2.27284810821759e+06 >>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2])) 2.18536732053241e+06 1.94738596431713e+06 >>> decimal = date2dec(calendar='standard', yr=year, mo=6, dy=15, hr=12, mi=minute, sc=second) >>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2])) 2.45171102100694e+06 2.38231401053241e+06 2.31657101435185e+06 2.27275102488426e+06 >>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2])) 2.18545602886574e+06 1.94731300598380e+06 # ascii input >>> if datetime.MINYEAR > 0: ... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50', ... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.0002 20:03:41', '01.01.0001 12:00:00']) ... else: ... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50', ... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.-1579 20:03:41', '01.01.-4712 12:00:00']) >>> decimal = date2dec(calendar='standard', ascii=a) >>> nn = a.size >>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2])) 2.45154902100694e+06 2.38226217719907e+06 2.31660093101852e+06 2.27284810821759e+06 >>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2])) 2.18536732053241e+06 1.94738596431713e+06 # calendar = 'julian' >>> decimal = date2dec(calendar='julian', ascii=a) >>> print('{:.14e} {:.14e} {:.14e} {:.14e}'.format(*decimal[:nn//2])) 2.45156202100694e+06 2.38227417719907e+06 2.31661093101852e+06 2.27284810821759e+06 >>> print('{:.14e} {:.14e}'.format(*decimal[nn//2:nn-2])) 2.18536732053241e+06 1.94738596431713e+06 # calendar = 'proleptic_gregorian' >>> decimal = date2dec(calendar='proleptic_gregorian', ascii=a) >>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2])) 730123.5210069 660836.6771991 595175.4310185 551412.6082176 >>> print('{:.7f} {:.7f}'.format(*decimal[nn//2:nn-2])) 463934.8205324 225957.4643171 # calendar = 'excel1900' WITH excelerr=True -> 1900 considered as leap year >>> d = np.array(['05.01.2000 12:30:15', '27.05.1950 16:25:10', '13.08.1910 10:40:55', ... '01.03.1900 00:00:00', '29.02.1900 00:00:00', '28.02.1900 00:00:00', ... '01.01.1900 00:00:00']) >>> decimal = date2dec(calendar='excel1900', ascii=d) >>> nn = d.size >>> print('{:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2])) 36530.5210069 18410.6841435 3878.4450810 >>> print('{:.1f} {:.1f} {:.1f} {:.1f}'.format(*decimal[nn//2:])) 61.0 60.0 59.0 1.0 # calendar = 'excel1900' WITH excelerr = False -> 1900 is NO leap year >>> decimal = date2dec(calendar='excel1900', ascii=d, excelerr=False) >>> print('{:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2])) 36529.5210069 18409.6841435 3877.4450810 >>> print('{:.1f} {:.1f} {:.1f} {:.1f}'.format(*decimal[nn//2:])) 60.0 60.0 59.0 1.0 # calendar = 'excel1904' >>> decimal = date2dec(calendar='excel1904', ascii=d[:nn//2]) >>> print('{:.7f} {:.7f} {:.7f}'.format(*decimal[:nn//2])) 35069.5210069 16949.6841435 2417.4450810 # calendar = '365_day' >>> g = np.array(['18.08.1972 12:30:15', '25.10.0986 12:30:15', '28.11.0493 22:20:40', '01.01.0001 00:00:00']) >>> decimal = date2dec(calendar='365_day', ascii=g) >>> nn = g.size >>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn])) 719644.5210069 359822.5210069 179911.9310185 0.0000000 # calendar = '366_day' >>> decimal = date2dec(calendar='366_day', ascii=g) >>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn])) 721616.5210069 360808.5210069 180404.9310185 0.0000000 # 360_day does not work with netcdftime.py version equal or below 0.9.2 # calendar = '360_day' >>> decimal = date2dec(calendar='360_day', ascii=g) >>> print('{:.7f} {:.7f} {:.7f} {:.7f}'.format(*decimal[:nn])) 709787.5210069 354894.5210069 177447.9310185 0.0000000 >>> print('{:.7f}'.format(date2dec(yr=1992, mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal'))) 1992.0685337 >>> print('{:.7f}'.format(date2dec(ascii='26.01.1992 02:00', calendar='decimal360'))) 1992.0696759 >>> print('{:.7f} {:.7f}'.format(*date2dec(ascii=['26.01.1992 02:00','26.01.1992 02:00'], calendar='decimal360'))) 1992.0696759 1992.0696759 >>> print('{:.7f} {:.7f}'.format(*date2dec(yr=[1992,1992], mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360'))) 1992.0696759 1992.0696759 >>> print('{:.7f} {:.7f}'.format(*date2dec(yr=np.array([1992,1992]), mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360'))) 1992.0696759 1992.0696759 >>> decimal = date2dec(ascii=[['26.01.1992 02:00','26.01.1992 02:00'], ... ['26.01.1992 02:00','26.01.1992 02:00'], ... ['26.01.1992 02:00','26.01.1992 02:00']], ... calendar='decimal360') >>> print('{:.7f} {:.7f}'.format(*decimal[0])) 1992.0696759 1992.0696759 >>> print('{:.7f} {:.7f}'.format(*decimal[2])) 1992.0696759 1992.0696759 >>> print((date2dec(ascii='01.03.2003 00:00:00') - date2dec(ascii='01.03.2003')) == 0.) True # en >>> decimal = date2dec(en='1992-01-26 02:00', calendar='decimal360') >>> print('{:.7f}'.format(decimal)) 1992.0696759 >>> decimal = date2dec(eng='1992-01-26 02:00', calendar='decimal360') >>> print('{:.7f}'.format(decimal)) 1992.0696759 History ------- Written Arndt Piayda, Jun 2010 Modified Matthias Cuntz, Feb 2012 - All input can be scalar, list or array, also a mix - Changed checks for easier extension - decimal, decimal360 Matthias Cuntz, Dec 2012 - change unit of proleptic_gregorian from 'days since 0001-01-01 00:00:00' to 'days since 0001-01-00 00:00:00' Matthias Cuntz, Feb 2013 - solved Excel leap year problem. Matthias Cuntz, Feb 2013 - ported to Python 3 Matthias Cuntz, Jul 2013 - ascii/eng without time defaults to 00:00:00 Matthias Cuntz, Oct 2013 - Excel starts at 1 not at 0 Matthias Cuntz, Oct 2013 - units bugs, e.g. 01.01.0001 was substracted if Julian calendar even with units Matthias Cuntz, Nov 2013 - removed remnant of time treatment before time check in eng keyword Matthias Cuntz, Jun 2015 - adapted to new netCDF4/netcdftime (>= v1.0) and datetime (>= Python v2.7.9) Matthias Cuntz, Oct 2015 - call date2num with list instead of single netCDF4.datetime objects Matthias Cuntz, Oct 2016 - netcdftime provided even with netCDF4 > 1.0.0; make mo for months always integer Matthias Cuntz, Nov 2016 - 00, 01, etc. for integers not accepted by Python3 Matthias Cuntz, May 2020 - numpy docstring format Matthias Cuntz, Jul 2020 - en for eng Matthias Cuntz, Jul 2020 - use proleptic_gregorian for Excel dates """ # # Checks calendars = ['standard', 'gregorian', 'julian', 'proleptic_gregorian', 'excel1900', 'excel1904', '365_day', 'noleap', '366_day', 'all_leap', '360_day', 'decimal', 'decimal360'] import netCDF4 as nt try: tst = nt.date2num tst = nt.datetime except: try: import netcdftime as nt if ((nt.__version__ <= '0.9.2') & (calendar == '360_day')): raise ValueError("date2dec error: Your version of netcdftime.py is equal" " or below 0.9.2. The 360_day calendar does not work with" " arrays here. Please download a newer one.") except: import cftime as nt # calendar = calendar.lower() if (calendar not in calendars): raise ValueError("date2dec error: Wrong calendar!" " Choose: "+''.join([i+' ' for i in calendars])) # obsolete eng if (eng is not None): if (en is not None): raise ValueError("date2dec error: 'eng' was succeeded by 'en'. Only one can be given.") else: en = eng # if ascii input is given by user, other input will be neglected # calculation of input size and shape if (ascii is not None) and (en is not None): raise ValueError("date2dec error: 'ascii' and 'en' mutually exclusive") if (ascii is not None): islist = type(ascii) != type(np.array(ascii)) isarr = np.ndim(ascii) if (islist & (isarr > 2)): raise ValueError("date2dec error: ascii input is list > 2D; Use array input") if isarr == 0: ascii = np.array([ascii]) else: ascii = np.array(ascii) insize = ascii.size outsize = insize outshape = ascii.shape asciifl = ascii.flatten() timeobj = np.zeros(insize, dtype=object) # slicing of ascii strings to implement in datetime object. missing times # will be set to 0. yr = np.zeros(insize, dtype=np.int) mo = np.zeros(insize, dtype=np.int) dy = np.zeros(insize, dtype=np.int) hr = np.zeros(insize, dtype=np.int) mi = np.zeros(insize, dtype=np.int) sc = np.zeros(insize, dtype=np.int) for i in range(insize): aa = asciifl[i].split('.') dy[i] = int(aa[0]) mo[i] = int(aa[1]) tail = aa[2].split() yr[i] = int(tail[0]) if len(tail) > 1: tim = tail[1].split(':') hr[i] = int(tim[0]) if len(tim) > 1: mi[i] = int(tim[1]) else: mi[i] = 0 if len(tim) > 2: sc[i] = int(tim[2]) else: sc[i] = 0 else: hr[i] = 0 mi[i] = 0 sc[i] = 0 if ((yr[i]==1900) & (mo[i]==2) & (dy[i]==29)): timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i]) else: timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i], sc[i]) if (en is not None): islist = type(en) != type(np.array(en)) isarr = np.ndim(en) if isarr == 0: en = np.array([en]) else: en = np.array(en) if (islist & (isarr > 2)): raise ValueError("date2dec error: en input is list > 2D; Use array input") en = np.array(en) insize = en.size outsize = insize outshape = en.shape enfl = en.flatten() timeobj = np.zeros(insize, dtype=object) # slicing of en strings to implement in datetime object. missing times # will be set to 0. yr = np.zeros(insize, dtype=np.int) mo = np.zeros(insize, dtype=np.int) dy = np.zeros(insize, dtype=np.int) hr = np.zeros(insize, dtype=np.int) mi = np.zeros(insize, dtype=np.int) sc = np.zeros(insize, dtype=np.int) for i in range(insize): ee = enfl[i].split('-') yr[i] = int(ee[0]) mo[i] = int(ee[1]) tail = ee[2].split() dy[i] = int(tail[0]) if len(tail) > 1: tim = tail[1].split(':') hr[i] = int(tim[0]) if len(tim) > 1: mi[i] = int(tim[1]) else: mi[i] = 0 if len(tim) > 2: sc[i] = int(tim[2]) else: sc[i] = 0 else: hr[i] = 0 mi[i] = 0 sc[i] = 0 if ((yr[i]==1900) & (mo[i]==2) & (dy[i]==29)): timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i]) else: timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i], sc[i]) # if no ascii input, other inputs will be concidered # calculation of input sizes, shapes and number of axis if (ascii is None) and (en is None): isnlist1 = type(yr) == type(np.array(yr)) isarr1 = np.ndim(yr) if isarr1 == 0: yr = np.array([yr]) else: yr = np.array(yr) isnlist2 = type(mo) == type(np.array(mo)) isarr2 = np.ndim(mo) if isarr2 == 0: mo = np.array([mo], dtype=np.int) else: mo = np.array(mo, dtype=np.int) isnlist3 = type(dy) == type(np.array(dy)) isarr3 = np.ndim(dy) if isarr3 == 0: dy = np.array([dy]) else: dy = np.array(dy) isnlist4 = type(hr) == type(np.array(hr)) isarr4 = np.ndim(hr) if isarr4 == 0: hr = np.array([hr]) else: hr = np.array(hr) isnlist5 = type(mi) == type(np.array(mi)) isarr5 = np.ndim(mi) if isarr5 == 0: mi = np.array([mi]) else: mi = np.array(mi) isnlist6 = type(sc) == type(np.array(sc)) isarr6 = np.ndim(sc) if isarr6 == 0: sc = np.array([sc]) else: sc = np.array(sc) islist = not (isnlist1 | isnlist2 | isnlist3 | isnlist4 | isnlist5 | isnlist6) isarr = isarr1 + isarr2 + isarr3 + isarr4 + isarr5 + isarr6 shapes = [np.shape(yr), np.shape(mo), np.shape(dy), np.shape(hr), np.shape(mi), np.shape(sc)] nyr = np.size(yr) nmo = np.size(mo) ndy = np.size(dy) nhr = np.size(hr) nmi = np.size(mi) nsc = np.size(sc) sizes = [nyr,nmo,ndy,nhr,nmi,nsc] nmax = np.amax(sizes) ii = np.argmax(sizes) outshape = shapes[ii] if (islist & (np.size(outshape) > 2)): raise ValueError("date2dec error: input is list > 2D; Use array input.") if nyr < nmax: if nyr == 1: yr = np.ones(outshape,)*yr else: raise ValueError("date2dec error: size of yr != max input or 1.") if nmo < nmax: if nmo == 1: mo = np.ones(outshape, dtype=np.int)*mo else: raise ValueError("date2dec error: size of mo != max input or 1.") if ndy < nmax: if ndy == 1: dy = np.ones(outshape)*dy else: raise ValueError("date2dec error: size of dy != max input or 1.") if nhr < nmax: if nhr == 1: hr = np.ones(outshape)*hr else: raise ValueError("date2dec error: size of hr != max input or 1.") if nmi < nmax: if nmi == 1: mi = np.ones(outshape)*mi else: raise ValueError("date2dec error: size of mi != max input or 1.") if nsc < nmax: if nsc == 1: sc = np.ones(outshape)*sc else: raise ValueError("date2dec error: size of sc != max input or 1.") indate = [yr, mo, dy, hr, mi, sc] insize = [np.size(i) for i in indate] inshape = [np.shape(i) for i in indate] dimnum = [len(i) for i in inshape] # calculation of maximum input size and maximum number of axis for # reshaping the output indate = [i.flatten() for i in indate] outsize = max(insize) timeobj = np.zeros(outsize, dtype=object) # datetime object is constructed for i in range(outsize): iyr = int(indate[0][i]) imo = int(indate[1][i]) idy = int(indate[2][i]) ihr = int(indate[3][i]) imi = int(indate[4][i]) isc = int(indate[5][i]) if ((iyr==1900) & (imo==2) & (idy==29)): timeobj[i] = nt.datetime(iyr, 3, 1, ihr, imi, isc) else: timeobj[i] = nt.datetime(iyr, imo, idy, ihr, imi, isc) # depending on chosen calendar and optional set of the time units # decimal date is calculated output = np.zeros(outsize) t0 = nt.datetime(1582, 10, 5, 0, 0, 0) t1 = nt.datetime(1582, 10, 15, 0, 0, 0) is121 = True if (min(timeobj)<t0) and (max(timeobj)>=t1) else False if (calendar == 'standard') or (calendar == 'gregorian'): if not units: units = 'days since 0001-01-01 12:00:00' dec0 = 1721424 else: dec0 = 0 if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='gregorian')+dec0 else: output = nt.date2num(timeobj, units, calendar='gregorian')+dec0 elif calendar == 'julian': if not units: units = 'days since 0001-01-01 12:00:00' dec0 = 1721424 else: dec0 = 0 if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='julian')+dec0 else: output = nt.date2num(timeobj, units, calendar='julian')+dec0 elif calendar == 'proleptic_gregorian': if not units: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='proleptic_gregorian') else: output = nt.date2num(timeobj, units, calendar='proleptic_gregorian') elif calendar == 'excel1900': doerr = False if not units: units = 'days since 1899-12-31 00:00:00' if excelerr: doerr = True if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='proleptic_gregorian') else: output = nt.date2num(timeobj, units, calendar='proleptic_gregorian') if doerr: output = np.where(output >= 60., output+1., output) # date2num treats 29.02.1900 as 01.03.1990, i.e. is the same decimal number if np.any((output >= 61.) & (output < 62.)): for i in range(outsize): # if (timeobj[i].year==1900) & (timeobj[i].month==2) & (timeobj[i].day==29): # output[i] -= 1. if (yr[i]==1900) & (mo[i]==2) & (dy[i]==29): output[i] -= 1. elif calendar == 'excel1904': if not units: units = 'days since 1903-12-31 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='proleptic_gregorian') else: output = nt.date2num(timeobj, units, calendar='proleptic_gregorian') elif (calendar == '365_day') or (calendar == 'noleap'): if not units: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='365_day') else: output = nt.date2num(timeobj, units, calendar='365_day') elif (calendar == '366_day') or (calendar == 'all_leap'): if not units: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='366_day') else: output = nt.date2num(timeobj, units, calendar='366_day') elif calendar == '360_day': if not units: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='360_day') else: output = nt.date2num(timeobj, units, calendar='360_day') elif calendar == 'decimal': ntime = np.size(yr) leap = np.array((((yr%4)==0) & ((yr%100)!=0)) | ((yr%400)==0)).astype(np.int) tdy = np.array(dy, dtype=np.float) diy = np.array([ [-9,0, 31, 59, 90,120,151,181,212,243,273,304,334,365], [-9,0, 31, 60, 91,121,152,182,213,244,274,305,335,366] ]) for i in range(ntime): tdy[i] = tdy[i] + np.array(diy[leap[i],mo[i]], dtype=np.float) days_year = 365. output = ( np.array(yr, dtype=np.float) + ((tdy-1.)*24. + np.array(hr, dtype=np.float) + np.array(mi, dtype=np.float)/60. + np.array(sc, dtype=np.float)/3600.) / ((days_year+np.array(leap, dtype=np.float))*24.) ) # for numerical stability, i.e. back and forth transforms output += 1e-08 # 1/3 sec elif calendar == 'decimal360': ntime = np.size(yr) tdy = np.array(dy, dtype=np.float) diy = np.array([-9, 0, 30, 60, 90,120,150,180,210,240,270,300,330,360]) for i in range(ntime): tdy[i] = tdy[i] + np.array(diy[mo[i]], dtype=np.float) days_year = 360. output = ( np.array(yr, dtype=np.float) + ((tdy-1.)*24. + np.array(hr, dtype=np.float) + np.array(mi, dtype=np.float)/60. + np.array(sc, dtype=np.float)/3600.) / (days_year*24.) ) # for numerical stability, i.e. back and forth transforms output += 1e-08 # 1/3 sec else: raise ValueError("date2dec error: calendar not implemented; should have been catched before.") # return of reshaped output output = np.reshape(output, outshape) if isarr == 0: output = np.float(output) else: if islist: ns = np.size(outshape) if ns == 1: output = [i for i in output] else: loutput = [ i for i in output[:,0]] for i in range(np.size(output[:,0])): loutput[i] = list(np.squeeze(output[i,:])) output = loutput return output
import pandas as pd import matplotlib.path as mpath import matplotlib.patches as mpatches from matplotlib.collections import PatchCollection import matplotlib.pyplot as plt import ogr import osr from math import * from scipy.ndimage.filters import gaussian_filter import mpl_util import laplace_filter import cmocean __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2017, 5, 16) __modified__ = datetime(2017, 5, 16) __version__ = "1.0" __status__ = "Development, modified on 16.05.2017" def getPathForPolygon(ring, mymap): codes = [] x = [ring.GetX(j) for j in range(ring.GetPointCount())] y = [ring.GetY(j) for j in range(ring.GetPointCount())] codes += [mpath.Path.MOVETO] + (len(x) - 1) * [mpath.Path.LINETO] pathX, pathY = mymap(x, y) mymappath = mpath.Path(np.column_stack((pathX, pathY)), codes) return mymappath
def plot_histograms(self, save=False, save_dir=None, fancy=True): '''Create an timeseries plot of specific variables. Parameters:: ---------- save = boolean True to save an output plot, False displays on screen. fancy = boolean If set to true uses ggplot style, otherwise default is used. ''' if fancy: matplotlib.style.use('ggplot') # Set the time string format fmt = '%H:%M:%S' # Create figure/axes instance fig, axs = plt.subplots(nrows=4, ncols=3, figsize=(8, 11), \ sharey="row") for ax, fills in zip(axs.flat, self.dts): # Calculate start and end times stime = netCDF4.datetime(fills[0][0], fills[0][1], fills[0][2],\ fills[0][3], fills[0][4], fills[0][5]) etime = netCDF4.datetime(fills[1][0], fills[1][1], fills[1][2],\ fills[1][3], fills[1][4], fills[1][5]) # Make title from these times figtitle = stime.strftime(fmt) + ' - ' + etime.strftime(fmt) ##if self.pcasp[ # Calculate Poisson error from Cai et al. AMT (2013), Equation A4 yerr = self.pcaspf.between_time(stime, etime).sum(axis=0).divide( self.flow3s.between_time(stime, etime).sum(axis=0)**2).pow(0.5).mul( 1. / self.dlog10d) # Calculate the dNd(log10{D}) value for this time period, # sum at each diameter (bin), and replace zero values with NaN dndlogp = self.pcaspf.between_time(stime, etime).sum(axis=0).divide(\ (self.flow3s.between_time(stime, etime).sum(axis=0) * self.dlog10d)) # Create a pandas Series for plotting dndlogpf = pd.Series(data=dndlogp.replace(to_replace=0., value=np.nan), \ index=self.pcasp_bin_mid, name=figtitle) xlabels = self.pcasp_bin_mid.astype('|S5') dndlogpf.plot(ax=ax, logx=True, logy=True, yerr=yerr, legend=False, x=xlabels) ax.set_title(figtitle) ax.set_xlim(left=0.1, right=10.) ax.set_ylim(bottom=0.01, top=10000.) ax.xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter()) ax.yaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter()) ax.set_xlabel(r'D ($\mu$m)') ax.set_ylabel(r"dN/dlog$_{10}$D (cm$^{-3}$)") fig.tight_layout() if save: # Set the output file name fname = "pcasp_timehist_" + os.path.splitext( self.filename)[0] + ".png" outfile = os.sep.join([save_dir, fname]) plt.savefig(outfile, format='png') print "Creating image: " + outfile else: plt.show()
def date2dec(calendar='standard', units=None, excelerr=True, yr=1, mo=1, dy=1, hr=0, mi=0, sc=0, ascii=None, eng=None): """ Converts numpy arrays with calendar date into numpy arrays with decimal date. Supported calendar formats are standard, gregorian, julian, proleptic_gregorian, excel1900, excel1904, 365_day, noleap, 366_day, all_leap, 360_day, decimal, or decimal360 Input is year, month day, hour, minute, second or a combination of them. ASCII input is possible, too. Output is decimal date with day as unit. Requires 'netcdftime.py' from the module netcdftime available at: http://netcdf4-python.googlecode.com Definition ---------- def date2dec(calendar = 'standard', units=None, excelerr = True, yr=1, mo=1, dy=1, hr=0, mi=0, sc=0, ascii=None, eng=None): Input ----- yr -> input array with year mo -> input array with month dy -> input array with day hr -> input array with hour mi -> input array with minute sc -> input array with second ascii -> input array with strings of the format 'dd.mm.yyyy hh:mm:ss'. If hour, minutes and/or seconds are missing, they will be set to 00. If ascii input is chosen by user, other inputs will be neglected. ascii and eng are mutually exclusive. eng -> input array with strings of the format 'yyyy-mm-dd hh:mm:ss'. If hour, minutes and/or seconds are missing, they will be set to 00. If eng input is chosen, other inputs will be neglected. ascii and eng are mutually exclusive. Parameters ---------- calendar -> Input date format. Default value is 'standard'. 'standard', 'gregorian' = Input date is standard format. Input is in julian calendar from 01.01.-4712 12:00:00 (BC) until 05.03.1583 00:00:00 and gregorian calendar from 15.03.1583 00:00:00 until now. Missing 10 days don't exsist. 'julian' = Input date is julian format. Input is in julian calendar from 01.01.-4712 12:00:00 (BC) until now. 'proleptic_gregorian' = Input date is gregorian format. Input is in gregorian calendar from 01.01.0000 00:00:00 until now. 'excel1900' = Input date is excel 1900 format. Input date is excel date with its units at 01.01.1900 00:00:00 until now. 'excel1904' = Input date is excel 1904 (lotus) format. Input date is excel date with its units at 01.01.1904 00:00:00 until now. '365_day', 'noleap' = Input date is 365 days format. Input date consists of common years only (No leap years) with its units at 01.01.0001 00:00:00 until now. '366_day', 'all_leap' = Input date is 366 days format. Input date consists of leap years only (No common years) with its units at 01.01.0001 00:00:00 until now. '360_day' = Input date is 360 days format. Input date consists of years with only 360 days (30 days per month)with its units at 'decimal' = Output is decimal year. 'decimal360' = Output is decimal year with a year of 360 days, i.e. 12 month with 30 days each. Optional Arguments ------------------ units -> Time units can be set by user. Input must be a string in the format 'days since yyyy-mm-dd hh:mm:ss'. Default values are set automatically. excelerr -> In Excel the year 1900 is normally considered as leap year, which is wrong. By default, this error is taken into account (excelerr = True). For excelerr = False, 1900 is considered as no leap year. Output ------ output -> Output numpy array with decimal date. Restrictions ------------ Most versions of datetime do not support neagtive years, i.e. Julian days < 1721423.5 = 01.01.0001 00:00. There is an issue in netcdftime version < 0.9.5 in proleptic_gregorian for dates before year 301: jams.dec2date(jams.date2dec(ascii='01.01.0300 00:00:00', calendar='proleptic_gregorian'), calendar='proleptic_gregorian') [300, 1, 2, 0, 0, 0] jams.dec2date(jams.date2dec(ascii='01.01.0301 00:00:00', calendar='proleptic_gregorian'), calendar='proleptic_gregorian') [301, 1, 1, 0, 0, 0] List input is only supported up to 2 dimensions. Requires 'netcdftime.py' from module netcdftime available at: http://netcdf4-python.googlecode.com Examples -------- #calendar = 'standard' # Some implementations of datetime have problems with negative years >>> import datetime >>> if datetime.MINYEAR > 0: ... print('The minimum year in your datetime implementation is ', datetime.MINYEAR) ... print('i.e. it does not support negative years (BC).') >>> if datetime.MINYEAR > 0: ... year = np.array([2000, 1810, 1630, 1510, 1271, 619, 2, 1]) ... else: ... year = np.array([2000, 1810, 1630, 1510, 1271, 619, -1579, -4712]) >>> month = np.array([1, 4, 7, 9, 3, 8, 8, 1]) >>> day = np.array([5, 24, 15, 20, 18, 27, 23, 1]) >>> hour = np.array([12, 16, 10, 14, 19, 11, 20, 12]) >>> minute = np.array([30, 15, 20, 35, 41, 8, 3, 0]) >>> second = np.array([15, 10, 40, 50, 34, 37, 41, 0]) >>> decimal = date2dec(calendar = 'standard', yr=year, mo=month, dy=day, hr=hour, mi=minute, sc=second) >>> from autostring import astr >>> nn = year.size >>> print(astr(decimal[:nn//2], 14, pp=True)) ['2.45154902100694e+06' '2.38226217719907e+06' '2.31660093101852e+06' '2.27284810821759e+06'] >>> print(astr(decimal[nn//2:nn-2], 14,pp=True)) ['2.18536732053241e+06' '1.94738596431713e+06'] >>> decimal = date2dec(calendar='standard', yr=year, mo=6, dy=15, hr=12, mi=minute, sc=second) >>> print(astr(decimal[:nn//2],14,pp=True)) ['2.45171102100694e+06' '2.38231401053241e+06' '2.31657101435185e+06' '2.27275102488426e+06'] >>> print(astr(decimal[nn//2:nn-2],14,pp=True)) ['2.18545602886574e+06' '1.94731300598380e+06'] # ascii input >>> if datetime.MINYEAR > 0: ... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50', ... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.0002 20:03:41', '01.01.0001 12:00:00']) ... else: ... a = np.array(['05.01.2000 12:30:15', '24.04.1810 16:15:10', '15.07.1630 10:20:40', '20.09.1510 14:35:50', ... '18.03.1271 19:41:34', '27.08. 619 11:08:37', '23.08.-1579 20:03:41', '01.01.-4712 12:00:00']) >>> decimal = date2dec(calendar='standard', ascii=a) >>> nn = a.size >>> print(astr(decimal[:nn//2],14,pp=True)) ['2.45154902100694e+06' '2.38226217719907e+06' '2.31660093101852e+06' '2.27284810821759e+06'] >>> print(astr(decimal[nn//2:nn-2],14,pp=True)) ['2.18536732053241e+06' '1.94738596431713e+06'] # calendar = 'julian' >>> decimal = date2dec(calendar='julian', ascii=a) >>> print(astr(decimal[:nn//2],14,pp=True)) ['2.45156202100694e+06' '2.38227417719907e+06' '2.31661093101852e+06' '2.27284810821759e+06'] >>> print(astr(decimal[nn//2:nn-2],14,pp=True)) ['2.18536732053241e+06' '1.94738596431713e+06'] # calendar = 'proleptic_gregorian' >>> decimal = date2dec(calendar='proleptic_gregorian', ascii=a) >>> print(astr(decimal[:nn//2], 7, pp=True)) ['730123.5210069' '660836.6771991' '595175.4310185' '551412.6082176'] >>> print(astr(decimal[nn//2:nn-2], 7, pp=True)) ['463934.8205324' '225957.4643171'] # calendar = 'excel1900' WITH excelerr=True -> 1900 considered as leap year >>> d = np.array(['05.01.2000 12:30:15', '27.05.1950 16:25:10', '13.08.1910 10:40:55', ... '01.03.1900 00:00:00', '29.02.1900 00:00:00', '28.02.1900 00:00:00', ... '01.01.1900 00:00:00']) >>> decimal = date2dec(calendar='excel1900', ascii=d) >>> nn = d.size >>> print(astr(decimal[:nn//2], 7, pp=True)) ['36530.5210069' '18410.6841435' ' 3878.4450810'] >>> print(astr(decimal[nn//2:],14,pp=True)) ['61.00000000000000' '60.00000000000000' '59.00000000000000' ' 1.00000000000000'] # calendar = 'excel1900' WITH excelerr = False -> 1900 is NO leap year >>> decimal = date2dec(calendar='excel1900', ascii=d, excelerr=False) >>> print(astr(decimal[:nn//2], 7, pp=True)) ['36529.5210069' '18409.6841435' ' 3877.4450810'] >>> print(astr(decimal[nn//2:],14,pp=True)) ['60.00000000000000' '60.00000000000000' '59.00000000000000' ' 1.00000000000000'] # calendar = 'excel1904' >>> decimal = date2dec(calendar='excel1904', ascii=d[:nn//2]) >>> print(astr(decimal[:nn//2], 7, pp=True)) ['35069.5210069' '16949.6841435' ' 2417.4450810'] # calendar = '365_day' >>> g = np.array(['18.08.1972 12:30:15', '25.10.0986 12:30:15', '28.11.0493 22:20:40', '01.01.0001 00:00:00']) >>> decimal = date2dec(calendar='365_day', ascii=g) >>> nn = g.size >>> print(astr(decimal[:nn],14,pp=True)) ['719644.52100694458932' '359822.52100694435649' '179911.93101851851679' ' 0.00000000000000'] # calendar = '366_day' >>> decimal = date2dec(calendar='366_day', ascii=g) >>> print(astr(decimal[:nn],14,pp=True)) ['721616.52100694458932' '360808.52100694435649' '180404.93101851851679' ' 0.00000000000000'] # 360_day does not work with netcdftime.py version equal or below 0.9.2 # calendar = '360_day' >>> decimal = date2dec(calendar='360_day', ascii=g) >>> print(astr(decimal[:nn],14,pp=True)) ['709787.52100694458932' '354894.52100694435649' '177447.93101851851679' ' 0.00000000000000'] >>> print(astr(date2dec(yr=1992, mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal'),14,pp=True)) 1992.06853370763201 >>> print(astr(date2dec(ascii='26.01.1992 02:00', calendar='decimal360'),14,pp=True)) 1992.06967593592572 >>> print(astr(date2dec(ascii=['26.01.1992 02:00','26.01.1992 02:00'], calendar='decimal360'),14,pp=True)) ['1992.06967593592572' '1992.06967593592572'] >>> print(astr(date2dec(yr=[1992,1992], mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360'),14,pp=True)) ['1992.06967593592572' '1992.06967593592572'] >>> print(astr(date2dec(yr=np.array([1992,1992]), mo=1, dy=26, hr=2, mi=0, sc=0, calendar='decimal360'), ... 14,pp=True)) ['1992.06967593592572' '1992.06967593592572'] >>> print(astr(date2dec(ascii=[['26.01.1992 02:00','26.01.1992 02:00'], ... ['26.01.1992 02:00','26.01.1992 02:00'], ... ['26.01.1992 02:00','26.01.1992 02:00']], calendar='decimal360'),14,pp=True)) [['1992.06967593592572' '1992.06967593592572'] ['1992.06967593592572' '1992.06967593592572'] ['1992.06967593592572' '1992.06967593592572']] >>> print((date2dec(ascii='01.03.2003 00:00:00') - date2dec(ascii='01.03.2003')) == 0.) True License ------- This file is part of the JAMS Python package, distributed under the MIT License. The JAMS Python package originates from the former UFZ Python library, Department of Computational Hydrosystems, Helmholtz Centre for Environmental Research - UFZ, Leipzig, Germany. Copyright (c) 2010-2016 Arndt Piayda, Matthias Cuntz - mc (at) macu (dot) de Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. History ------- Written AP, Jun 2010 Modified MC, Feb 2012 - All input can be scalar, list or array, also a mix - Changed checks for easier extension - decimal, decimal360 MC, Dec 2012 - change unit of proleptic_gregorian from 'days since 0001-01-01 00:00:00' to 'days since 0001-01-00 00:00:00' MC, Feb 2013 - solved Excel leap year problem. MC, Feb 2013 - ported to Python 3 MC, Jul 2013 - ascii/eng without time defaults to 00:00:00 MC, Oct 2013 - Excel starts at 1 not at 0 MC, Oct 2013 - units bugs, e.g. 01.01.0001 was substracted if Julian calendar even with units MC, Nov 2013 - removed remnant of time treatment before time check in eng keyword MC, Jun 2015 - adapted to new netCDF4/netcdftime (>= v1.0) and datetime (>= Python v2.7.9) MC, Oct 2015 - call date2num with list instead of single netCDF4.datetime objects MC, Oct 2016 - netcdftime provided even with netCDF4 > 1.0.0; make mo for months always integer MC, Nov 2016 - 00, 01, etc. for integers not accepted by Python3 """ # # Checks calendars = [ 'standard', 'gregorian', 'julian', 'proleptic_gregorian', 'excel1900', 'excel1904', '365_day', 'noleap', '366_day', 'all_leap', '360_day', 'decimal', 'decimal360' ] import netCDF4 as nt try: tst = nt.date2num tst = nt.datetime except: try: import netcdftime as nt if ((nt.__version__ <= '0.9.2') & (calendar == '360_day')): raise ValueError( "date2dec error: Your version of netcdftime.py is equal" " or below 0.9.2. The 360_day calendar does not work with" " arrays here. Please download a newer one.") except: import cftime as nt # calendar = calendar.lower() if (calendar not in calendars): raise ValueError("date2dec error: Wrong calendar!" " Choose: " + ''.join([i + ' ' for i in calendars])) # if ascii input is given by user, other input will be neglected # calculation of input size and shape if (ascii is not None) and (eng is not None): raise ValueError( "date2dec error: 'ascii' and 'eng' mutually exclusive") if ascii is not None: islist = type(ascii) != type(np.array(ascii)) isarr = np.ndim(ascii) if (islist & (isarr > 2)): raise ValueError( "date2dec error: ascii input is list > 2D; Use array input") if isarr == 0: ascii = np.array([ascii]) else: ascii = np.array(ascii) insize = ascii.size outsize = insize outshape = ascii.shape asciifl = ascii.flatten() timeobj = np.zeros(insize, dtype=object) # slicing of ascii strings to implement in datetime object. missing times # will be set to 0. yr = np.zeros(insize, dtype=np.int) mo = np.zeros(insize, dtype=np.int) dy = np.zeros(insize, dtype=np.int) hr = np.zeros(insize, dtype=np.int) mi = np.zeros(insize, dtype=np.int) sc = np.zeros(insize, dtype=np.int) for i in range(insize): aa = asciifl[i].split('.') dy[i] = int(aa[0]) mo[i] = int(aa[1]) tail = aa[2].split() yr[i] = int(tail[0]) if len(tail) > 1: tim = tail[1].split(':') hr[i] = int(tim[0]) if len(tim) > 1: mi[i] = int(tim[1]) else: mi[i] = 0 if len(tim) > 2: sc[i] = int(tim[2]) else: sc[i] = 0 else: hr[i] = 0 mi[i] = 0 sc[i] = 0 if ((yr[i] == 1900) & (mo[i] == 2) & (dy[i] == 29)): timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i]) else: timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i], sc[i]) if eng is not None: islist = type(eng) != type(np.array(eng)) isarr = np.ndim(eng) if isarr == 0: eng = np.array([eng]) else: eng = np.array(eng) if (islist & (isarr > 2)): raise ValueError( "date2dec error: eng input is list > 2D; Use array input") eng = np.array(eng) insize = eng.size outsize = insize outshape = eng.shape engfl = eng.flatten() timeobj = np.zeros(insize, dtype=object) # slicing of eng strings to implement in datetime object. missing times # will be set to 0. yr = np.zeros(insize, dtype=np.int) mo = np.zeros(insize, dtype=np.int) dy = np.zeros(insize, dtype=np.int) hr = np.zeros(insize, dtype=np.int) mi = np.zeros(insize, dtype=np.int) sc = np.zeros(insize, dtype=np.int) for i in range(insize): ee = engfl[i].split('-') yr[i] = int(ee[0]) mo[i] = int(ee[1]) tail = ee[2].split() dy[i] = int(tail[0]) if len(tail) > 1: tim = tail[1].split(':') hr[i] = int(tim[0]) if len(tim) > 1: mi[i] = int(tim[1]) else: mi[i] = 0 if len(tim) > 2: sc[i] = int(tim[2]) else: sc[i] = 0 else: hr[i] = 0 mi[i] = 0 sc[i] = 0 if ((yr[i] == 1900) & (mo[i] == 2) & (dy[i] == 29)): timeobj[i] = nt.datetime(yr[i], 3, 1, hr[i], mi[i], sc[i]) else: timeobj[i] = nt.datetime(yr[i], mo[i], dy[i], hr[i], mi[i], sc[i]) # if no ascii input, other inputs will be concidered # calculation of input sizes, shapes and number of axis if ((ascii is None) and (eng is None)): isnlist1 = type(yr) == type(np.array(yr)) isarr1 = np.ndim(yr) if isarr1 == 0: yr = np.array([yr]) else: yr = np.array(yr) isnlist2 = type(mo) == type(np.array(mo)) isarr2 = np.ndim(mo) if isarr2 == 0: mo = np.array([mo], dtype=np.int) else: mo = np.array(mo, dtype=np.int) isnlist3 = type(dy) == type(np.array(dy)) isarr3 = np.ndim(dy) if isarr3 == 0: dy = np.array([dy]) else: dy = np.array(dy) isnlist4 = type(hr) == type(np.array(hr)) isarr4 = np.ndim(hr) if isarr4 == 0: hr = np.array([hr]) else: hr = np.array(hr) isnlist5 = type(mi) == type(np.array(mi)) isarr5 = np.ndim(mi) if isarr5 == 0: mi = np.array([mi]) else: mi = np.array(mi) isnlist6 = type(sc) == type(np.array(sc)) isarr6 = np.ndim(sc) if isarr6 == 0: sc = np.array([sc]) else: sc = np.array(sc) islist = not (isnlist1 | isnlist2 | isnlist3 | isnlist4 | isnlist5 | isnlist6) isarr = isarr1 + isarr2 + isarr3 + isarr4 + isarr5 + isarr6 shapes = [ np.shape(yr), np.shape(mo), np.shape(dy), np.shape(hr), np.shape(mi), np.shape(sc) ] nyr = np.size(yr) nmo = np.size(mo) ndy = np.size(dy) nhr = np.size(hr) nmi = np.size(mi) nsc = np.size(sc) sizes = [nyr, nmo, ndy, nhr, nmi, nsc] nmax = np.amax(sizes) ii = np.argmax(sizes) outshape = shapes[ii] if (islist & (np.size(outshape) > 2)): raise ValueError( "date2dec error: input is list > 2D; Use array input.") if nyr < nmax: if nyr == 1: yr = np.ones(outshape, ) * yr else: raise ValueError( "date2dec error: size of yr != max input or 1.") if nmo < nmax: if nmo == 1: mo = np.ones(outshape, dtype=np.int) * mo else: raise ValueError( "date2dec error: size of mo != max input or 1.") if ndy < nmax: if ndy == 1: dy = np.ones(outshape) * dy else: raise ValueError( "date2dec error: size of dy != max input or 1.") if nhr < nmax: if nhr == 1: hr = np.ones(outshape) * hr else: raise ValueError( "date2dec error: size of hr != max input or 1.") if nmi < nmax: if nmi == 1: mi = np.ones(outshape) * mi else: raise ValueError( "date2dec error: size of mi != max input or 1.") if nsc < nmax: if nsc == 1: sc = np.ones(outshape) * sc else: raise ValueError( "date2dec error: size of sc != max input or 1.") indate = [yr, mo, dy, hr, mi, sc] insize = [np.size(i) for i in indate] inshape = [np.shape(i) for i in indate] dimnum = [len(i) for i in inshape] # calculation of maximum input size and maximum number of axis for # reshaping the output indate = [i.flatten() for i in indate] outsize = max(insize) timeobj = np.zeros(outsize, dtype=object) # datetime object is constructed for i in range(outsize): iyr = int(indate[0][i]) imo = int(indate[1][i]) idy = int(indate[2][i]) ihr = int(indate[3][i]) imi = int(indate[4][i]) isc = int(indate[5][i]) if ((iyr == 1900) & (imo == 2) & (idy == 29)): timeobj[i] = nt.datetime(iyr, 3, 1, ihr, imi, isc) else: timeobj[i] = nt.datetime(iyr, imo, idy, ihr, imi, isc) # depending on chosen calendar and optional set of the time units # decimal date is calculated output = np.zeros(outsize) t0 = nt.datetime(1582, 10, 5, 0, 0, 0) t1 = nt.datetime(1582, 10, 15, 0, 0, 0) is121 = True if (min(timeobj) < t0) and (max(timeobj) >= t1) else False if (calendar == 'standard') or (calendar == 'gregorian'): if units is None: units = 'days since 0001-01-01 12:00:00' dec0 = 1721424 else: dec0 = 0 if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='gregorian') + dec0 else: output = nt.date2num(timeobj, units, calendar='gregorian') + dec0 elif calendar == 'julian': if units is None: units = 'days since 0001-01-01 12:00:00' dec0 = 1721424 else: dec0 = 0 if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='julian') + dec0 else: output = nt.date2num(timeobj, units, calendar='julian') + dec0 elif calendar == 'proleptic_gregorian': if units is None: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='proleptic_gregorian') else: output = nt.date2num(timeobj, units, calendar='proleptic_gregorian') elif calendar == 'excel1900': doerr = False if units is None: units = 'days since 1899-12-31 00:00:00' if excelerr: doerr = True if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='gregorian') else: output = nt.date2num(timeobj, units, calendar='gregorian') if doerr: output = np.where(output >= 60., output + 1., output) # date2num treats 29.02.1900 as 01.03.1990, i.e. is the same decimal number if np.any((output >= 61.) & (output < 62.)): for i in range(outsize): # if (timeobj[i].year==1900) & (timeobj[i].month==2) & (timeobj[i].day==29): # output[i] -= 1. if (yr[i] == 1900) & (mo[i] == 2) & (dy[i] == 29): output[i] -= 1. elif calendar == 'excel1904': if units is None: units = 'days since 1903-12-31 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='gregorian') else: output = nt.date2num(timeobj, units, calendar='gregorian') elif (calendar == '365_day') or (calendar == 'noleap'): if units is None: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='365_day') else: output = nt.date2num(timeobj, units, calendar='365_day') elif (calendar == '366_day') or (calendar == 'all_leap'): if units is None: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='366_day') else: output = nt.date2num(timeobj, units, calendar='366_day') elif calendar == '360_day': if units is None: units = 'days since 0001-01-01 00:00:00' if is121 and (nt.__version__ < '1.2.2'): for ii, tt in enumerate(timeobj): output[ii] = nt.date2num(tt, units, calendar='360_day') else: output = nt.date2num(timeobj, units, calendar='360_day') elif calendar == 'decimal': ntime = np.size(yr) leap = np.array((((yr % 4) == 0) & ((yr % 100) != 0)) | ((yr % 400) == 0)).astype(np.int) tdy = np.array(dy, dtype=np.float) diy = np.array( [[-9, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365], [-9, 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366]]) for i in range(ntime): tdy[i] = tdy[i] + np.array(diy[leap[i], mo[i]], dtype=np.float) days_year = 365. output = (np.array(yr, dtype=np.float) + ((tdy - 1.) * 24. + np.array(hr, dtype=np.float) + np.array(mi, dtype=np.float) / 60. + np.array(sc, dtype=np.float) / 3600.) / ((days_year + np.array(leap, dtype=np.float)) * 24.)) # for numerical stability, i.e. back and forth transforms output += 1e-08 # 1/3 sec elif calendar == 'decimal360': ntime = np.size(yr) tdy = np.array(dy, dtype=np.float) diy = np.array( [-9, 0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360]) for i in range(ntime): tdy[i] = tdy[i] + np.array(diy[mo[i]], dtype=np.float) days_year = 360. output = (np.array(yr, dtype=np.float) + ((tdy - 1.) * 24. + np.array(hr, dtype=np.float) + np.array(mi, dtype=np.float) / 60. + np.array(sc, dtype=np.float) / 3600.) / (days_year * 24.)) # for numerical stability, i.e. back and forth transforms output += 1e-08 # 1/3 sec else: raise ValueError( "date2dec error: calendar not implemented; should have been catched before." ) # return of reshaped output output = np.reshape(output, outshape) if isarr == 0: output = np.float(output) else: if islist: ns = np.size(outshape) if ns == 1: output = [i for i in output] else: loutput = [i for i in output[:, 0]] for i in range(np.size(output[:, 0])): loutput[i] = list(np.squeeze(output[i, :])) output = loutput return output
import calendar import grd import barotropic import IOinitial import IOsubset import datetimeFunctions try: import ESMF except ImportError: print "Could not find module ESMF" pass __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2008, 8, 15) __modified__ = datetime(2014, 12, 1) __version__ = "1.5" __status__ = "Development, modified on 15.08.2008,01.10.2009,07.01.2010, 15.07.2014, 01.12.2014, 07.08.2015" def VerticalInterpolation(myvar, array1, array2, grdROMS, grdMODEL): outINDEX_ST = (grdROMS.Nlevels, grdROMS.eta_rho, grdROMS.xi_rho) outINDEX_U = (grdROMS.Nlevels, grdROMS.eta_u, grdROMS.xi_u) outINDEX_UBAR = (grdROMS.eta_u, grdROMS.xi_u) outINDEX_V = (grdROMS.Nlevels, grdROMS.eta_v, grdROMS.xi_v) outINDEX_VBAR = (grdROMS.eta_v, grdROMS.xi_v) if myvar in ['salinity', 'temperature']: print 'Start vertical interpolation for %s (dimensions=%s x %s)' % (myvar, grdROMS.xi_rho, grdROMS.eta_rho) outdata = np.empty((outINDEX_ST), dtype=np.float64, order='Fortran')
def getTime(dataPath, indatatype, grdROMS, grdMODEL, year, month, day, mytime, firstRun): """ Create a date object to keep track of Julian dates etc. Also create a reference date starting at 1948/01/01. Go here to check results:http://lena.gsfc.nasa.gov/lenaDEV/html/doy_conv.html """ if indatatype == 'SODA': filename = getSODAfilename(year, month, day, None, dataPath) if indatatype == 'SODAMONTHLY': filename = getSODAMONTHLYfilename(year, month, day, None, dataPath) if indatatype == 'GLORYS': filename = getGLORYSfilename(year, month, day, "S", dataPath) if indatatype == 'WOAMONTHLY': filename = getWOAMONTHLYfilename(year, month, day, "temperature", dataPath) if indatatype == 'NORESM': filename = getNORESMfilename(year, month, day, "saln", dataPath) if indatatype == 'NS8KM': filename = getNS8KMfilename(year, month, day, "salt", dataPath) if indatatype == 'NS8KMZ': filename, readFromOneFile = getNS8KMZfilename(year, month, day, "salt", dataPath) # Now open the input file and get the time cdf = Dataset(filename) if (indatatype)=='NORESM': jdref = date2num(datetime(1800,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar) elif indatatype=='NS8KMZ': jdref = date2num(datetime(1948,1,1),units="days since 1948-01-01 00:00:00",calendar="gregorian") elif (indatatype)=='GLORYS': jdref = date2num(datetime(1948,1,1),cdf.variables["time_counter"].units,calendar=cdf.variables["time_counter"].calendar) elif indatatype=='NS8KM': jdref = date2num(datetime(1948,1,1),cdf.variables["ocean_time"].units,calendar=cdf.variables["ocean_time"].calendar) else: jdref = date2num(datetime(1948,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar) if indatatype == 'SODA': # Find the day and month that the SODA file respresents based on the year and ID number. # Each SODA file represents a 5 day average, therefore we let the date we find be the first day # of those 5 days. Thats the reason we subtract 4 below for day of month. import date days = 0.0; month = 1; loop = True while loop is True: d = date.NumberDaysMonth(month, year) if days + d < int(ID) * 5: days = days + d month += 1 else: day = int(int(ID) * 5 - days) loop = False mycalendar = cdf.variables["time"].calendar myunits = cdf.variables["time"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, myunits, calendar=mycalendar) if indatatype == 'SODAMONTHLY': # Find the day and month that the SODAMONTHLY file respresents based on the year and ID number. # Each SODA file represents a 1 month average. mycalendar = cdf.variables["time"].calendar myunits = cdf.variables["time"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, myunits, calendar=mycalendar) if indatatype == 'GLORYS': # Find the day and month that the GLORYS file respresents based on the year and ID number. # Each file represents a 1 month average. mycalendar = cdf.variables["time_counter"].calendar myunits = cdf.variables["time_counter"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, myunits, calendar=mycalendar) if indatatype == 'NS8KM': # Find the day and month that the GLORYS file respresents based on the year and ID number. # Each file represents a 1 month average. mycalendar = cdf.variables["ocean_time"].calendar myunits = cdf.variables["ocean_time"].units currentdate = datetime(year, month, day) jd = date2num(currentdate, myunits, calendar=mycalendar) if indatatype == 'NS8KMZ': # Find the day and month that the GLORYS file respresents based on the year and ID number. # Each file represents a 1 month average. mycalendar = "gregorian" refdate = datetime(1948, 1, 1) currentdate = datetime(year, month, day) myunits = cdf.variables["time"].units jd = date2num(currentdate, myunits, calendar="gregorian") print "Days:", jd, currentdate," year month day ",year, month, day if indatatype == 'NORESM': # Find the day and month that the NORESM file. We need to use the time modules from # netcdf4 for python as they handle calendars that are no_leap. # http://www.esrl.noaa.gov/psd/people/jeffrey.s.whitaker/python/netcdftime.html#datetime mydays = cdf.variables["time"][0] mycalendar = cdf.variables["time"].calendar myunits = cdf.variables["time"].units # Fake the start date first time around # if (firstRun): # currentdate = datetime(2006,1,1) # print "NOTICE!\n First datestamp in result files are hardcoded to %s"%(currentdate) # else: currentdate = num2date(mydays, units=myunits, calendar=mycalendar) jd = date2num(currentdate, myunits, calendar='noleap') grdROMS.time = (jd - jdref) grdROMS.reftime = jdref grdROMS.timeunits=myunits cdf.close() print '\nCurrent time of %s file : %s' % (indatatype, currentdate)
def main(): start_year = 1979 end_year = 1981 HL_LABEL = "CRCM5_HL" NEMO_LABEL = "CRCM5_NEMO" file_prefix = "dm" level = 1 level_type = level_kinds.HYBRID wind_comp_names = ["UU", "VV"] sim_label_to_path = OrderedDict( [(HL_LABEL, "/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples"), (NEMO_LABEL, "/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples")] ) # get a coord file ... coord_file = "" found_coord_file = False for mdir in os.listdir(sim_label_to_path[HL_LABEL]): mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir) if not os.path.isdir(mdir_path): continue for fn in os.listdir(mdir_path): print(fn) if fn[:2] not in ["pm", "dm", "pp", "dp"]: continue coord_file = os.path.join(mdir_path, fn) found_coord_file = True if found_coord_file: break bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file(path=coord_file) xx, yy = bmp(lons, lats) lons[lons > 180] -= 360 # loop through all files rotate vaectors and save to netcdf for sim_label, samples_dir in sim_label_to_path.items(): samples = Path(samples_dir) po = samples.parent monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map(file_prefix=file_prefix, start_year=start_year, end_year=end_year, samples_dir_path=samples) # Netcdf output file to put rotated winds po /= "rotated_wind_{}.nc".format(sim_label) with Dataset(str(po), "w") as ds: ds.createDimension("time", None) ds.createDimension("lon", lons.shape[0]) ds.createDimension("lat", lons.shape[1]) # create the schema of the output file vname_to_ncvar = {} for vname in wind_comp_names: vname_to_ncvar[vname] = ds.createVariable(vname, "f4", dimensions=("time", "lon", "lat")) vname_to_ncvar[vname].units = "knots" lons_var = ds.createVariable("lon", "f4", dimensions=("lon", "lat")) lats_var = ds.createVariable("lat", "f4", dimensions=("lon", "lat")) time_var = ds.createVariable("time", "i8", dimensions=("time",)) time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format(datetime(start_year, 1, 1)) lons_var[:] = lons lats_var[:] = lats # use sorted dates record_count = 0 for month_date in sorted(monthdate_to_path_list): # select only dm files mr = MultiRPN(path=monthdate_to_path_list[month_date]) vname_to_fields = {} for vname in wind_comp_names: vname_to_fields[vname] = mr.get_all_time_records_for_name_and_level(varname=vname, level=level, level_kind=level_type) for ti, t in enumerate(sorted(vname_to_fields[wind_comp_names[0]])): time_var[record_count] = date2num(t, time_var.units) uu = vname_to_fields[wind_comp_names[0]][t] vv = vname_to_fields[wind_comp_names[1]][t] uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu, vv, lons, lats, bmp=bmp) # in knots not in m/s vname_to_ncvar[wind_comp_names[0]][record_count, :, :] = uu_rot vname_to_ncvar[wind_comp_names[1]][record_count, :, :] = vv_rot record_count += 1
import interpolation as interp import IOwrite import os import barotropic import IOinitial import IOsubset import datetimeFunctions try: import ESMF except ImportError: print("Could not find module ESMF") pass __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2008, 8, 15) __modified__ = datetime(2014, 12, 1) __version__ = "1.5" __status__ = "Development, modified on 15.08.2008,01.10.2009,07.01.2010, 15.07.2014, 01.12.2014, 07.08.2015" def verticalinterpolation(myvar, array1, array2, grdROMS, grdMODEL): outINDEX_ST = (grdROMS.nlevels, grdROMS.eta_rho, grdROMS.xi_rho) outINDEX_U = (grdROMS.nlevels, grdROMS.eta_u, grdROMS.xi_u) outINDEX_UBAR = (grdROMS.eta_u, grdROMS.xi_u) outINDEX_V = (grdROMS.nlevels, grdROMS.eta_v, grdROMS.xi_v) outINDEX_VBAR = (grdROMS.eta_v, grdROMS.xi_v) if myvar in ['salinity', 'temperature']: print('Start vertical interpolation for %s (dimensions=%s x %s)' % (myvar, grdROMS.xi_rho, grdROMS.eta_rho))
def main(): start_year = 1979 end_year = 1981 HL_LABEL = "CRCM5_HL" NEMO_LABEL = "CRCM5_NEMO" file_prefix = "dm" level = 1 level_type = level_kinds.HYBRID wind_comp_names = ["UU", "VV"] sim_label_to_path = OrderedDict([ (HL_LABEL, "/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples" ), (NEMO_LABEL, "/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples" ) ]) # get a coord file ... coord_file = "" found_coord_file = False for mdir in os.listdir(sim_label_to_path[HL_LABEL]): mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir) if not os.path.isdir(mdir_path): continue for fn in os.listdir(mdir_path): print(fn) if fn[:2] not in ["pm", "dm", "pp", "dp"]: continue coord_file = os.path.join(mdir_path, fn) found_coord_file = True if found_coord_file: break bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file( path=coord_file) xx, yy = bmp(lons, lats) lons[lons > 180] -= 360 # loop through all files rotate vaectors and save to netcdf for sim_label, samples_dir in sim_label_to_path.items(): samples = Path(samples_dir) po = samples.parent monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map( file_prefix=file_prefix, start_year=start_year, end_year=end_year, samples_dir_path=samples) # Netcdf output file to put rotated winds po /= "rotated_wind_{}.nc".format(sim_label) with Dataset(str(po), "w") as ds: ds.createDimension("time", None) ds.createDimension("lon", lons.shape[0]) ds.createDimension("lat", lons.shape[1]) # create the schema of the output file vname_to_ncvar = {} for vname in wind_comp_names: vname_to_ncvar[vname] = ds.createVariable(vname, "f4", dimensions=("time", "lon", "lat")) vname_to_ncvar[vname].units = "knots" lons_var = ds.createVariable("lon", "f4", dimensions=("lon", "lat")) lats_var = ds.createVariable("lat", "f4", dimensions=("lon", "lat")) time_var = ds.createVariable("time", "i8", dimensions=("time", )) time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format( datetime(start_year, 1, 1)) lons_var[:] = lons lats_var[:] = lats # use sorted dates record_count = 0 for month_date in sorted(monthdate_to_path_list): # select only dm files mr = MultiRPN(path=monthdate_to_path_list[month_date]) vname_to_fields = {} for vname in wind_comp_names: vname_to_fields[ vname] = mr.get_all_time_records_for_name_and_level( varname=vname, level=level, level_kind=level_type) for ti, t in enumerate( sorted(vname_to_fields[wind_comp_names[0]])): time_var[record_count] = date2num(t, time_var.units) uu = vname_to_fields[wind_comp_names[0]][t] vv = vname_to_fields[wind_comp_names[1]][t] uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu, vv, lons, lats, bmp=bmp) # in knots not in m/s vname_to_ncvar[wind_comp_names[0]][ record_count, :, :] = uu_rot vname_to_ncvar[wind_comp_names[1]][ record_count, :, :] = vv_rot record_count += 1
def get3ddata(confM2R, myvar, year, month, day): if myvar == 'temperature': varN = 0 if myvar == 'salinity': varN = 1 if myvar == 'uvel': varN = 3 if myvar == 'vvel': varN = 4 # The variable splitExtract is defined in IOsubset.py and depends on the orientation # and indatatype of grid (-180-180 or 0-360). Assumes regular grid. if confM2R.useesmf: if confM2R.indatatype == "SODA": filename = getSODAfilename(confM2R, year, month, None) cdf = Dataset(filename) data = cdf.variables[confM2R.inputdatavarnames[varN]][0, :, :, :] if confM2R.indatatype == "SODA3": filename = getSODA3filename(confM2R, year, month, confM2R.inputdatavarnames[varN]) cdf = Dataset(filename) print("=>Extracting data for month %s from SODA3 %s " % (month - 1, filename)) data = cdf.variables[confM2R.inputdatavarnames[varN]][month - 1, :, :, :] if confM2R.indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(confM2R, year, month, confM2R.inputdatavarnames[varN]) cdf = Dataset(filename) data = cdf.variables[str(confM2R.inputdatavarnames[varN])][:, :, :] if confM2R.indatatype == "WOAMONTHLY": filename = getWOAMONTHLYfilename(confM2R, year, month, confM2R.inputdatavarnames[varN]) cdf = Dataset(filename) data = cdf.variables[str( confM2R.inputdatavarnames[varN])][month - 1, :, :, :] if confM2R.indatatype == "NORESM": cdf = Dataset( getNORESMfilename(confM2R, year, month, confM2R.inputdatavarnames[varN])) myunits = cdf.variables[str(confM2R.inputdatavarnames[varN])].units data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][0, :, :, :]) data = np.where(data.mask, confM2R.grdROMS.fillval, data) print("Data range", np.min(data), np.max(data)) if confM2R.indatatype == "NS8KMZ": filename, readFromOneFile = getNS8KMZfilename( confM2R, year, month, confM2R.inputdatavarnames[varN]) cdf = Dataset(filename) print("Reading from one file %s" % (readFromOneFile)) myunits = cdf.variables[str(confM2R.inputdatavarnames[varN])].units if (readFromOneFile): currentdate = datetime(year, month, day, 12) jd = date2num(currentdate, cdf.variables["time"].units, calendar="gregorian") print("Days:", jd, currentdate) timesteps = (cdf.variables["time"][:]).tolist() timeindex = timesteps.index(jd) data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][timeindex, :, :, :]) else: data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][0, :, :, :]) print("Range of data", varN, np.min(data), np.max(data)) data = np.where(data.mask, confM2R.grdROMS.fillval, data) if confM2R.indatatype == "GLORYS": cdf = Dataset( getGLORYSfilename(confM2R, year, month, confM2R.inputdatavarnames[varN])) myunits = cdf.variables[str(confM2R.inputdatavarnames[varN])].units data = np.squeeze(cdf.variables[str( confM2R.inputdatavarnames[varN])][0, :, :, :]) data = np.where(data.mask, confM2R.grdROMS.fillval, data) cdf.close() if myvar == 'temperature' and confM2R.indatatype in [ "NS8KMZ", "GLORYS", "NORESM" ]: if myunits == "degree_Kelvin" or myunits == "K": if confM2R.indatatype in ["GLORYS"]: data = np.where(data <= -32.767, confM2R.grdROMS.fillval, data) data = data - 273.15 if confM2R.indatatype == "GLORYS": data = np.where(data <= -32.767, confM2R.grdROMS.fillval, data) data = np.ma.masked_where(data <= confM2R.grdROMS.fillval, data) if __debug__: print( "Data range of %s just after extracting from netcdf file: %s - %s" % (str(confM2R.inputdatavarnames[varN]), data.min(), data.max())) return data
def main(in_dir="/RESCUE/skynet3_rech1/huziy/anusplin_links", out_dir="/HOME/huziy/skynet3_rech1/hail/anusplin_ts"): out_dir_p = Path(out_dir) in_dir_p = Path(in_dir) lon0 = -114.0708 lat0 = 51.0486 vname = "daily_precipitation_accumulation" vname_alternatives = ["daily_accumulation_precipitation"] vname_alternatives.append(vname) var_list = [vname] fname_hint = "pcp" spatial_ind = None varname_to_list_of_frames = {vname: [] for vname in var_list} for fin in in_dir_p.iterdir(): if fin.name.lower().endswith("ret"): continue if fin.name.lower().endswith("verif"): continue if fname_hint not in fin.name.lower(): continue if not fin.name.endswith(".nc"): continue print(fin) year, month = get_ym_from_path(fin) with Dataset(str(fin)) as ds: if spatial_ind is None: lons, lats = ds.variables["lon"][:], ds.variables["lat"][:] x, y, z = lat_lon.lon_lat_to_cartesian(lons.flatten(), lats.flatten()) ktree = KDTree(list(zip(x, y, z))) x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon0, lat0) dist, spatial_ind = ktree.query((x0, y0, z0)) for vname_alt in vname_alternatives: try: values = ds[vname_alt][:] values = [field.flatten()[spatial_ind] for field in values] break except IndexError as ierr: pass dates = [datetime(year, month, int(d)) for d in ds["time"][:]] varname_to_list_of_frames[vname].append( pd.DataFrame(index=dates, data=values)) for vname in var_list: df = pd.concat(varname_to_list_of_frames[vname]) assert isinstance(df, pd.DataFrame) df.sort_index(inplace=True) df.to_csv(str(out_dir_p.joinpath("{}.csv".format(vname))), float_format="%.3f", index_label="Time")
def __init__(self): print('\n--------------------------\n') print('Started ' + time.ctime(time.time())) # EDIT =================================================================== # Set show_progress to "False" if you do not want to see the progress # indicator for horizontal interpolation. self.showprogress = True # Set compileAll to True if you want automatic re-compilation of all the # fortran files necessary to run model2roms. Options are "gfortran" or "ifort". Edit # compile.py to add other Fortran compilers. self.compileall = False # Extract time-series of data for given longitude/latitude self.extractstations = False # Define a set of longitude/latitude positions with names to extract into # station files (using extractStations) if self.extractstations: # stationNames = ['NorthSea', 'Iceland', 'EastandWestGreenland', 'Lofoten', 'Georges Bank'] # lonlist = [2.4301, -22.6001, -47.0801, 13.3801, -67.2001] # latlist = [54.5601, 63.7010, 60.4201, 67.5001, 41.6423] self.stationnames = ["Ytre Utsira", "Indre Utsira", "Lista"] self.latlist = [59.316667, 59.316667, 58.016667] self.lonliost = [4.800000, 4.983333, 6.533333] # Create the bry, init, and clim files for a given grid and input data self.createoceanforcing = True # Create atmospheric forcing for the given grid self.createatmosforcing = False # currently in beta stages and unavailable # Create a smaller resolution grid based on your original. Decimates every second for # each time run self.decimategridfile = False # Write ice values to file (for Arctic regions) self.writeice = True # Use ESMF for the interpolation. This requires that you have ESMF and ESMPy installed (import ESMF) self.useesmf = True # Apply filter to smooth the 2D fields after interpolation (time consuming but enhances results) self.usefilter = True # Format to write the ouput to: 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', or 'NETCDF3_CLASSIC' # Using NETCDF4 automatically turns on compression of files (ZLIB) self.myformat = 'NETCDF4' self.myzlib = True # Frequency of the input data: usually monthly self.timefrequencyofinputdata = "month" # , "month", "hour" # IN GRIDTYPES ------------------------------------------------------------------------------ # Define what grid type you wnat to interpolate from (input MODEL data) # Options: # 1. SODA, 2. SODAMONTHLY, 3.WOAMONTHLY, 4. NORESM, 4. GLORYS, 5. SODA3 self.indatatype = 'SODA3' # Define what grid type you wnat to interpolate from: Can be Z for SIGMA for ROMS # vertical coordinate system or ZLEVEL. also define the name of the dimensions in the input files. # Options: # 1. SIGMA (not prpoerly implemented yet), 2. ZLEVEL self.ingridtype = "SIGMA" # Define the names of ythe geographical variables in the input files self.grdType = 'regular' self.lonName = "longitude" self.latName = "latitude" self.depthName = "depth" self.fill_value = -1.e+20 # OUT GRIDTYPES ------------------------------------------------------------------------------ # Define what grid type you wnat to interpolate to # Options: This is just the name of your grid used to identify your selection later self.outgrid = "ROHO800" self.outgridtype = "ROMS" # Subset input data. If you have global data you may want to seubset these to speed up reading. Make # sure that your input data are cartesian (0-360 or -180:180, -90:90) self.subsetindata = False if self.subsetindata: self.subset = self.definesubsetforindata() # Define nmber of output depth levels self.nlevels = 40 # Define the grid stretching properties (leave default if uncertain what to pick) self.vstretching = 4 self.vtransform = 2 self.theta_s = 7.0 self.theta_b = 0.1 self.tcline = 250.0 self.hc = 250 # PATH TO FORCINGDATA -------------------------------------------------------------------- # Define the path to the input data self.modelpath = self.defineforcingdatapath() print("model path ", self.modelpath) # PATH TO GRID ----------------------------------------------------------------------------- # Define the path to the grid file self.romsgridpath = self.defineromsgridpath() # Climatology is only monthly and model2roms needs to know this self.isclimatology = True if self.indatatype == 'WOAMONTHLY' else False # DATE AND TIME DETAILS --------------------------------------------------------- # Define the period to create forcing for self.start_year = 2013 self.end_year = 2013 self.start_month = 1 self.end_month = 12 self.start_day = 1 self.end_day = 1 if int(calendar.monthrange(self.start_year, self.start_month)[1]) < self.start_day: self.start_day = int( calendar.monthrange(self.start_year, self.start_month)[1]) if int(calendar.monthrange(self.end_year, self.end_month)[1]) < self.end_day: self.end_day = int( calendar.monthrange(self.end_year, self.end_month)[1]) print "hello, startdate here" self.startdate = datetime(self.start_year, self.start_month, self.start_day) self.enddate = datetime(self.end_year, self.end_month, self.end_day) self.years = [ self.start_year + year for year in range(self.end_year + 1 - self.start_year) ] # DEFINE VARIABLE NAMES --------------------------------------------------------- # Define what and name of variables to include in the forcing files # -> myvars is the name model2roms uses to identify variables # -> varNames is the name of the variable found in the NetCDF input files self.globalvarnames = self.defineglobalvarnames() self.inputdatavarnames = self.defineinputdatavarnames() # NO EDIT BELOW ==================================================================================================== if self.compileall is True: import compile compile.compileallgfortran() if self.createatmosforcing or self.createoceanforcing: self.abbreviation = self.defineabbreviation() self.climname, self.initname, self.bryname = self.defineoutputfilenames( ) if self.isclimatology is True: self.climname = self.abbreviation + '_' + str( self.indatatype) + '_climatology.nc' self.showinfo() if self.useesmf: import ESMF print("Starting logfile for ESMF") manager = ESMF.Manager(debug=True) # Create the grid object for the output grid self.grdROMS = grd.Grd("ROMS", self.outgridtype, self.useesmf, 'ocean', self.outgrid) self.grdROMS.Nlevels = self.nlevels self.grdROMS.vstretching = self.vstretching self.grdROMS.vtransform = self.vtransform self.grdROMS.theta_s = self.theta_s self.grdROMS.theta_b = self.theta_b self.grdROMS.Tcline = self.tcline self.grdROMS.hc = self.hc self.grdROMS.vars = self.inputdatavarnames self.grdROMS.varNames = self.globalvarnames self.grdROMS.lonName = 'lon_rho' self.grdROMS.latName = 'lat_rho' self.grdROMS.openNetCDF(self.romsgridpath) self.grdROMS.createObject() self.grdROMS.getDims() # Create the grid object for the input grid self.grdMODEL = grd.Grd("FORCINGDATA", self.outgridtype, self.useesmf, 'ocean', self.outgrid) self.grdMODEL.grdType = self.grdType self.grdMODEL.lonName = self.lonName self.grdMODEL.latName = self.latName self.grdMODEL.depthName = self.depthName self.grdMODEL.fill_value = self.fill_value #if self.createoceanforcing: # model2roms.convertMODEL2ROMS() # clim2bry.writeBry(grdROMS, start_year, bryName, climname, writeice, indatatype, myformat) #if self.createAtmosForcing: # atmosForcing.createAtmosFileUV(grdROMS, modelpath, atmospath, startdate, enddate, useESMF, # myformat, abbreviation, indatatype, gridtype, show_progress) if self.decimategridfile: decimateGrid.createGrid( grdROMS, "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015.nc", "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015v2.nc", 2) if self.extractstations: print( "Running in station mode and extracting pre-defined station locations" ) IOstation.getStationData(years, IDS, modelpath, latlist, lonlist, stationNames) print('Finished ' + time.ctime(time.time()))
def selseasyear(year, seas, t_coord, t_units, t_cal): if seas == ['ANN']: ti = netcdf.date2num(netcdf.datetime(year, 1, 1), t_units, t_cal) if t_cal == '360_day': tf = netcdf.date2num(netcdf.datetime(year, 12, 30), t_units, t_cal) else: tf = netcdf.date2num(netcdf.datetime(year, 12, 31), t_units, t_cal) if seas == ['DJF']: ti = netcdf.date2num(netcdf.datetime(year - 1, 12, 1), t_units, t_cal) if t_cal == '360_day': tf = netcdf.date2num(netcdf.datetime(year, 2, 30), t_units, t_cal) else: tf = netcdf.date2num(netcdf.datetime(year, 2, 28), t_units, t_cal) if seas == ['MAM']: ti = netcdf.date2num(netcdf.datetime(year, 3, 1), t_units, t_cal) if t_cal == '360_day': tf = netcdf.date2num(netcdf.datetime(year, 5, 30), t_units, t_cal) else: tf = netcdf.date2num(netcdf.datetime(year, 5, 31), t_units, t_cal) if seas == ['JJA']: ti = netcdf.date2num(netcdf.datetime(year, 6, 1), t_units, t_cal) if t_cal == '360_day': tf = netcdf.date2num(netcdf.datetime(year, 8, 30), t_units, t_cal) else: tf = netcdf.date2num(netcdf.datetime(year, 8, 31), t_units, t_cal) if seas == ['SON']: ti = netcdf.date2num(netcdf.datetime(year, 9, 1), t_units, t_cal) tf = netcdf.date2num(netcdf.datetime(year, 11, 30), t_units, t_cal) ind_t = np.where(np.bitwise_and(t_coord >= ti, t_coord <= tf + 1)) return ind_t
import time, calendar from netCDF4 import Dataset, datetime, date2num, num2date import model2roms import IOstation import clim2bry import decimateGrid import grd import numpy as np import atmosForcing __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2009, 1, 30) __modified__ = datetime(2016, 4, 21) __version__ = "1.5" __status__ = "Development" def myhelp(): """ This program is run by typing: python main.py in the command window. """ def defineSubsetForIndata(): # Subset the input data. The more you subset the less memory is needed for calculations # and the faster the process is performed. The subset is initially performed in IOsubset.py if gridtype == "NS8KM": abbreviation = "nordsjoen_8km" minLat = 40 maxLat = 70
'/dmf2/scenario/external_data/cccma/CanESM2_large_ensemble/historical-' + f + '/day/seaIce/sic/' + m + 'i1p1/sic_day_CanESM2_historical-' + f + '_' + m + 'i1p1_' + yi2 + '0101-' + yf2 + '1231.nc', 'r') # concatenating on time axis, selecting data for 20-year periods and plotting maps time1 = nc1.variables['time'] time2 = nc2.variables['time'] lat = nc1.variables['lat'][32:] lon = nc1.variables['lon'][:] sic1 = nc1.variables['sic'][:, 32:, :] sic2 = nc2.variables['sic'][:, 32:, :] fig1 = plt.figure(figsize=(7.2, 5)) for i in range(len(yi)): ti = netcdf.date2num(netcdf.datetime(yi[i], 1, 1), time1.units, time1.calendar) tf = netcdf.date2num(netcdf.datetime(yf[i], 1, 1), time1.units, time1.calendar) if i <= 1: ind_t = np.where( np.bitwise_and(time1[:] >= ti, time1[:] <= tf + 1)) sie_tmp = sic1[ind_t[0], :, :] sie = np.mean(sie_tmp, 0) else: ind_t = np.where( np.bitwise_and(time2[:] >= ti, time2[:] <= tf + 1)) sie_tmp = sic2[ind_t[0], :, :] sie = np.mean(sie_tmp, 0) im = fct_pt.map_contourfAR(fig1, subplt[i], sie, lon, lat,
from netCDF4 import Dataset, datetime, date2num,num2date import os import calendar __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2015, 8, 11) __modified__ = datetime(2015, 8, 11) __version__ = "1.5" __status__ = "Development, modified on 11.08.2015" # Methods for returning list of months and days for the given time-step. def createListOfMonths(currentyear,startdate,enddate,isClimatology): print currentyear, startdate.year, enddate.year if currentyear == startdate.year: IDS=[startdate.month+m for m in xrange(13-startdate.month)] # months from start month to end of first year elif currentyear == enddate.year: IDS=[1+m for m in xrange(int(enddate.month))] # months from first month to last month of last year elif startdate.year < currentyear < enddate.year: # months from first month to last month of last year IDS = [1+m for m in xrange(12)] if startdate.year == enddate.year: # months from first month to last month of last year IDS = [startdate.month + m for m in xrange(enddate.month - startdate.month)]
def get_fields(path, var_key, wildcards='*', num_members=4, chunks=60): fields_dict = {} # create coordinates domain = path.split('/')[-1] coordn = loadmat(os.path.join(path, 'coords_' + domain + '.mat')) grid_shape = coordn['lon'].shape fields_dict['lon'] = coordn['lon'] fields_dict['lat'] = coordn['lat'] files = sorted(glob(os.path.join(path, wildcards))) # Find unique dates for members ... print('Creating model data: ', len(files), ' files found') members = {} for file in files: data = loadmat(file) for time, var in zip(data['dates'], data[var_key]): if not members.has_key(time): # create empty lists for unseen data.. members[time] = [] # append member to existing ones members[time].append(var) # create output arrays: valid = 0 times = [] for time, values in members.items(): # excluding last value (is an initialization) if len(values) > num_members: check_values = not np.isnan(values).any() and np.ptp(values) < 1e3 if check_values: members[time] = np.array(values[-num_members - 1:-1]) print('processed: ', time, 'shape: ', members[time].shape) # create times: year, month, days_hours = time.split('-') day, hour = days_hours.split('_') time_args = (int(year), int(month), int(day), int(hour.split('.')[0])) times.append(datetime(*time_args)) else: print('discarded: ', time, 'var contains nans: ', np.max(members.pop(time))) else: print('discarded: ', time, 'shape: ', np.shape(members.pop(time))) # store dates dates = date2num(times, time_units) # index for ordered dates ind = np.argsort(dates) # stack arrays and send members to last dimension (as channels) dates = dates[ind] values = np.stack(members.values()).transpose([0, 2, 3, 1])[ind] if var_key is 'rain': # create actual 3h acumm variables time_s = dates[0] value_s = values[0] fields_dict['time'] = [] fields_dict['values'] = [] for time, value in zip(dates[1:], values[1:]): if time - time_s == 10800: # 3 h print('calculate acummulated rain from: ', time_s, 'to: ', time) fields_dict['time'].append(time) fields_dict['values'].append((value - value_s).clip(min=0.0)) time_s = time value_s = value fields_dict['time'] = np.array(fields_dict['time']) fields_dict['values'] = np.stack(fields_dict['values']) else: fields_dict['time'] = dates fields_dict['values'] = values return fields_dict
import time, calendar from netCDF4 import Dataset, datetime, date2num,num2date import model2roms import IOstation import clim2bry import decimateGrid import grd import numpy as np import atmosForcing __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2009, 1, 30) __modified__ = datetime(2015, 8, 7) __version__ = "1.5" __status__ = "Development" def myhelp(): """ This program is run by typing: python main.py in the command window. """ def defineSubsetForIndata(): # Subset the input data. The more you subset the less memory is needed for calculations # and the faster the process is performed. The subset is initially performed in IOsubset.py if gridtype == "NS8KM": abbreviation = "nordsjoen_8km" minLat = 40 maxLat = 70 minLon = -30
def get3Ddata(grdROMS, grdMODEL, myvar, indatatype, year, month, day, varNames, dataPath): if myvar == 'temperature': varN = 0; if myvar == 'salinity': varN = 1; if myvar == 'uvel': varN = 3; if myvar == 'vvel': varN = 4; # The variable splitExtract is defined in IOsubset.py and depends on the orientation # and indatatype of grid (-180-180 or 0-360). Assumes regular grid. if grdMODEL.useESMF: if indatatype == "SODA": filename = getSODAfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[varNames[varN]][0,:,:,:] if indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][:,:,:] if indatatype == "WOAMONTHLY": filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][month - 1, :,:,:] if indatatype == "NORESM": cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath)) myunits = cdf.variables[str(varNames[varN])].units data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:,:]) data=np.where(data.mask,grdROMS.fill_value,data) # data = np.where(abs(data)>=32768 , grdROMS.fill_value, data) print "Data range", np.min(data),np.max(data) if indatatype == "NS8KMZ": filename, readFromOneFile = getNS8KMZfilename(year, month, day, varNames[varN], dataPath) cdf = Dataset(filename) print "Reading from one file %s"%(readFromOneFile) myunits = cdf.variables[str(varNames[varN])].units if (readFromOneFile): jdref = date2num(datetime(1948,1,1),cdf.variables["time"].units,calendar=cdf.variables["time"].calendar) currentdate = datetime(year, month, day, 12) jd = date2num(currentdate, cdf.variables["time"].units, calendar="gregorian") print "Days:", jd, currentdate timesteps = (cdf.variables["time"][:]).tolist() timeindex = timesteps.index(jd) data = np.squeeze(cdf.variables[str(varNames[varN])][timeindex,:,:,:]) else: data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:,:]) print "Range of data",varN, np.min(data),np.max(data) data=np.where(data.mask,grdROMS.fill_value,data) if indatatype == "GLORYS": cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath)) myunits = cdf.variables[str(varNames[varN])].units data = np.squeeze(cdf.variables[str(varNames[varN])][0,:,:,:]) data=np.where(data.mask,grdROMS.fill_value,data) cdf.close() else: if grdMODEL.splitExtract is True: if indatatype == "SODA": filename = getSODAfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data1 = cdf.variables[varNames[varN]][0, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] data2 = cdf.variables[varNames[varN]][0, :, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])] if indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data1 = cdf.variables[str(varNames[varN])][:, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] data2 = cdf.variables[str(varNames[varN])][:, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])] if indatatype in ["WOAMONTHLY"]: filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath) cdf = Dataset(filename) data1 = cdf.variables[str(varNames[varN])][month - 1, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] data2 = cdf.variables[str(varNames[varN])][month - 1, :, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])] if indatatype == "GLORYS": cdf = Dataset(getGLORYSfilename(year, month, day, varNames[varN], dataPath)) myunits = cdf.variables[str(varNames[varN])].units data1 = np.squeeze(cdf.variables[str(varNames[varN])][0, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) data2 = np.squeeze(cdf.variables[str(varNames[varN])][0, :, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]) if indatatype == "NORESM": cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath)) myunits = cdf.variables[str(varNames[varN])].units data1 = np.squeeze(cdf.variables[str(varNames[varN])][0, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) data2 = np.squeeze(cdf.variables[str(varNames[varN])][0, :, int(grdMODEL.indices[1, 2]):int(grdMODEL.indices[1, 3]), int(grdMODEL.indices[1, 0]):int(grdMODEL.indices[1, 1])]) cdf.close() data = np.concatenate((data1, data2), axis=2) else: if indatatype == "SODA": filename = getSODAfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][0, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] if indatatype == "SODAMONTHLY": filename = getSODAMONTHLYfilename(year, month, day, None, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][:, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] if indatatype == "WOAMONTHLY": filename = getWOAMONTHLYfilename(year, month, day, myvar, dataPath) cdf = Dataset(filename) data = cdf.variables[str(varNames[varN])][month, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])] if indatatype == "GLORYS": if myvar == 'temperature': cdf = Dataset(getGLORYSfilename(year, month, day, 'T', dataPath)) myunits = cdf.variables[str(varNames[varN])].units if myvar == 'salinity': cdf = Dataset(getGLORYSfilename(year, month, day, 'S', dataPath)) if myvar == 'uvel': cdf = Dataset(getGLORYSfilename(year, month, day, 'U', dataPath)) if myvar == 'vvel': cdf = Dataset(getGLORYSfilename(year, month, day, 'V', dataPath)) data = np.squeeze(cdf.variables[str(varNames[varN])][0, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) if indatatype == "NORESM": cdf = Dataset(getNORESMfilename(year, month, day, varNames[varN], dataPath)) myunits = cdf.variables[str(varNames[varN])].units data = np.squeeze(cdf.variables[str(varNames[varN])][0, :, int(grdMODEL.indices[0, 2]):int(grdMODEL.indices[0, 3]), int(grdMODEL.indices[0, 0]):int(grdMODEL.indices[0, 1])]) cdf.close() if myvar == 'temperature' and indatatype in ["NS8KMZ", "GLORYS", "NORESM"]: if myunits == "degree_Kelvin" or myunits == "K": if indatatype in ["GLORYS"]: data = np.where(data <= -32.767, grdROMS.fill_value, data) data = data - 273.15 # if time == 0 and myvar == myvars[0]: # tmp = np.squeeze(data[0, :, :]) #grdMODEL.mask = np.zeros(grdMODEL.lon.shape, dtype=np.float64) #grdMODEL.mask[:, :] = np.where(tmp == grdROMS.fill_value, 1, 0) if indatatype == "GLORYS": data = np.where(data <= -32.767, grdROMS.fill_value, data) data = np.ma.masked_where(data <= grdROMS.fill_value, data) if __debug__: print "Data range of %s just after extracting from netcdf file: %s - %s" % (str(varNames[varN]), data.min(), data.max()) return data
import time, calendar from netCDF4 import Dataset, datetime, date2num, num2date import model2roms import IOstation import clim2bry import decimateGrid import grd import numpy as np import atmosForcing import sys __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2009, 1, 30) __modified__ = datetime(2018, 4, 5) __version__ = "1.5" __status__ = "Development" class Model2romsConfig(object): def definesubsetforindata(self): # Subset the input data. The more you subset the less memory is needed for calculations # and the faster the process is performed. The subset is initially performed in IOsubset.py subset = np.zeros(4) if self.outgrid == "NS8KM": subset[1] = 40 subset[1] = 70 subset[2] = -30 subset[3] = 40
def main(): print '\n--------------------------\n' print 'Started ' + time.ctime(time.time()) # EDIT =================================================================== # Set show_progress to "False" if you do not want to see the progress # indicator for horizontal interpolation. show_progress = True # Set compileAll to True if you want automatic re-compilation of all the # fortran files necessary to run model2roms. Options are "gfortran" or "ifort". Edit # compile.py to add other Fortran compilers. compileAll = False if compileAll is True: import compile; compile.compileAll("ifort") # Extract time-series of data for given longitude/latitude extractStations = False # Define a set of longitude/latitude positions with names to extract into # station files (using extractStations) if (extractStations): stationNames = ['NorthSea', 'Iceland', 'EastandWestGreenland', 'Lofoten', 'Georges Bank'] lonlist = [2.4301, -22.6001, -47.0801, 13.3801, -67.2001] latlist = [54.5601, 63.7010, 60.4201, 67.5001, 41.6423] # Create the bry, init, and clim files for a given grid and input data createOceanForcing = True # Create atmospheric forcing for the given grid createAtmosForcing = False # Create a smaller resolution grid based on your original. Decimates every second for # each time run decimateGridfile = False # Write ice values to file (for Arctic regions) writeIce = False # Use ESMF for the interpolation. This requires that you have ESMF and ESMPy installed (import ESMF) useESMF = True # Apply filter to smooth the 2D fields after interpolation (time consuming) useFilter = True # Format to write the ouput to: 'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', or 'NETCDF3_CLASSIC' # Using NETCDF4 automatically turns on compression of files (ZLIB) myformat='NETCDF4' # Frequency of the input data: usually monthly timeFrequencyOfInputData = "day" #, "month", "hour" # Subset input data. If you have global data you may want to seubset these to speed up reading. Make # sure that your input data are cartesian (0-360 or -180:180, -90:90) subsetIndata = False if subsetIndata: subset = defineSubsetForIndata() # Set the input data MODEL indatatype indatatype = 'SODA' indatatype = 'SODAMONTHLY' indatatype = 'WOAMONTHLY' indatatype = 'NORESM' indatatype = 'GLORYS' #indatatype = 'NS8KM' #indatatype = 'NS8KMZ' # GRIDTYPES ------------------------------------------------------------------------------ # Define what grid type you wnat to interpolate to outgrid = "NS8KM" #outgrid = "REGSCEN" #outgrid = "KINO" # Define what grid type you wnat to interpolate from: Can be Z for SIGMA for ROMS # vertical coordinate system or ZLEVEL ingridtype = "SIGMA" ingridtype = "ZLEVEL" outgridtype="ROMS" # PATH TO DATA ---------------------------------------------------------------------------- # Define the path to the input data if indatatype == 'SODA': modelpath = "/Volumes/MacintoshHD2/Datasets/SODA/" if indatatype == 'SODAMONTHLY': modelpath = "/Volumes/MacintoshHD2/Datasets/SODAMonthly/" if indatatype == 'GLORYS': modelpath = "/Volumes/MacintoshHD2/Datasets/GLOBAL_REANALYSIS_PHYS_001_009/" modelpath = "/Users/trondkr/Projects/is4dvar/GLORYS2V3/" modelpath = "/work/shared/imr/NS8KM/FORCING/GLORYS2V3/ftp.myocean.mercator-ocean.fr/Core/GLOBAL_REANALYSIS_PHYS_001_009/" if indatatype == 'NORESM': modelpath = "/Users/trondkr/Projects/RegScen/NRCP45AERCN_f19_g16_CLE_01/" #modelpath = "/work/users/trondk/REGSCEN/NRCP45AERCN_f19_g16_CLE_01/" #modelpath = "/work/users/trondk/REGSCEN/N20TRAERCN/" if createAtmosForcing: atmospath = "/Users/trondkr/Projects/RegScen/model2roms/TESTFILES/" if indatatype == 'NS8KM': modelpath = "/Users/trondkr/Projects/is4dvar/grid2lonlat/RESULTS/" if indatatype == 'NS8KMZ': modelpath = "/Users/trondkr/Dropbox/deliveryFOFINAL/" modelpath = "/work/shared/imr/NS8KM/deliveryFO/FINAL/" if indatatype == 'WOAMONTHLY': modelpath = "/Users/trondkr/Projects/is4dvar/createSSS/" # PATH TO GRID ----------------------------------------------------------------------------- # Define the path to the grid file if outgrid == "NS8KM": romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_smoothed02022015.nc" romsgridpath = "/work/users/trondk/NS8km/FORCING/GRID/nordsjoen_8km_grid_hmax20m_v3.nc" if outgrid == "KINO": romsgridpath = "/work/users/trondk/KINO/GRID/kino_1600m_07082015_vf20.nc" #romsgridpath = "/Users/trondkr/Projects/KINO/GRID/kino_1600m_07082015_vf20.nc" if outgrid == "REGSCEN": romsgridpath = "/Users/trondkr/Projects/RegScen/Grid/AA_10km_grid_noest.nc" #romsgridpath = "/Users/trondkr/Projects/is4dvar/Grid/nordsjoen_8km_grid_hmax20m_v3.nc" #romsgridpath = "/work/users/trondk/REGSCEN/GRID/AA_10km_grid_noest.nc" if outgrid == "GREENLAND": romsgridpath="/Users/trondkr/Projects/RegScen/Grid/Sermilik_grid_4000m.nc" romsgridpath="/Users/trondkr/Projects/RegScen/model2roms/polarlowr_grid.nc" if indatatype == 'WOAMONTHLY': isClimatology = True else: isClimatology = False # DETAILS ----------------------------------------------------------------------------------- # Define the period to create forcing for start_year = 2012 end_year = 2013 start_month = 11 end_month = 12 start_day = 15 end_day = 15 if (int(calendar.monthrange(start_year, start_month)[1]) < start_day): start_day = int(calendar.monthrange(start_year, start_month)[1]) if (int(calendar.monthrange(end_year, end_month)[1]) < end_day): end_day = int(calendar.monthrange(end_year, end_month)[1]) startdate = datetime(start_year, start_month, start_day) enddate = datetime(end_year, end_month, end_day) years = [start_year+year for year in xrange(end_year+1-start_year)] # Define what and name of variables to include in the forcing files # -> myvars is the name model2roms uses to identify variables # -> varNames is the name of the variable found in the NetCDF input files if indatatype == 'SODA': myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = ['TEMP', 'SALT', 'SSH', 'U', 'V'] if indatatype == 'SODAMONTHLY': myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = ['temp', 'salt', 'ssh', 'u', 'v'] if indatatype == 'NS8KM': myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty'] if indatatype == 'NS8KMZ': fileNameIn, readFromOneFile = model2roms.getNS8KMZfilename(startdate.year, startdate.month, startdate.day, "S", modelpath) myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] if (readFromOneFile): varNames = ['temp', 'salt', 'zeta', 'u_eastward', 'v_northward'] else: varNames = ['votemper', 'vosaline', 'zeta', 'vozocrtx', 'vomecrty'] if indatatype == 'GLORYS': if (writeIce): myvars = ['temperature','salinity', 'ssh', 'uvel', 'vvel','uice','vice','aice','hice'] varNames = ['votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty','iicevelu', 'iicevelv', 'ileadfra', 'iicethic'] else: myvars = ['temperature', 'salinity', 'ssh', 'uvel', 'vvel'] varNames = ['votemper', 'vosaline', 'sossheig', 'vozocrtx', 'vomecrty'] if indatatype == 'WOAMONTHLY': myvars = ['temperature','salinity'] varNames = ['t_an', 's_an'] if indatatype == 'NORESM': myvars = ['temperature','salinity', 'ssh', 'uvel', 'vvel','ageice','uice','vice','aice','hice','snow_thick'] varNames = ['templvl','salnlvl','sealv', 'uvellvl', 'vvellvl','iage', 'uvel', 'vvel', 'aice', 'hi', 'hs'] # NO EDIT BELOW ==================================================================================================== abbreviation = defineAbbreviation(outgrid) climName,initName,bryName = defineOutputFilenames(abbreviation,start_year,end_year,start_month,end_month,start_day,end_day,indatatype) if isClimatology is True: climName=abbreviation + '_' + str(indatatype) + '_climatology.nc' # Create the grid object for the output grid grdROMS = grd.grdClass(romsgridpath, "ROMS", outgridtype, useESMF,'ocean', outgrid) grdROMS.vars=myvars if (useESMF): import ESMF manager = ESMF.Manager(logkind = ESMF.LogKind.MULTI, debug = True) if createOceanForcing: showInfo(myvars, romsgridpath, climName, initName, bryName, start_year, end_year, isClimatology, useESMF, myformat) model2roms.convertMODEL2ROMS(years, startdate, enddate, timeFrequencyOfInputData, climName, initName, modelpath, romsgridpath, myvars, varNames, show_progress, indatatype, outgridtype, isClimatology, writeIce, useESMF, useFilter, myformat, subsetIndata, outgrid, subset=None) clim2bry.writeBry(grdROMS, start_year, bryName, climName, writeIce, indatatype, myformat) if createAtmosForcing: atmosForcing.createAtmosFileUV(grdROMS,modelpath,atmospath,startdate,enddate,useESMF, myformat,abbreviation,indatatype,gridtype,show_progress) #if decimateGridfile: #decimateGrid.createGrid(grdROMS, "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015.nc", "/Users/trondkr/Projects/KINO/GRID/kino_1600m_18072015v2.nc", 2) if extractStations: print "Running in station mode and extracting pre-defined station locations" IOstation.getStationData(years, IDS, modelpath, latlist, lonlist, stationNames) print 'Finished ' + time.ctime(time.time())
ncfile = Dataset('/ninod/baird/cmip5/obs/ERSSTv4/sst.mnmean.v4_invertlat_72x144regrid.nc', 'r', format='NETCDF4') sst_data_orig = ncfile.variables['sst'][:] sst_lat = ncfile.variables['lat'][:] sst_lon = ncfile.variables['lon'][:] global_nlat, global_nlon = sst_data_orig.shape[1:3] global_lat_vals = sst_lat[:] global_lon_vals = sst_lon[:] ########################################################################################## ########################################################################################## ########################################################################################## # (YYYY,MM,DD) hist_start = netCDF4.datetime(1969,12,1) hist_end = netCDF4.datetime(2000,2,28) sst_histmonths_list = [] sst_histclim_list = [] hist_stdevs_list = [] # OPEN HISTORICAL PERIOD PR DATA file_name = '/ninod/baird/cmip5/obs/ERSSTv4/sst.mnmean.v4_invertlat_72x144regrid.nc' ncfile = Dataset(file_name, 'r', format='NETCDF4') sst_hist_data = ncfile.variables['sst'][:,:,:] time_variable = ncfile.variables['time'] date_start = netCDF4.date2num(hist_start, time_variable.units, time_variable.calendar) date_end = netCDF4.date2num(hist_end, time_variable.units, time_variable.calendar)
season_names = ['djf','mam','jja','son','ondjfm'] season='djf' #season='jja' #season='annual' ########################################################################################## ########################################################################################## ########################################################################################## # (YYYY,MM,DD) #hist_start = netCDF4.datetime(1979,1,1) #hist_end = netCDF4.datetime(2009,12,31) hist_start = netCDF4.datetime(1970,1,1) hist_end = netCDF4.datetime(2000,12,31) u200_histmonths_list = [] u200_histclim_list = [] hist_stdevs_list = [] #ncfile = Dataset('/Users/baird/google_drive/_data_original/era_interim/u200_MERRA_monthly_197901-201412_invertlat_96x144regrid.nc', 'r', format='NETCDF4') ncfile = Dataset('/ninod/baird/cmip5/obs/MERRA/u200/MERRA_197901-201412_u200_monthly_2.5x2.5regrid.nc', 'r', format='NETCDF4') u200_hist_data = ncfile.variables['u200'][:] u200_lat = ncfile.variables['lat'][:] u200_lon = ncfile.variables['lon'][:] global_nlat, global_nlon = u200_hist_data.shape[1:3] global_lat_vals = u200_lat[:] global_lon_vals = u200_lon[:]
def selmonthyear(year, month, t_coord, t_units, t_cal): ti = netcdf.date2num(netcdf.datetime(year, month, 1), t_units, t_cal) tf = netcdf.date2num(netcdf.datetime(year, month + 1, 1), t_units, t_cal) ind_t = np.where(np.bitwise_and(t_coord >= ti, t_coord <= tf)) return ind_t
from netCDF4 import Dataset, datetime, date2num,num2date import numpy as np import os from subprocess import call __author__ = 'Trond Kristiansen' __email__ = '*****@*****.**' __created__ = datetime(2017, 2, 1) __modified__ = datetime(2017, 2, 1) __version__ = "1.0" __status__ = "Development, modified on 01.02.2015" myprefix='ws4km' myvars=['Pair','Uwind','Vwind','cloud','Tair','Qair','rain'] years=[year for year in xrange(1989,2015,1)] command = "export SKIP_SAME_TIME=1" call(command,shell=True) for myvar in myvars: newfilename="%s_%s_%s_%s.nc"%(myprefix,myvar,years[0],years[-1]) mylistoffiles=[] if os.path.exists(newfilename): os.remove(newfilename) for year in years: filename="%s_%s_%s.nc"%(myprefix,myvar,year) mylistoffiles.append(filename) if len(mylistoffiles)>0: options="" for f in mylistoffiles: