def ndbc2df(collector, ndbc_id): """ Ugly hack because `collector.raw(responseFormat="text/csv")` Usually times out. """ from netCDF4 import MFDataset, date2index, num2date # FIXME: Only sea_water_temperature for now. if len(collector.variables) > 1: msg = "Expected only 1 variables to download, got {}".format raise ValueError(msg(collector.variables)) if collector.variables[0] == 'sea_water_temperature': columns = 'sea_water_temperature (C)' ncvar = 'sea_surface_temperature' data_type = 'stdmet' # adcp, adcp2, cwind, dart, mmbcur, ocean, oceansites, pwind, # swden, tao-ctd, wlevel, z-hycom else: msg = "Do not know how to download {}".format raise ValueError(msg(collector.variables)) uri = 'http://dods.ndbc.noaa.gov/thredds/dodsC/data/{}'.format(data_type) url = ('%s/%s/' % (uri, ndbc_id)) urls = url_lister(url) filetype = "*.nc" file_list = [filename for filename in fnmatch.filter(urls, filetype)] files = [fname.split('/')[-1] for fname in file_list] urls = ['%s/%s/%s' % (uri, ndbc_id, fname) for fname in files] if not urls: raise Exception("Cannot find data at {!r}".format(url)) nc = MFDataset(urls) kw = dict(calendar='gregorian', select='nearest') time_dim = nc.variables['time'] time = num2date(time_dim[:], units=time_dim.units, calendar=kw['calendar']) idx_start = date2index(collector.start_time.replace(tzinfo=None), time_dim, **kw) idx_stop = date2index(collector.end_time.replace(tzinfo=None), time_dim, **kw) if idx_start == idx_stop: raise Exception("No data within time range" " {!r} and {!r}".format(collector.start_time, collector.end_time)) data = nc.variables[ncvar][idx_start:idx_stop, ...].squeeze() time_dim = nc.variables['time'] time = time[idx_start:idx_stop].squeeze() df = pd.DataFrame(data=data, index=time, columns=[columns]) df.index.name = 'date_time' return df
def get_tindex(t,start_date,end_date,stride=None): tindex = [] tindex.append(date2index(start_date,t,select='before')) tindex.append(date2index(end_date,t,select='after') + 1) if stride is None: tindex.append(1) else: tindex.append(stride) return tindex
def surf_vel(x,y,url,date_mid=datetime.datetime.utcnow(), uvar='u',vvar='v',isurf_layer=0,lonvar='lon',latvar='lat', tvar='time',hours_ave=24,lon360=False,ugrid=False,lonlat_sub=1,time_sub=1): nc=netCDF4.Dataset(url) lon = nc.variables[lonvar][:]-360.*lon360 lat = nc.variables[latvar][:] if ugrid: lon2d=lon lat2d=lat elif lon.ndim==1: # ai and aj are logical arrays, True in subset region igood = np.where((lon>=x.min()) & (lon<=x.max())) jgood = np.where((lat>=y.min()) & (lat<=y.max())) bi=np.arange(igood[0].min(),igood[0].max(),lonlat_sub) bj=np.arange(jgood[0].min(),jgood[0].max(),lonlat_sub) [lon2d,lat2d]=np.meshgrid(lon[bi],lat[bj]) elif lon.ndim==2: igood=np.where(((lon>=x.min())&(lon<=x.max())) & ((lat>=y.min())&(lat<=y.max()))) bj=np.arange(igood[0].min(),igood[0].max(),lonlat_sub) bi=np.arange(igood[1].min(),igood[1].max(),lonlat_sub) lon2d=nc.variables[lonvar][bj,bi] lat2d=nc.variables[latvar][bj,bi] else: print 'uh oh' #desired_stop_date=datetime.datetime(2011,9,9,17,00) # specific time (UTC) desired_stop_date=date_mid+datetime.timedelta(0,3600.*hours_ave/2.) istop = netCDF4.date2index(desired_stop_date,nc.variables[tvar],select='nearest') actual_stop_date=netCDF4.num2date(nc.variables[tvar][istop],nc.variables[tvar].units) actual_date_mid_est=actual_stop_date-datetime.timedelta(0,3600.*hours_ave/2.+3600.*5) start_date=actual_stop_date-datetime.timedelta(0,3600.*hours_ave) istart = netCDF4.date2index(start_date,nc.variables[tvar],select='nearest') print(date_mid.strftime('Requested mid-date: %I:00 %p on %B %d, %Y')) print(actual_date_mid_est.strftime('Returned mid-date (EST): %I:00 %p on %B %d, %Y')) print(start_date.strftime('start: %I:00 %p on %B %d, %Y')) print(actual_stop_date.strftime('stop: %I:00 %p on %B %d, %Y')) if ugrid: u1=np.mean(nc.variables[uvar][istart:istop:time_sub,isurf_layer,:],axis=0) v1=np.mean(nc.variables[vvar][istart:istop:time_sub,isurf_layer,:],axis=0) else: print('reading u...') u1=np.mean(nc.variables[uvar][istart:istop:time_sub,isurf_layer,bj,bi],axis=0) print('reading v...') v1=np.mean(nc.variables[vvar][istart:istop:time_sub,isurf_layer,bj,bi],axis=0) xx2,yy2=np.meshgrid(x,y) ui=scipy.interpolate.griddata((lon2d.flatten(),lat2d.flatten()),u1.flatten(),(xx2,yy2),method='linear',fill_value=0.0) vi=scipy.interpolate.griddata((lon2d.flatten(),lat2d.flatten()),v1.flatten(),(xx2,yy2),method='linear',fill_value=0.0) ui[np.isnan(ui)]=0.0 vi[np.isnan(vi)]=0.0 return ui,vi,actual_date_mid_est
def get_nam_ts(url,vname,start=None,stop=None,j=None,i=None): nc = netCDF4.Dataset(url) ncv = nc.variables time_var = ncv['time'] dtime = netCDF4.num2date(time_var[:],time_var.units) istart = netCDF4.date2index(start,time_var,select='nearest') istop = netCDF4.date2index(stop,time_var,select='nearest') var = ncv[vname] v = var[istart:istop,j,i] tim = dtime[istart:istop] return v,tim
def surf_vel_roms(x,y,url,date_mid=datetime.datetime.utcnow,hours_ave=24,tvar='ocean_time',lonlat_sub=1,time_sub=6): #url = 'http://testbedapps-dev.sura.org/thredds/dodsC/alldata/Shelf_Hypoxia/tamu/roms/tamu_roms.nc' #url='http://tds.ve.ismar.cnr.it:8080/thredds/dodsC/field2_test/run1/his' ##################################################################################### nc = netCDF4.Dataset(url) mask = nc.variables['mask_rho'][:] lon_rho = nc.variables['lon_rho'][:] lat_rho = nc.variables['lat_rho'][:] anglev = nc.variables['angle'][:] desired_stop_date = date_mid+datetime.timedelta(0,3600.*hours_ave/2.) # specific time (UTC) istop = netCDF4.date2index(desired_stop_date,nc.variables[tvar],select='nearest') actual_stop_date=netCDF4.num2date(nc.variables[tvar][istop],nc.variables[tvar].units) start_date=actual_stop_date-datetime.timedelta(0,3600.*hours_ave) istart = netCDF4.date2index(start_date,nc.variables[tvar],select='nearest') uvar='u' vvar='v' isurf_layer = -1 print('reading u...') u=np.mean(nc.variables[uvar][istart:istop:time_sub,isurf_layer,:,:],axis=0) print('reading v...') v=np.mean(nc.variables[vvar][istart:istop:time_sub,isurf_layer,:,:],axis=0) print('done reading data...') u = roms_utils.shrink(u, mask[1:-1, 1:-1].shape) v = roms_utils.shrink(v, mask[1:-1, 1:-1].shape) u, v = roms_utils.rot2d(u, v, anglev[1:-1, 1:-1]) # <codecell> lon=lon_rho[1:-1,1:-1] lat=lat_rho[1:-1,1:-1] # <codecell> # <codecell> xx2,yy2=np.meshgrid(x,y) print('interpolating u to uniform grid...') ui=scipy.interpolate.griddata((lon.flatten(),lat.flatten()),u.flatten(),(xx2,yy2),method='linear',fill_value=0.0) print('interpolating v to uniform grid...') vi=scipy.interpolate.griddata((lon.flatten(),lat.flatten()),v.flatten(),(xx2,yy2),method='linear',fill_value=0.0) ui[np.isnan(ui)]=0.0 vi[np.isnan(vi)]=0.0 return ui,vi
def mean_precip(nc,bbox=None,start=None,stop=None): lon=nc.variables['lon'][:] lat=nc.variables['lat'][:] tindex0=netCDF4.date2index(start,nc.variables['time'],select='nearest') tindex1=netCDF4.date2index(stop,nc.variables['time'],select='nearest') bi=(lon>=box[0])&(lon<=box[2]) bj=(lat>=box[1])&(lat<=box[3]) p=nc.variables['precip_mean'][tindex0:tindex1,bj,bi] latmin=np.min(lat[bj]) p=np.mean(p,axis=0) lon=lon[bi] lat=lat[bj] return p,lon,lat
def test_nonuniform(self): """Check that the fallback mechanism works. """ nutime = self.TestTime(datetime(1950, 1, 1), 366, 24, "hours since 1900-01-01", "standard") # Let's remove the second entry, so that the computed stride is not # representative and the bisection method is needed. nutime._data = nutime._data[numpy.r_[0, slice(2, 200)]] t = date2index(datetime(1950, 2, 1), nutime) assert_equal(t, 30) t = date2index([datetime(1950, 2, 1), datetime(1950, 2, 3)], nutime) assert_equal(t, [30, 32])
def test_select_dummy(self): nutime = self.TestTime(datetime(1950, 1, 1), 366, 24, "hours since 1400-01-01", "standard") dates = [datetime(1950, 1, 2, 6), datetime(1950, 1, 3), datetime(1950, 1, 3, 18)] t = date2index(dates, nutime, select="before") assert_equal(t, [1, 2, 2]) t = date2index(dates, nutime, select="after") assert_equal(t, [2, 2, 3]) t = date2index(dates, nutime, select="nearest") assert_equal(t, [1, 2, 3])
def test_select_dummy(self): nutime = self.TestTime(datetime(1950, 1, 1), 366, 24, 'hours since 1400-01-01', 'standard') dates = [datetime(1950, 1, 2, 6), datetime( 1950, 1, 3), datetime(1950, 1, 3, 18)] t = date2index(dates, nutime, select='before') assert_equal(t, [1, 2, 2]) t = date2index(dates, nutime, select='after') assert_equal(t, [2, 2, 3]) t = date2index(dates, nutime, select='nearest') assert_equal(t, [1, 2, 3])
def netcdf2PCRobj(ncFile,varName,dateInput): # EHS (04 APR 2013): To convert netCDF (tss) file to PCR file. # The cloneMap is globally defined (outside this method). # Get netCDF file and variable name: f = nc.Dataset(ncFile) varName = str(varName) # date date = dateInput if isinstance(date, str) == True: date = \ datetime.datetime.strptime(str(date),'%Y-%m-%d') date = datetime.datetime(date.year,date.month,date.day) # time index (in the netCDF file) nctime = f.variables['time'] # A netCDF time variable object. idx = nc.date2index(date, nctime, calendar=nctime.calendar, \ select='exact') # convert to PCR object and close f outPCR = pcr.numpy2pcr(pcr.Scalar,(f.variables[varName][idx].data), \ float(f.variables[varName]._FillValue)) f.close(); f = None ; del f # PCRaster object return (outPCR)
def runTest(self): # Get the real dates # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this if parse_version(cftime.__version__) >= parse_version('1.0.1'): dates = [] for file in self.files: f = Dataset(file) t = f.variables['time'] dates.extend(num2date(t[:], t.units, t.calendar)) f.close() # Compare with the MF dates f = MFDataset(self.files,check=True) t = f.variables['time'] mfdates = num2date(t[:], t.units, t.calendar) T = MFTime(t) assert_equal(len(T), len(t)) assert_equal(T.shape, t.shape) assert_equal(T.dimensions, t.dimensions) assert_equal(T.typecode(), t.typecode()) # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this if parse_version(cftime.__version__) >= parse_version('1.0.1'): assert_array_equal(num2date(T[:], T.units, T.calendar), dates) assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366) f.close()
def get_time_variable_overlap(self, dates): """Figure out if a new date array has a overlap with the already existing time variable. Return the index of the existing time variable where the new dates should be located. At the moment this only handles cases where all dates are new or none are new. Parameters ---------- dates: list list of datetime objects Returns ------- indexes: np.ndarray Array of indexes that overlap """ timevar = self.dataset.variables[self.time_var] if timevar.size == 0: indexes = np.array([0]) else: try: indexes = netCDF4.date2index( dates, timevar) except ValueError: indexes = np.array([timevar.size]) return indexes
def getFVCOM_bottom_tempsalt_netcdf(lati,loni,starttime,endtime,layer,vname):#vname='temp'or'salinity' ''' Function written by Yacheng Wang generates model data as a DataFrame according to time and local position different from getFVCOM_bottom_temp: this function only return time-temp dataframe and ues netcdf4 getFVCOM_bottom_temp return depth and temp ''' urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3' nc = netCDF4.Dataset(urlfvcom) nc.variables lat = nc.variables['lat'][:] lon = nc.variables['lon'][:] times = nc.variables['time'] jd = netCDF4.num2date(times[:],times.units) var = nc.variables[vname] inode = nearlonlat(lon,lat,loni,lati) modindex=netCDF4.date2index([starttime.replace(tzinfo=None),endtime.replace(tzinfo=None)],times,select='nearest') modtso = pd.DataFrame() # CHUNK THROUGH 'XDAYS' AT A TIME SINCE IT WAS HANGING UP OTHERWISE xdays=100 for k in range(0,(endtime-starttime).days*24,xdays*24): print 'Generating a dataframe of model data requested from '+str(k)+' to ',str(k+xdays*24) #modtso=pd.DataFrame(var[modindex[0]:modindex[1],layer,inode],index=jd[modindex[0]:modindex[1]]) modtso1=pd.DataFrame(var[modindex[0]+k:modindex[0]+k+xdays*24,layer,inode],index=jd[modindex[0]+k:modindex[0]+k+xdays*24]) modtso=pd.concat([modtso,modtso1]) return modtso
def getFVCOM_bottom_tempsaltvel_netcdf(lati,loni,starttime,endtime,layer,vname):#vname='temp'or'salinity' ''' Function written by Yacheng Wang generates model data as a DataFrame according to time and local position different from getFVCOM_bottom_temp: this function only return time-temp dataframe and ues netcdf4 getFVCOM_bottom_temp return depth and temp ''' urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3' nc = netCDF4.Dataset(urlfvcom) nc.variables times = nc.variables['time'] jd = netCDF4.num2date(times[:],times.units) if vname=='vel': u = nc.variables['u'] v = nc.variables['v'] lat = nc.variables['latc'][:] lon = nc.variables['lonc'][:] else: var=nc.variables[vname] lat = nc.variables['lat'][:] lon = nc.variables['lon'][:] inode = nearlonlat(lon,lat,loni,lati) modindex=netCDF4.date2index([starttime.replace(tzinfo=None),endtime.replace(tzinfo=None)],times,select='nearest') print modindex #print [u[modindex[0]:modindex[1],layer,inode][0],v[modindex[0]:modindex[1],layer,inode][0]] if vname=='vel': modtso=pd.DataFrame(np.array([u[modindex[0]:modindex[1],layer,inode],v[modindex[0]:modindex[1],layer,inode]]).T,index=jd[modindex[0]:modindex[1]]) #modtso=pd.DataFrame(np.array([u[modindex[0],layer,inode],v[modindex[0],layer,inode]]).T,index=jd[modindex[0]]) else: modtso=pd.DataFrame(var[modindex[0]:modindex[1],layer,inode],index=jd[modindex[0]:modindex[1]]) return modtso
def get_roms(url, time_slice, n=3): url = parse_url(url) with Dataset(url) as nc: ncv = nc.variables time = ncv['ocean_time'] tidx = date2index(time_slice, time, select='nearest') time = num2date(time[tidx], time.units, time.calendar) mask = ncv['mask_rho'][:] lon_rho = ncv['lon_rho'][:] lat_rho = ncv['lat_rho'][:] anglev = ncv['angle'][:] u = ncv['u'][tidx, -1, ...] v = ncv['v'][tidx, -1, ...] u = shrink(u, mask[1:-1, 1:-1].shape) v = shrink(v, mask[1:-1, 1:-1].shape) u, v = rot2d(u, v, anglev[1:-1, 1:-1]) lon = lon_rho[1:-1, 1:-1] lat = lat_rho[1:-1, 1:-1] u, v = u[::n, ::n], v[::n, ::n] lon, lat = lon[::n, ::n], lat[::n, ::n] u = ma.masked_invalid(u) v = ma.masked_invalid(v) return dict(lon=lon, lat=lat, u=u, v=v, time=time)
def test_singletime(self): # issue 215 test (date2index with time variable length == 1) f = Dataset(self.file) time2 = f.variables['time2'] result_index = date2index(self.first_timestamp, time2, select="exact") assert_equal(result_index, 0) f.close()
def main(O3File, MeteoMask): # # #If a file with data on ozone profiles already contains information on the #height of the tropopause - just updated it, otherwise - create a new variable. FO3 = ncdf.Dataset(O3File,'r+') FMeteo = ncdf.MFDataset(MeteoMask) if not 'HTropo' in FO3.variables: var=FO3.createVariable('HTropo','float32',('Time',), zlib=True, complevel=9, fill_value=np.nan) var.units = 'm.' var.description = 'Height of the tropopause.' else: var=FO3.variables['HTropo'] TimeO3 = FO3.variables['Time'] TimeMet = ncdf.MFTime(FMeteo.variables['Time']) dtTimeO3 = ncdf.num2date(TimeO3, TimeO3.units, TimeO3.calendar) idxMet = ncdf.date2index(dtTimeO3, TimeMet, calendar=TimeMet.calendar, select='nearest') HTropo = FMeteo.variables['HTropo'][idxMet,0] var[...] = HTropo FO3.close() FMeteo.close() return 0
def get_ncfiles_catalog(station_id, jd_start, jd_stop): station_name = station_id.split(":")[-1] uri = 'http://dods.ndbc.noaa.gov/thredds/dodsC/data/stdmet' url = ('%s/%s/' % (uri, station_name)) urls = _url_lister(url) filetype = "*.nc" file_list = [filename for filename in fnmatch.filter(urls, filetype)] files = [fname.split('/')[-1] for fname in file_list] urls = ['%s/%s/%s' % (uri, station_name, fname) for fname in files] try: nc = MFDataset(urls) time_dim = nc.variables['time'] calendar = 'gregorian' idx_start = date2index(jd_start, time_dim, calendar=calendar, select='nearest') idx_stop = date2index(jd_stop, time_dim, calendar=calendar, select='nearest') dir_dim = np.array(nc.variables['wind_dir'][idx_start:idx_stop, 0, 0], dtype=float) speed_dim = np.array(nc.variables['wind_spd'][idx_start:idx_stop, 0, 0]) # Replace fill values with NaN speed_dim[speed_dim == nc.variables['wind_spd']._FillValue] = np.nan if dir_dim.ndim != 1: dir_dim = dir_dim[:, 0] speed_dim = speed_dim[:, 0] time_dim = nc.variables['time'] dates = num2date(time_dim[idx_start:idx_stop], units=time_dim.units, calendar='gregorian').squeeze() mask = np.isfinite(speed_dim) data = dict() data['wind_speed (m/s)'] = speed_dim[mask] data['wind_from_direction (degree)'] = dir_dim[mask] time = dates[mask] # columns = ['wind_speed (m/s)', # 'wind_from_direction (degree)'] df = DataFrame(data=data, index=time) return df except Exception as e: print str(e) df = DataFrame() return df
def test_failure(self): nutime = self.TestTime(datetime(1950, 1, 1), 366, 24, "hours since 1900-01-01", "standard") try: t = date2index(datetime(1949, 2, 1), nutime) except ValueError: pass else: raise ValueError("This test should have failed.")
def findNCDTInd(target, nc): ds = netCDF4.Dataset(nc) time_var = ds.variables['time'] try: ind = netCDF4.date2index(target, time_var) except ValueError: return None, None else: return ds, ind
def netcdf2pcrTimeIdx( self, pcr, forcing ): #ncFile, varName, dateInput, method, factor, x, y, xi, yi, xIdxSta, xIdxEnd, yIdxSta, yIdxEnd): #-read netcdf file f = nc.Dataset(getattr(self, forcing + 'NC')) filecache[getattr(self, forcing + 'NC')] = f #-get index from netcdf corresponding with current date setattr(self, forcing + 'TimeIdx', nc.date2index(self.curdate, f.variables['time'], select='exact'))
def test_issue444(self): # make sure integer overflow not causing error in # calculation of nearest index when sum of adjacent # time values won't fit in 32 bits. ntimes = 20 f = Dataset(self.file, 'r') query_time = datetime(2037, 1, 3, 21, 12) index = date2index(query_time, f.variables['time3'], select='nearest') assert(index == 11) f.close()
def test_issue444(self): # make sure integer overflow not causing error in # calculation of nearest index when sum of adjacent # time values won't fit in 32 bits. ntimes = 20 f = Dataset(self.file, 'r') query_time = datetime(2037, 1, 3, 21, 12) index = date2index(query_time, f.variables['time3'], select='nearest') assert (index == 11) f.close()
def read_ecmwf_macc(filename, latitude, longitude, utc_time_range=None): """ Read data from ECMWF MACC reanalysis netCDF4 file. Parameters ---------- filename : string full path to netCDF4 data file. latitude : float latitude in degrees longitude : float longitude in degrees utc_time_range : sequence of datetime.datetime pair of start and end naive or UTC date-times Returns ------- data : pandas.DataFrame dataframe for specified range of UTC date-times """ ecmwf_macc = ECMWF_MACC(filename) try: ilat, ilon = ecmwf_macc.get_nearest_indices(latitude, longitude) nctime = ecmwf_macc.data['time'] if utc_time_range: start_idx = netCDF4.date2index( utc_time_range[0], nctime, select='before') end_idx = netCDF4.date2index( utc_time_range[-1], nctime, select='after') time_slice = slice(start_idx, end_idx + 1) else: time_slice = slice(0, ecmwf_macc.time_size) times = netCDF4.num2date(nctime[time_slice], nctime.units) df = {k: ecmwf_macc.data[k][time_slice, ilat, ilon] for k in ecmwf_macc.keys} if ECMWF_MACC.TCWV in df: # convert total column water vapor in kg/m^2 at (1-atm, 25-degC) to # precipitable water in cm df['precipitable_water'] = df[ECMWF_MACC.TCWV] / 10.0 finally: ecmwf_macc.data.close() return pd.DataFrame(df, index=times.astype('datetime64[s]'))
def read_ecmwf_macc(filename, latitude, longitude, utc_time_range=None): """ Read data from ECMWF MACC reanalysis netCDF4 file. Parameters ---------- filename : string full path to netCDF4 data file. latitude : float latitude in degrees longitude : float longitude in degrees utc_time_range : sequence of datetime.datetime pair of start and stop naive or UTC date-times Returns ------- data : pandas.DataFrame dataframe for specified range of UTC date-times """ ecmwf_macc = ECMWF_MACC(filename) try: ilat, ilon = ecmwf_macc.get_nearest_indices(latitude, longitude) nctime = ecmwf_macc.data['time'] if utc_time_range: start_idx = netCDF4.date2index( utc_time_range[0], nctime, select='before') stop_idx = netCDF4.date2index( utc_time_range[-1], nctime, select='after') time_slice = slice(start_idx, stop_idx + 1) else: time_slice = slice(0, ecmwf_macc.time_size) times = netCDF4.num2date(nctime[time_slice], nctime.units) df = {k: ecmwf_macc.data[k][time_slice, ilat, ilon] for k in ecmwf_macc.keys} if ECMWF_MACC.TCWV in df: # convert total column water vapor in kg/m^2 at (1-atm, 25-degC) to # precipitable water in cm df['precipitable_water'] = df[ECMWF_MACC.TCWV] / 10.0 finally: ecmwf_macc.data.close() return pd.DataFrame(df, index=times.astype('datetime64[s]'))
def test_select_nc(self): f = Dataset(self.file, 'r') nutime = f.variables['time'] dates = [datetime(1950, 1, 2, 6), datetime( 1950, 1, 3), datetime(1950, 1, 3, 18)] t = date2index(dates, nutime, select='before') assert_equal(t, [1, 2, 2]) t = date2index(dates, nutime, select='after') assert_equal(t, [2, 2, 3]) t = date2index(dates, nutime, select='nearest') assert_equal(t, [1, 2, 3]) # Test dates outside the support with select t = date2index(datetime(1949, 12, 1), nutime, select='nearest') assert_equal(t, 0) t = date2index(datetime(1978, 1, 1), nutime, select='nearest') assert_equal(t, 365) # Test dates outside the support with before self.assertRaises( ValueError, date2index, datetime(1949, 12, 1), nutime, select='before') t = date2index(datetime(1978, 1, 1), nutime, select='before') assert_equal(t, 365) # Test dates outside the support with after t = date2index(datetime(1949, 12, 1), nutime, select='after') assert_equal(t, 0) self.assertRaises( ValueError, date2index, datetime(1978, 1, 1), nutime, select='after') # test microsecond and millisecond units unix_epoch = "milliseconds since 1970-01-01T00:00:00Z" d = datetime(2038, 1, 19, 3, 14, 7) millisecs = int( date2num(d, unix_epoch, calendar='proleptic_gregorian')) assert_equal(millisecs, (2 ** 32 / 2 - 1) * 1000) unix_epoch = "microseconds since 1970-01-01T00:00:00Z" microsecs = int(date2num(d, unix_epoch)) assert_equal(microsecs, (2 ** 32 / 2 - 1) * 1000000) # test microsecond accuracy in date2num/num2date roundtrip # note: microsecond accuracy lost for time intervals greater # than about 270 years. units = 'microseconds since 1776-07-04 00:00:00-12:00' dates =\ [datetime(1962, 10, 27, 6, 1, 30, 9001), datetime( 1993, 11, 21, 12, 5, 25, 999), datetime(1995, 11, 25, 18, 7, 59, 999999)] times2 = date2num(dates, units) dates2 = num2date(times2, units) for date, date2 in zip(dates, dates2): assert_equal(date, date2) f.close()
def get_data(start,ilayer): # get current at layer [0 = surface, -1 = bottom] itime = netCDF4.date2index(start,time_var,select='nearest') dtime = netCDF4.num2date(time_var[itime],time_var.units) daystr = dtime.strftime('%Y-%b-%d %H:%M') u = nc['u'][itime, ilayer, :] v = nc['v'][itime, ilayer, :] t = nc['temp'][itime,ilayer,:] # t = 32. + t*9./5. #convert from C to F # u = u*1.94 # convert m/s to knots # v = v*1.94 # convert m/s to knots return u,v,t,daystr
def ww2raster(url ='http://motherlode.ucar.edu/thredds/dodsC/fmrc/NCEP/WW3/Regional_US_West_Coast/NCEP-WW3-Regional_US_West_Coast_best.ncd', box = [-132.95925,35.442,-117.279,51.12225], var = 'Significant_height_of_combined_wind_waves_and_swell'): ''' NetCDF4-Python test to read DEM data via OPeNDAP and create Arc Raster also tests out writing a small plot using Matplotlib Global: http://motherlode.ucar.edu/thredds/dodsC/fmrc/NCEP/WW3/Global/NCEP-WW3-Global_best.ncd West Coast: http://motherlode.ucar.edu/thredds/dodsC/fmrc/NCEP/WW3/Regional_US_West_Coast/NCEP-WW3-Regional_US_West_Coast_best.ncd ''' nc = netCDF4.Dataset(url) print "Source name: %s" % nc.title lon = nc.variables['lon'][:]-360.0 lat = nc.variables['lat'][:] bi = (lon>=box[0]) & (lon<=box[2]) bj = (lat>=box[1]) & (lat<=box[3]) # find time index to read hours_from_now = 0 # Examples: 0=>nowcast, 3 => forecast 3 hours from now, etc. date = datetime.datetime.utcnow()+datetime.timedelta(0,3600*hours_from_now) #date=datetime.datetime(2011,9,9,17,00) # specific time (UTC) tindex = netCDF4.date2index(date,nc.variables['time'],select='nearest') z = nc.variables[var][tindex,bj,bi] lonmin = np.min(lon[bi]) latmin = np.min(lat[bj]) dx = np.diff(lon) dy = np.diff(lat) # check if dx or dy vary by more than one percent assert np.abs(np.ptp(dx)/np.mean(dx))<=0.01,'longitude spacing is not uniform' assert np.abs(np.ptp(dy)/np.mean(dy))<=0.01,'latitude spacing is not uniform' if dy[0]>0: # lat increasing z=np.array(z[::-1,:]) if dx[0]<0: # lon decreasing z=np.array(z[:,::-1]) dx=np.abs(np.mean(dx)) dy=np.abs(np.mean(dy)) xyOrig = arcpy.Point(float(lonmin),float(latmin)) # create Arc Raster arcpy.workspace = "C:\\workspace" arcpy.env.overwriteOutput = True rasterName = "sig_ht" outRaster = os.path.normpath(os.path.join(arcpy.workspace,rasterName)) print outRaster grid1=arcpy.NumPyArrayToRaster(z,xyOrig,dx,dy) grid1.save(os.path.join(arcpy.workspace,outRaster)) strPrj = "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID"\ "['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],"\ "UNIT['Degree',0.0174532925199433]]" arcpy.DefineProjection_management(outRaster,strPrj) print "Written: %s" % grid1 arcpy.AddMessage("Written: %s" % grid1) nc.close()
def nc_doy(doy, times): """ Finds day of year for netcdf time dimension :param doy: day of year netcdf variable :param times: half hourly times as netcdf variable :return: """ for t in times[:]: day_time = nC.num2date(t, times.units) idx = nC.date2index(day_time, times) tt = day_time.timetuple() doy[idx] = tt.tm_yday return 'yay'
def get_timestep_netcdf(self, tstep): """ Pull out a time step from the forcing files and place that time step into a dict Args: force: input array of forcing variables tstep: datetime time step Returns: inpt: dictionary of forcing variable images """ inpt = {} for f in self.force.keys(): if isinstance(self.force[f], np.ndarray): # If it's a constant value then just read in the numpy array # pull out the value # ensures not a reference (especially if T_g) inpt[self.MAP_INPUTS[f]] = self.force[f].copy() else: # determine the index in the netCDF file # compare the dimensions and variables to get the variable name v = list( set(self.force[f].variables.keys()) - set(self.force[f].dimensions.keys())) v = [fv for fv in v if fv != 'projection'][0] # make sure you're in the same timezone if hasattr(self.force[f].variables['time'], 'time_zone'): tstep_zone = tstep.astimezone( pytz.timezone( self.force[f].variables['time'].time_zone)) tstep_zone = tstep.tz_localize(None) else: tstep_zone = tstep.tz_localize(None) # find the index based on the time step t = nc.date2index( tstep_zone, self.force[f].variables['time'], calendar=self.force[f].variables['time'].calendar, select='exact') # pull out the value inpt[self.MAP_INPUTS[f]] = \ self.force[f].variables[v][t, :].astype(np.float64) return inpt
def interp_data(cls, lat, lon, utc_time, data, key): """ Interpolate data using nearest neighbor. """ nctime = data['time'] # time ilat, ilon = cls.get_nearest_indices(lat, lon) # time index before before = netCDF4.date2index(utc_time, nctime, select='before') fbefore = data[key][before, ilat, ilon] fafter = data[key][before + 1, ilat, ilon] dt_num = netCDF4.date2num(utc_time, nctime.units) time_ratio = (dt_num - nctime[before]) / cls.dtime return fbefore + (fafter - fbefore) * time_ratio
def nc_mean_daily_temp(half_hourly_temps, daily_mean_temps, times, time_lst): """ Finds mean daily temperatures from half hourly temperature data :param half_hourly_temps: half hourly temperatures as netcdf variable :param daily_mean_temps: empty daily mean temperature netcdf variable :param times: half hourly times as netcdf variable :param time_lst: list of daily datetime objects :return: """ for t in enumerate(time_lst): idx = nC.date2index(t[1], times) mean_daily_temp = np.mean(half_hourly_temps[idx:idx + 48, 0, 0]) daily_mean_temps[t[0], 0, 0] = mean_daily_temp return 'yay'
def find_date_idx(date, data): """ Finds index in netcdf file for given date :param date: date in format specified in DalecData class :return: date index """ if type(date) == int: d_time = dt.datetime(date, 1, 1) elif type(date) == tuple: d_time = dt.datetime(date[0], date[1], date[2]) else: raise ValueError('Date wrong format, please check input') times = data.variables['time'] return nC.date2index(d_time, times, select='nearest')
def nc_day_len(is_day, day_len, times, time_lst): """ Finds total daily global radiation from half hourly global radiation data :param is_day: half hourly 'is day' netcdf variable with values of 1 day or 0 night :param day_len: empty day length netcdf variable (hours) :param times: half hourly times as netcdf variable :param time_lst: list of daily datetime objects :return: """ for t in enumerate(time_lst): idx = nC.date2index(t[1], times) where_day = np.where(is_day[idx:idx + 48, 0, 0] == 1)[0] day_len[t[0], 0, 0] = len(where_day) * 0.5 return 'yay'
def netcdf2NumpyDailyTimeSlice( ncFile, varName, startDate, #endDate, useDoy=None, cloneMapFileName=None, LatitudeLongitude=True, specificFillValue=None): logger.debug('reading variable: ' + str(varName) + ' from the file: ' + str(ncFile)) f = read_netCDF(ncFile) varName = str(varName) f = rename_latlong_dims(f, LatitudeLongitude) t_varname = get_time_variable_name(f) t_dimname = get_time_dimension_name(f) t_unit = get_time_units(f.variables[t_varname]) t_calendar = get_time_calendar(f.variables[t_varname]) startDate = format_date(startDate, f.variables[t_varname], useDoy) endDate = startDate + datetime.timedelta(days=1) lastDateInNC = nc.num2date(f.variables[t_varname][-1], units=t_unit, calendar=t_calendar) startIndex = nc.date2index( datetime.datetime(startDate.year, startDate.month, startDate.day, 0, 0, 0), f.variables[t_varname]) if endDate <= lastDateInNC: endIndex = nc.date2index( datetime.datetime(endDate.year, endDate.month, endDate.day, 0, 0, 0), f.variables[t_varname]) # print endIndex else: endIndex = f.variables[t_varname].size timeIndex = np.arange(startIndex, endIndex) # print timeIndex arr = resample_nc_data(f, varName, cloneMapFileName, t_dimname, timeIndex) f = None return arr
def build_temps(f, x, y): """ This also needs a docstring. """ # Read in and build the netCDF4 parameters nc = netCDF4.Dataset(f) lat = nc.variables['y'][:] lon = nc.variables['x'][:] time_var = nc.variables['time'] dtime = netCDF4.num2date(time_var[:], time_var.units) # Building the indexes points. # By default this starts when Daymet starts, though this could be flexible. # Currently, this only accepts Daymet data. start = dt.datetime(1980, 1, 1) end = dt.datetime.utcnow() istart = netCDF4.date2index(start, time_var, select='nearest') istop = netCDF4.date2index(end, time_var, select='nearest') lati = y loni = x ix = near(lon, loni) iy = near(lat, lati) # Selecting/subsetting the NetCDF dataset. temps = nc.variables['tmax'][:] hs = temps[istart:istop, ix, iy] tim = dtime[istart:istop] # Arranging data into pandas df. temps_ts = pd.Series(hs, index=tim, name='temperature (celsius)') temps_ts = pd.DataFrame(temps_ts) temps_ts.reset_index(inplace=True) temps_ts.columns = ['Index', 'temperature (celsius)'] temps_ts['date'] = temps_ts['Index'] temps_ts.set_index('Index', drop=True, inplace=True) return temps_ts
def build_prcp(f, x, y): """ This needs a docstring! """ # Read in and build the netCDF4 parameters nc = netCDF4.Dataset(f) lat = nc.variables['y'][:] lon = nc.variables['x'][:] time_var = nc.variables['time'] dtime = netCDF4.num2date(time_var[:], time_var.units) # Building the indexes points. # By default this starts when Daymet starts, though this could be flexible. # Currently, this only accepts Daymet data. start = dt.datetime(1980, 1, 1) end = dt.datetime.utcnow() istart = netCDF4.date2index(start, time_var, select='nearest') istop = netCDF4.date2index(end, time_var, select='nearest') lati = y loni = x ix = near(lon, loni) iy = near(lat, lati) # Selecting the variables. prcp = nc.variables['prcp'][:] hs = prcp[istart:istop, ix, iy] tim = dtime[istart:istop] # Arranging data into pandas df. prcp_ts = pd.Series(hs, index=tim, name='precipitation (mm/day)') prcp_ts = pd.DataFrame(prcp_ts) prcp_ts.reset_index(inplace=True) prcp_ts.columns = ['Index', 'precipitation (mm/day)'] prcp_ts['date'] = prcp_ts['Index'] prcp_ts.set_index('Index', drop=True, inplace=True) return prcp_ts
def getMeteoValue(meteoVar, time, latitude, longitude): global year global root_grp meteoVariables = [] currentYear = pd.Timestamp(time).year #currentYear = pd.to_datetime(time).dt.year #print(currentYear) if (currentYear != year): year = currentYear print 'data/d3_' + str(year) + '.nc' root_grp = nc4.Dataset('data/d3_' + str(year) + '.nc') meteoVariable = root_grp.variables[meteoVar] #print meteoVariable.dimensions latVar = root_grp.variables['latitude'] lonVar = root_grp.variables['longitude'] totalLats = len(latVar) totalLons = len(lonVar) maxLat = max(latVar) maxLon = max(lonVar) minLat = min(latVar) minLon = min(lonVar) cellSizeLat = (max(latVar) - min(latVar)) / totalLats cellSizeLat = latVar[1] - latVar[0] cellSizeLon = lonVar[1] - lonVar[0] #(max(lonVar) - min(lonVar)) / totalLons cellTileLat = math.ceil((-minLat + latitude) / cellSizeLat) cellTileLon = math.ceil((-minLon + longitude) / cellSizeLon) time_var = root_grp.variables['time'] #dtime = nc4.num2date(time_var[:], time_var.units) #prova = time_var.sel(time = time) test = nc4.date2index(pd.Timestamp(time), time_var, select="nearest") #print("hello") #print precipitation[test,cellTileLat,cellTileLon] return meteoVariable[test, cellTileLat, cellTileLon] #print(len(temp)) #print(temp[0]) #hs = pd.Series(root_grp.variables['area'].dimensions['lat']) #getMeteoValue(datetime.datetime.strptime("2015-01-04 10:00", "%Y-%m-%d %H:%M"),52,6)
def get_ncfiles_catalog(station_id, jd_start, jd_stop): station_name = station_id.split(":")[-1] uri = 'http://dods.ndbc.noaa.gov/thredds/dodsC/data/adcp' url = ('%s/%s/' % (uri, station_name)) urls = _url_lister(url) filetype = "*.nc" file_list = [filename for filename in fnmatch.filter(urls, filetype)] files = [fname.split('/')[-1] for fname in file_list] urls = ['%s/%s/%s' % (uri, station_name, fname) for fname in files] nc = MFDataset(urls) time_dim = nc.variables['time'] calendar = 'gregorian' idx_start = date2index(jd_start, time_dim, calendar=calendar, select='nearest') idx_stop = date2index(jd_stop, time_dim, calendar=calendar, select='nearest') dir_dim = nc.variables['water_dir'][idx_start:idx_stop, ...].squeeze() speed_dim = nc.variables['water_spd'][idx_start:idx_stop, ...].squeeze() if dir_dim.ndim != 1: dir_dim = dir_dim[:, 0] speed_dim = speed_dim[:, 0] time_dim = nc.variables['time'] dates = num2date(time_dim[idx_start:idx_stop], units=time_dim.units, calendar='gregorian').squeeze() data = dict() data['sea_water_speed (cm/s)'] = speed_dim col = 'direction_of_sea_water_velocity (degree)' data[col] = dir_dim time = dates columns = ['sea_water_speed (cm/s)', 'direction_of_sea_water_velocity (degree)'] df = DataFrame(data=data, index=time, columns=columns) return df
def get_FVCOM_temp( latp, lonp, dtime, depth='bottom', mindistance=2, fortype='temperature'): # gets modeled temp using GOM3 forecast ''' fortype list ['tempdepth','temperature'] the unite of the mindistance is mile Taken primarily from Rich's blog at: http://rsignell-usgs.github.io/blog/blog/2014/01/08/fvcom/ on July 30, 2018 where lati and loni are the position of interest, dtime is the datetime, and layer is "-1" for bottom ''' m2k_factor = 0.62137119 #mile to kilometers parameter #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3' #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc' # urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc' urlfvcom = get_FVCOM_url(dtime) #if math.isnan(urlfvcom): if urlfvcom == np.nan: if fortype == 'temperature': return np.nan elif fortype == 'tempdepth': return np.nan, np.nan else: 'please input write fortype' nc = netCDF4.Dataset(urlfvcom).variables #first find the index of the grid lat = nc['lat'][:] lon = nc['lon'][:] inode, dist = nearlonlat(lon, lat, lonp, latp) if dist > mindistance / m2k_factor / 111: return np.nan, np.nan #second find the index of time time_var = nc['time'] itime = netCDF4.date2index(dtime, time_var, select='nearest') # where startime in datetime if depth == 'bottom': layer = -1 else: depth_distance = abs(nc['siglay'][:, inode] * nc['h'][inode] + depth) layer = np.argmin(depth_distance) if fortype == 'temperature': return nc['temp'][itime, layer, inode] elif fortype == 'tempdepth': return nc['temp'][itime, layer, inode], nc['h'][inode] else: return 'please input write fortype'
def netcdf2pcrDynamic( self, pcr, forcing ): #ncFile, varName, dateInput, method, factor, x, y, xi, yi, xIdxSta, xIdxEnd, yIdxSta, yIdxEnd): #-read netcdf file f = nc.Dataset(getattr(self, forcing + 'NC')) filecache[getattr(self, forcing + 'NC')] = f #-get index from netcdf corresponding with current date idx = int(nc.date2index(self.curdate, f.variables['time'], select='exact')) #-get raw netcdf gridded data from netcdf, transform to array and multiply with factor if getattr(self, forcing + 'InProj') == "rotated": z = f.variables[getattr( self, forcing + 'VarName')][idx, getattr(self, forcing + 'xyUL'):(getattr(self, forcing + 'xyLL') + 1), getattr(self, forcing + 'xyUR'):(getattr(self, forcing + 'xyLR') + 1)] else: z = f.variables[getattr( self, forcing + 'VarName')][idx, getattr(self, forcing + 'yIdxSta'):( getattr(self, forcing + 'yIdxEnd') + 1), getattr(self, forcing + 'xIdxSta'):( getattr(self, forcing + 'xIdxEnd') + 1)] z = np.asarray(z).ravel() with np.errstate( invalid='ignore' ): # surpress error message when there are already nans in the z array z = np.where(z <= -9999, np.nan, z) * getattr(self, forcing + 'Factor') #-remove nans from arrays x = getattr(self, forcing + 'x')[~np.isnan(z)] y = getattr(self, forcing + 'y')[~np.isnan(z)] z = z[~np.isnan(z)] #-interpolate with method (linear or cubic) zi = griddata( (x, y), z, (getattr(self, forcing + 'xi'), getattr(self, forcing + 'yi')), method=getattr(self, forcing + 'Method')) zi = np.where(np.isnan(zi), -9999, zi) #-convert to PCRaster Python map output = pcr.numpy2pcr(pcr.Scalar, zi, -9999) return output
def __getitem__(self, key): if type(key) == datetime.datetime: index = netCDF4.date2index( key, self.dataset.variables[self.time_var]) data = {} for var in self._get_all_ts_variables(): data[var] = self.dataset.variables[var][:, index] return data else: gpi = np.atleast_1d(key) for i, gp in enumerate(gpi): data = super(ArrayStack, self).read_all_ts(gp) return pd.DataFrame(data, index=self.times)
def nc_day_mean_temp(is_day, hh_temp, mean_t_day, times, time_lst): """ Finds total daily global radiation from half hourly global radiation data :param is_day: half hourly 'is day' netcdf variable with values of 1 day or 0 night :param hh_temp: half hourly temperatures netcdf variable :param mean_t_day: netcdf variable to fill with mean daytime temperatures :param times: half hourly times as netcdf variable :param time_lst: list of daily datetime objects :return: """ for t in enumerate(time_lst): idx = nC.date2index(t[1], times) where_day = np.where(is_day[idx:idx + 48, 0, 0] == 1)[0] mean_t_day[t[0], 0, 0] = np.mean(hh_temp[idx + where_day[0]:idx + where_day[-1]]) return 'yay'
def nc_total_daily_rg(half_hourly_rg, daily_rg, times, time_lst): """ Finds total daily global radiation from half hourly global radiation data :param half_hourly_rg: half hourly global radiation as netcdf variable (W m-2) :param daily_rg: empty total daily global radiation netcdf variable (M J m-2 day-1) :param times: half hourly times as netcdf variable :param time_lst: list of daily datetime objects :return: """ for t in time_lst: idx = nC.date2index(t, times) total_daily_rg = 30 * 60 * 1e-6 * np.sum( half_hourly_rg[idx:idx + 48, 0, 0]) # Convert W m-2 to M J m-2 day-1 daily_rg[idx, 0, 0] = total_daily_rg return 'yay'
def __getitem__(self, key): if type(key) == datetime.datetime: index = netCDF4.date2index(key, self.dataset.variables[self.time_var]) data = {} for var in self._get_all_ts_variables(): data[var] = self.dataset.variables[var][:, index] return data else: gpi = np.atleast_1d(key) for i, gp in enumerate(gpi): data = super(ArrayStack, self).read_all_ts(gp) return pd.DataFrame(data, index=self.times)
def runTest(self): # The test files have no calendar attribute on the time variable. calendar = 'standard' # Get the real dates # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this if parse_version(cftime.__version__) >= parse_version('1.0.1'): dates = [] for file in self.files: f = Dataset(file) t = f.variables['time'] dates.extend(num2date(t[:], t.units, calendar)) f.close() # Compare with the MF dates f = MFDataset(self.files,check=True) t = f.variables['time'] T = MFTime(t, calendar=calendar) assert_equal(T.calendar, calendar) assert_equal(len(T), len(t)) assert_equal(T.shape, t.shape) assert_equal(T.dimensions, t.dimensions) assert_equal(T.typecode(), t.typecode()) # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this if parse_version(cftime.__version__) >= parse_version('1.0.1'): assert_array_equal(num2date(T[:], T.units, T.calendar), dates) assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366) f.close() # Test exception is raised when no calendar attribute is available on the # time variable. with MFDataset(self.files, check=True) as ds: with self.assertRaises(ValueError): MFTime(ds.variables['time']) # Test exception is raised when the calendar attribute is different on the # variables. First, add calendar attributes to file. Note this will modify # the files inplace. calendars = ['standard', 'gregorian'] for idx, f in enumerate(self.files): with Dataset(f, 'a') as ds: ds.variables['time'].calendar = calendars[idx] with MFDataset(self.files, check=True) as ds: with self.assertRaises(ValueError): MFTime(ds.variables['time'])
def runTest(self): # The test files have no calendar attribute on the time variable. calendar = 'standard' # Get the real dates # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this if parse_version(cftime.__version__) >= parse_version('1.0.1'): dates = [] for file in self.files: f = Dataset(file) t = f.variables['time'] dates.extend(num2date(t[:], t.units, calendar)) f.close() # Compare with the MF dates f = MFDataset(self.files, check=True) t = f.variables['time'] T = MFTime(t, calendar=calendar) assert_equal(T.calendar, calendar) assert_equal(len(T), len(t)) assert_equal(T.shape, t.shape) assert_equal(T.dimensions, t.dimensions) assert_equal(T.typecode(), t.typecode()) # skip this until cftime pull request #55 is in a released # version (1.0.1?). Otherwise, fix for issue #808 breaks this if parse_version(cftime.__version__) >= parse_version('1.0.1'): assert_array_equal(num2date(T[:], T.units, T.calendar), dates) assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366) f.close() # Test exception is raised when no calendar attribute is available on the # time variable. with MFDataset(self.files, check=True) as ds: with self.assertRaises(ValueError): MFTime(ds.variables['time']) # Test exception is raised when the calendar attribute is different on the # variables. First, add calendar attributes to file. Note this will modify # the files inplace. calendars = ['standard', 'gregorian'] for idx, f in enumerate(self.files): with Dataset(f, 'a') as ds: ds.variables['time'].calendar = calendars[idx] with MFDataset(self.files, check=True) as ds: with self.assertRaises(ValueError): MFTime(ds.variables['time'])
def get_location_cdd(lat: float, long: float, year: int, model: str) -> float: d = get_dataset_for_climate_model(model) # print(d) time = d['/time'] time_idx = date2index(datetime(year, 1, 1, 0, 0), time) lats = d['/lat'] lat_idx = bisect_left(lats[:], lat) longs = d['/lon'] long_idx = bisect_left(longs[:], long) cdd = d['/cdd'] return cdd[time_idx][lat_idx][long_idx]
def get_doppio(lat,lon,time,depth): """ notice: the format of time is like "%Y-%m-%d %H:%M:%S" the default depth is under the bottom depth the module only output the temperature of point location """ time=dt.strptime(time,'%Y-%m-%d %H:%M:%S') # transform time format if (time -datetime.datetime(2017,11,1,0,0,0)).total_seconds()<0: print('the date can\'t be earlier than 2017-11-1') return np.nan url_time=time.strftime('%Y-%m-%d')# url=get_doppio_url(url_time) nc=netCDF4.Dataset(url).variables #first find the index of the grid lons=nc['lon_rho'][:] lats=nc['lat_rho'][:] temp=nc['temp'] #second find the index of time doppio_time=nc['time'] itime = netCDF4.date2index(time,doppio_time,select='nearest')# where startime in datetime # figure out layer from depth min_distance=dist(lat1=lat,lon1=lon,lat2=lats[0][0],lon2=lons[0][0]) index_1,index_2=0,0 for i in range(len(lons)): for j in range(len(lons[i])): if min_distance>dist(lat1=lat,lon1=lon,lat2=lats[i][j],lon2=lons[i][j]): min_distance=dist(lat1=lat,lon1=lon,lat2=lats[i][j],lon2=lons[i][j]) index_1=i index_2=j doppio_depth=nc['h'][index_1][index_2] if depth > doppio_depth:# case of bottom S_coordinate=1 else: S_coordinate=float(depth)/float(doppio_depth) if 0<=S_coordinate<1: doppio_temp=temp[itime,39-int(S_coordinate/0.025),index_1,index_2]# because there are 0.025 between each later elif S_coordinate==1: doppio_temp=temp[itime][0][index_1][index_2] else: doppio_temp=temp[itime][0][index_1][index_2] return doppio_temp
def get_training_data(analog_dates,leadtime,train_fname,predictor_name,reforecast_dir): """ Function that collects training data for statistical post-processing. analog_dates - list of datetime objects of potential analogous dates leadtime - forecast leadtime reforecast_dir - path to verification data """ # --- First, get analog dates nc_fname = '{}refcst2_{}_day{}.nc'.format(reforecast_dir,train_fname,leadtime) trainData = Dataset(nc_fname,'r') train_idxs = date2index(analog_dates,trainData.variables['time']) trainData = np.asfortranarray(trainData.variables[predictor_name][train_idxs,:,:]) return trainData
def get_jules_state(date_utc, nc_file='jules/output/wallerfing_79_12.3_hourly.nc'): """Function that returns a stateVector instance for a given time. :param date_utc: datetime object of when to extract JULES output. :type date_utc: object :param nc_file: JULES output file from which to extract data. :type nc_file: str :return: Instance of stateVector class. :rtype: instance """ nc_dat = nc.Dataset(nc_file, 'r') t_idx = nc.date2index(date_utc, nc_dat.variables['time'], select='nearest') state_inst = stateVector() state_inst.date_utc = nc.num2date(nc_dat.variables['time'][t_idx], nc_dat.variables['time'].units) state_inst.lai = nc_dat.variables['croplai'][t_idx, 0, 0, 0] # (m2 m-2) state_inst.can_height = nc_dat.variables['cropcanht'][t_idx, 0, 0, 0] # (m) state_inst.soil_moisture = nc_dat.variables['smcl'][t_idx, 0, 0, 0] # (kg m-2) nc_dat.close() return state_inst
def process_co2_flux_nighttime_d(clipped_co2_flux, qc_co2_flux, nee_night, nee_night_std, is_day, wind_dir, origin, foot_print, times, time_lst, qc2_tol=1, qc1_tol=5): """ Produces a nighttime NEE product :param clipped_co2_flux: half hourly clipped co2 flux as netcdf variable :param qc_co2_flux: qc flags as nc variable corresponding to half hourly co2 flux :param nee_night: nc variable to fill with processed data :param nee_night_std: nc variable to fill with processed data standard deviation :param times: half hourly times as netcdf variable :param time_lst: list of daily datetime objects :return: """ for t in enumerate(time_lst): idx = nC.date2index(t[1], times) night_idx1 = idx + np.where(is_day[idx:idx + 48, 0, 0] == 1)[0][-1] + 1 night_idx2 = night_idx1 while is_day[night_idx2, 0, 0] == 0: night_idx2 += 1 if night_idx2 >= len(times): night_idx2 = float('NaN') break if np.isnan(night_idx2) == True: nee_night[t[0], 0, 0] = float('NaN') break else: quality_control_co2_flux_daily(clipped_co2_flux, qc_co2_flux, nee_night, nee_night_std, wind_dir, origin, foot_print, night_idx1, night_idx2 - 1, t[0], is_day, qc2_tol, qc1_tol) if nee_night[t[0], 0, 0] < 0.0: nee_night[t[0], 0, 0] = float('NaN') else: continue return 'yay'
def get_FVCOM_bottom_temp(lati, loni, dtime, layer): # gets modeled temp using GOM3 forecast ''' Taken primarily from Rich's blog at: http://rsignell-usgs.github.io/blog/blog/2014/01/08/fvcom/ on July 30, 2018 where lati and loni are the position of interest, dtime is the datetime, and layer is "-1" for bottom ''' #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3' #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc' urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc' nc = netCDF4.Dataset(urlfvcom).variables #first find the index of the grid lat = nc['lat'][:] lon = nc['lon'][:] inode = nearlonlat(lon, lat, loni, lati) #second find the index of time time_var = nc['time'] itime = netCDF4.date2index(dtime, time_var, select='nearest') # where startime in datetime return nc['temp'][itime, layer, inode]
def getERA5_1DAYfilename(confM2R, year, month, day, myvar): if len(confM2R.timeobject) == 0: mcdf = MFDataset(confM2R.atmosphericpath + "*.nc") confM2R.timeobject = mcdf.variables["time"] print("Loaded all ERA5 timesteps: {}".format(confM2R.timeobject[:])) index = date2index(datetime(year, month, day, 0, 0), confM2R.timeobject, calendar=confM2R.timeobject.calendar, select="nearest") seldate = num2date(confM2R.timeobject[index], units=confM2R.timeobject.units, calendar=confM2R.timeobject.calendar) print("selected index {}".format(seldate)) return '{}soda3.3.2_5dy_ocean_reg_{:04}_{:02}_{:02}.nc'.format( confM2R.atmosphericpath, seldate.year, seldate.month, seldate.day)
def get_espresso_temp(time,lat,lon,depth) : #according to doppio model structure , data is from 2009-10-12 to 2017-1-1 time=pd.to_datetime(time) url=get_url(time) nc=netCDF4.Dataset(url).variables #first find the index of the grid lons=nc['lon_rho'][:] lats=nc['lat_rho'][:] temp=nc['temp'] #second find the index of time if time<=datetime(2013,5,18): espresso_time=nc['ocean_time'] else: espresso_time=nc['time'] itime = netCDF4.date2index(time,espresso_time,select='nearest') index = nearest_point_index2(lon,lat,lons,lats) depth_layers=nc['h'][index[0][0]][index[1][0]]*nc['s_rho'] index_depth=np.argmin(abs(depth+depth_layers))#depth_layers are negative numbers espresso_temp=temp[itime,index_depth,index[0][0],index[1][0]] return espresso_temp
def runTest(self): # Get the real dates dates = [] for file in self.files: f = Dataset(file) t = f.variables['time'] dates.extend(num2date(t[:], t.units, t.calendar)) f.close() # Compare with the MF dates f = MFDataset(self.files,check=True) t = f.variables['time'] mfdates = num2date(t[:], t.units, t.calendar) T = MFTime(t) assert_equal(len(T), len(t)) assert_equal(T.shape, t.shape) assert_equal(T.dimensions, t.dimensions) assert_equal(T.typecode(), t.typecode()) assert_array_equal(num2date(T[:], T.units, T.calendar), dates) assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366)