def main(): crs = ccrs.PlateCarree(central_longitude=0) x fname = ('../DTU/DTU10MDT_2min.nc') dataset = netcdf_dataset(fname) # print(dataset.variables) mdt = dataset.variables['mdt'][:] # mdt = bound_arr(mdt, -2, 2) lats = dataset.variables['lat'][:] lons = dataset.variables['lon'][:] print(mdt.shape) print(lats, lons) fig = plt.figure() ax = fig.add_subplot(1, 1, 1, projection=crs) # ax = plt.axes(projection=crs) # mdt = mdt + mask vmin = np.min(mdt) vmax = np.max(mdt) print(vmin, vmax) # plt.pcolormesh(lons, lats, mdt, transform=crs) im = ax.pcolormesh(lons, lats, mdt, transform=crs, cmap='turbo', vmin=vmin, vmax=vmax) ax.set_xticks([-180, -135, -90, -45, 0, 45, 90, 135, 180], crs=crs) ax.set_yticks([-90, -60, -30, 0, 30, 60, 90], crs=crs) lon_formatter = LongitudeFormatter(zero_direction_label=True) lat_formatter = LatitudeFormatter() # Following axes formatters cannot be used with non-rectangular projections ax.xaxis.set_major_formatter(lon_formatter) ax.yaxis.set_major_formatter(lat_formatter) ax.fontsize = 20 # ax.add_feature(cfeature.LAND, resolution='10m') # ax.coastlines() # ax.gridlines() # ax.set_extent([100, 160, 0, 60]) # ax.set_aspect('auto', adjustable=None) # cbar_arr = (np.linspace(vmin, vmax, 5, dtype=int)) # Uncomment following to produce colorbar corresponding to each individual plot: fig.colorbar(im, ax=ax, fraction=0.0235, pad=0.04, ticks=[-2, -1.5, -1, -0.5, 0, 0.5, 1, 1., 1.5, 2], format=ticker.FuncFormatter(cbar_fmt)) plt.title('DTU10 Global Mean Dynamic Topography: 2 min resolution', fontsize=21) plt.show()
def request_netcdf_dataset(request_url, nc_filename): req = Request(url) try: response = urlopen(req, timeout=70) except HTTPError as he: # Update config file to show that the dataset is not valid print("Could not access data from ERDDAP") print(he.reason) except URLError as e: print(' * Failed to reach erddap server. Stopping.') print(' * reason: ', e.reason) except Exception: print('generic exception: ' + traceback.format_exc()) print("General Error (likely timeout)") else: print(' * data file returned from erddap') CHUNK = 16 *1024 with open(nc_filename,'wb') as f: while True: chunk = response.read(CHUNK) if not chunk: break f.write(chunk) try: dataset = netcdf_dataset(nc_filename) except: print("Not valid netcdf returned") else: return dataset
def getPointDF(dirname, filename, in_lat, in_lon, make_csv=True): ##open netcdf nc = netcdf_dataset(os.path.join('.', dirname, filename)) ##pick a point lats = nc.variables['lat'][:] lons = nc.variables['lon'][:] lat_idx = geo_idx(in_lat, lats) lon_idx = geo_idx(in_lon, lons) ##pull out variables at that point (lev1 only) for each variable (pre-defined list) vardf = pd.DataFrame(index=variable_names, columns=month_names) #dict/df: key=varname, value=12x1 array retrieved ##pull out 1 layer of variable to add to df -- internal loop for varname in variable_names: var = nc[varname] if (len(var.shape) == 3): ##2d var vardf.loc[varname, :] = var[:, lat_idx, lon_idx] elif (len(var.shape) == 4): ##3d var vardf.loc[varname, :] = var[:, 0, lat_idx, lon_idx] ##write to csv with newfilename if (make_csv): newfilename = os.path.join( '.', dirname, filename.split(".")[0] + '_' + str(in_lat) + '_' + str(in_lon) + ".csv") vardf.to_csv(newfilename) return (vardf)
def getData(dataset_id, parameter, bounds, time, sub, mapImageDir): lat1 = qbounds[0] lat2 = qbounds[1] lon1 = qbounds[2] lon2 = qbounds[3] lat_order = qbounds[4] altf = qbounds[5] time1 = time # using just one timestamp for this example time2 = time print(lat1) print(lat2) print(lon1) print(lon2) print(sub) print(time1) print(time2) if altf == 1: #write query with altitude, haven't tested this alt1 = float(bounds[6]) alt2 = float(bounds[7]) altsub = '1' print(alt1) print(alt2) print(altsub) query = parameter + '[(%s):%s:(%s)][(%f):%s:(%f)][(%f):%s:(%f)][(%f):%s:(%f)]' % ( time1, sub, time2, alt1, altsub, alt2, lat1, sub, lat2, lon1, sub, lon2) else: #write query without altitude query = parameter + '[(%s):%s:(%s)][(%f):%s:(%f)][(%f):%s:(%f)]' % ( time1, sub, time2, lat1, sub, lat2, lon1, sub, lon2) base_url = 'http://polarwatch.noaa.gov/erddap/griddap/' + dataset_id + '.nc?' url = base_url + query print(url) file = mapImageDir + '/' + parameter + '.nc' http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) r = http.request('GET', url, preload_content=False) with open(file, 'wb') as out: while True: data = r.read(1024 * 1024) if not data: break out.write(data) r.release_conn() # add a real check here print('Satellite Data File Retrieved') data_netcdf_dataset = netcdf_dataset(file) dataset = {"data": data_netcdf_dataset, "url": url} return dataset
def load_cls(fname, path=None, var='mdt'): if path is None: path = "" filepath = os.path.join(os.path.normpath(path), fname) dataset = netcdf_dataset(filepath) var = dataset.variables[var][0, :, :] lats = dataset.variables['latitude'][:] lons = dataset.variables['longitude'][:] return var, lats, lons
def __init__(self,base:str,peturbed:str): ''' Arguments: base - the initial simulation path and filename peturbed - the changes simulation path and filename Both arguments require the full path to the netCDF file we wish to load. ''' self.basename = base.split('/')[-1] self.base = netcdf_dataset(base) self.basevars = self.base.variables self.peturbedname = peturbed.split('/')[-1] self.peturbed = netcdf_dataset(peturbed) self.peturbedvars = self.base.variables self.varboth = set(self.basevars) & set(self.basevars)
def load_dtu(fname, path=None, var='mdt'): if path is None: path = "" filepath = os.path.join(os.path.normpath(path), fname) dataset = netcdf_dataset(filepath) print(dataset.variables) var = dataset.variables[var][:] lats = dataset.variables['lat'][:] lons = dataset.variables['lon'][:] return var, lats, lons
def getData(dId, parameter, bounds, time, sub): print(bounds) print(parameter) print(time) lat1 = bounds[0] lat2 = bounds[1] lon1 = bounds[2] lon2 = bounds[3] altf = bounds[4] time1 = time # using just one timestamp for this example time2 = time if altf == 1: #write query with altitude, haven't tested this alt1 = bounds[5] alt2 = bounds[6] altsub = 1 query = parameter + '[(%s):%s:(%s)][(%f):%s:(%f)][(%f):%s:(%f)][(%f):%s:(%f)]' % ( time1, sub, time2, alt1, altsub, alt2, lat1, sub, lat2, lon1, sub, lon2) else: #write query without altitude query = parameter + '[(%s):%s:(%s)][(%f):%s:(%f)][(%f):%s:(%f)]' % ( time1, sub, time2, lat1, sub, lat2, lon1, sub, lon2) base_url = 'http://polarwatch.noaa.gov/erddap/griddap/' + dId + '.nc?' url = base_url + query print(url) file = 'dataset.nc' http = urllib3.PoolManager() r = http.request('GET', url, preload_content=False) with open(file, 'wb') as out: while True: data = r.read(1024 * 1024) if not data: break out.write(data) r.release_conn() # add a real check here print('Satellite Data File Retrieved') datafile = netcdf_dataset(file) dataset = {"data": datafile, "url": url} return dataset
def datareader(dp=[], d=[], v=[], *args): Dpath = [] Dread = [] for i in range(0, len(dp)): Dpath.append(dp[i] + d[i]) if (d[i] == ''): continue temp = netcdf_dataset(Dpath[i]) try: Dread.append(temp.variables[v[i]][0, :, :]) except: Dread.append(temp.variables[v[i]]) try: Dread.append(temp.variables['lat'][:]) Dread.append(temp.variables['lon'][:]) except: print() return Dread
def read_dataset(name='', granule_name='', variable=None, path='/tmp'): d = netcdf_dataset(path, mode='r') dataset = d.variables[variable] # By convention, but not by standard, if the dimensions exist, they will be in the order: # time (t), altitude (z), latitude (y), longitude (x) # but conventions aren't always followed and all dimensions aren't always present so # see if we can make some educated deductions before defaulting to just pulling the first three # columns. temp_dimensions = list(map(lambda x: x.lower(), dataset.dimensions)) dataset_dimensions = dataset.dimensions time = dataset_dimensions[temp_dimensions.index('time') if 'time' in temp_dimensions else 0] lat = dataset_dimensions[temp_dimensions.index('lat') if 'lat' in temp_dimensions else 1] lon = dataset_dimensions[temp_dimensions.index('lon') if 'lon' in temp_dimensions else 2] # Time is given to us in some units since an epoch. We need to convert # these values to datetime objects. Note that we use the main object's # time object and not the dataset specific reference to it. We need to # grab the 'units' from it and it fails on the dataset specific object. times = np.array(convert_times_to_datetime(d[time])) lats = np.array(d.variables[lat][:]) lons = np.array(d.variables[lon][:]) values = np.array(dataset[:]) origin = {'source': 'PO.DAAC', 'url': 'podaac.jpl.nasa.gov'} # Removing the downloaded temporary granule before creating the OCW # dataset. d.close() path = os.path.join(os.path.dirname(__file__), granule_name) os.remove(path) return Dataset(lats, lons, times, values, variable, name=name, origin=origin)
def read_dataset(name='', granule_name ='', variable=None, path='/tmp'): d = netcdf_dataset(path, mode='r') dataset = d.variables[variable] # By convention, but not by standard, if the dimensions exist, they will be in the order: # time (t), altitude (z), latitude (y), longitude (x) # but conventions aren't always followed and all dimensions aren't always present so # see if we can make some educated deductions before defaulting to just pulling the first three # columns. temp_dimensions = list(map(lambda x: x.lower(), dataset.dimensions)) dataset_dimensions = dataset.dimensions time = dataset_dimensions[temp_dimensions.index( 'time') if 'time' in temp_dimensions else 0] lat = dataset_dimensions[temp_dimensions.index( 'lat') if 'lat' in temp_dimensions else 1] lon = dataset_dimensions[temp_dimensions.index( 'lon') if 'lon' in temp_dimensions else 2] # Time is given to us in some units since an epoch. We need to convert # these values to datetime objects. Note that we use the main object's # time object and not the dataset specific reference to it. We need to # grab the 'units' from it and it fails on the dataset specific object. times = np.array(convert_times_to_datetime(d[time])) lats = np.array(d.variables[lat][:]) lons = np.array(d.variables[lon][:]) values = np.array(dataset[:]) origin = { 'source': 'NASA JPL PO.DAAC', 'url': 'https://podaac.jpl.nasa.gov' } # Removing the downloaded temporary granule before creating the OCW # dataset. d.close() path = os.path.join(os.path.dirname(__file__), granule_name) os.remove(path) return Dataset(lats, lons, times, values, variable, name=name, origin=origin)
def read_nc(filepath,filename): """ Read satellite nc file :param string filepath: path to file :param string filename: name of file :return netcdf dataset: Array of sst over time and space :return list: Time indices :return list: Time labels """ dataset = netcdf_dataset(filepath+filename,"r",format="NETCDF3_64BIT_DATA") time = dataset.variables['analysed_sst'][:,0,0] time=len(time) time=np.arange(time) time=np.asarray(time) print(time) ################# # Getting date labels to put in the images: ################# time_label = dataset.variables['time'][:] #Time is in epoch time, need to convert it to human readable time #Use datetime to convert to human readable: time_list = [] for x in time: lab = datetime.datetime.fromtimestamp(time_label[x]).strftime('%Y-%m-%d %H:%M:%S') #print(lab) time_list.append(lab) #Convert to data frame, separate date from time into different columns, keep date column only: time_label = DataFrame(time_list, columns=['date']) time_label['date'] = time_label['date'].str.split(r'\ ').str.get(0) return [dataset,time,time_label]
def get_grid(filename): print(filename) nc = netcdf_dataset(filename) sat_height = nc.variables[ 'goes_imager_projection'].perspective_point_height radar_lon = -98.128 radar_lat = 36.741 _x = nc.variables['x'] * sat_height _y = nc.variables['y'] * sat_height _c = nc.variables['CMI_C13'][:] * -1 data = nc.variables['CMI_C13'] proj_var = nc.variables[data.grid_mapping] globe = ccrs.Globe(ellipse='sphere', semimajor_axis=proj_var.semi_major_axis, semiminor_axis=proj_var.semi_minor_axis) proj = ccrs.Geostationary(central_longitude=-75, sweep_axis='x', satellite_height=sat_height, globe=globe) trans = ccrs.PlateCarree(central_longitude=0) transform_xy = trans.transform_points(proj, _x, _y) lim = [ _nearest(transform_xy[:, 0], -103), _nearest(transform_xy[:, 0], -92), _nearest(transform_xy[:, 1], 42), _nearest(transform_xy[:, 1], 30) ] x = _x[lim[0]:lim[1]] y = _y[lim[2]:lim[3]] c = _c[lim[2]:lim[3], lim[0]:lim[1]] dx = x.max() - x.min() dy = y.max() - y.min() print('X: ' + str(dx)) print('Y: ' + str(dy)) # x = _x # y = _y # c = _c x_mesh, y_mesh = np.meshgrid(x, y) print(proj) print(x_mesh) print(y_mesh) lonlat = trans.transform_points(proj, x_mesh, y_mesh) lons = lonlat[:, :, 0] lats = lonlat[:, :, 1] interp_c = interp_lonlat(lons, lats, c, radar_lon, radar_lat, grid_x, grid_y) _time = { 'calendar': 'gregorian', 'data': np.array([0.934]), 'long_name': 'Time of grid', 'standard_name': 'time', 'units': str('seconds since ' + nc.time_coverage_end) } # _fields = {'reflectivity': {'_FillValue': -9999.0, 'data': np.ma.masked_array(c, mask= False), # 'long_name': 'reflectivity', # 'standard_name': 'equivalent_reflectivity_factor', # 'units': 'dBZ', 'valid_max': c.max(), 'valid_min': c.min()}} _fields = { 'c13': { '_FillValue': -9999.0, 'data': interp_c, 'long_name': 'channel 13 10.3 microns K', 'standard_name': 'c13', 'units': 'K', 'valid_max': c.max(), 'valid_min': c.min() } } _metadata = { 'Conventions': '', 'comment': '', 'history': '', 'institution': '', 'instrument_name': '', 'original_container': 'NEXRAD Level II', 'references': '', 'source': '', 'title': '', 'vcp_pattern': '', 'version': '' } _origin_latitude = { 'data': np.array([0]), 'long_name': 'Latitude at grid origin', 'standard_name': 'latitude', 'units': 'degrees_north', 'valid_max': 90.0, 'valid_min': -90.0 } # _origin_longitude = {'data': np.ma.array([radar_lon]), # 'long_name': 'Longitude at grid origin', # 'standard_name': 'longitude', 'units': 'degrees_east', # 'valid_max': 180.0, 'valid_min': -180.0} # # _origin_altitude = {'data': np.ma.masked_array(np.array([0]), mask= False), # 'long_name': 'Altitude at grid origin', # 'standard_name': 'altitude', 'units': 'm'} _x = { 'axis': 'X', 'data': grid_x['data'], 'long_name': 'X distance on the projection plane from the origin', 'standard_name': 'projection_x_coordinate', 'units': 'm' } _y = { 'axis': 'Y', 'data': grid_y['data'], 'long_name': 'Y distance on the projection plane from the origin', 'standard_name': 'projection_x_coordinate', 'units': 'm' } _z = { 'axis': 'Z', 'data': np.array([0, mesh_size]), 'long_name': 'Z distance on the projection plane from the origin', 'positive': 'up', 'standard_name': 'projection_z_coordinate', 'units': 'm' } grid = pyart.core.grid.Grid(time=_time, fields=_fields, metadata=_metadata, origin_latitude=_orig_lat, origin_longitude=_orig_lon, origin_altitude=_orig_alt, x=_x, y=_y, z=_z, projection=_projection, radar_longitude=_orig_lon, radar_latitude=_orig_lat, radar_altitude=_orig_alt) grid_name = os.path.basename(filename[:-3] + '_grid.nc') full_name = os.path.join(out_dir, grid_name)
from mpl_toolkits.axes_grid1 import AxesGrid import matplotlib.path as mpath import matplotlib.colors as colors # numpy import numpy as np # parameters from get_parameters import * # scipy from scipy import stats # data path data_path="./precip_Arctic/" file_name=data_path+"precip_statis_2x2.nc" # read data precp_file=netcdf_dataset(file_name,"r") cnt_rn_cert=np.array(precp_file.variables["cnt_rain_cert"]) cnt_rn_poss=np.array(precp_file.variables["cnt_rain_poss"]) cnt_sn_cert=np.array(precp_file.variables["cnt_snow_cert"]) cnt_sn_poss=np.array(precp_file.variables["cnt_snow_poss"]) cnt_no_prec=np.array(precp_file.variables["cnt_noprecp"]) pre_rn_cert=np.array(precp_file.variables["pre_rain_cert"]) pre_rn_poss=np.array(precp_file.variables["pre_rain_poss"]) pre_sn_cert=np.array(precp_file.variables["pre_snow_cert"]) pre_sn_poss=np.array(precp_file.variables["pre_snow_poss"]) lat=np.array(precp_file.variables["lat"]) lon=np.array(precp_file.variables["lon"]) nlat=len(lat) nlon=len(lon) # calculate frequency and ratio
means_yby_ctl_net_diag=np.zeros((years.size,2,nlat)) #year by year mean for each variable means_yby_exp_net_diag=np.zeros((years.size,2,nlat)) #year by year mean for each variable means_ctl_net_diag=np.zeros((2,nlat)) #multi-year mean for each variable means_exp_net_diag=np.zeros((2,nlat)) #multi-year mean for each variable diffs_net_diag=np.zeros((2,nlat)) #multi-year exp-ctl diff for each variable gm_yby_ctl_net_diag=np.zeros((2,years.size)) #year by year mean for each variable gm_yby_exp_net_diag=np.zeros((2,years.size)) #year by year mean for each variable means_yby_exp_fice=np.zeros((years.size,nlat)) #year by year mean for each variable means_exp_fice=np.zeros((nlat)) #multi-year mean for each variable for iy in range(0,years.size): # open data file fctl=fpath_ctl+ctl_pref+"_ANN_"+str(years[iy])+".nc" fexp=fpath_exp+exp_pref+"_ANN_"+str(years[iy])+".nc" file_ctl=netcdf_dataset(fctl,"r") file_exp=netcdf_dataset(fexp,"r") fctl_diag=fpath_ctl_diag+ctl_pref_diag+"_climo_ANN.nc" fexp_diag=fpath_exp_diag+exp_pref_diag+"_climo_ANN.nc" file_ctl_diag=netcdf_dataset(fctl_diag,"r") file_exp_diag=netcdf_dataset(fexp_diag,"r") # read lat and lon lat=file_ctl.variables["lat"] lon=file_ctl.variables["lon"] means_yby_exp_fice[iy,:]=means_yby_exp_fice[iy,:]+np.mean(file_exp.variables["ICEFRAC"][0,:,:],axis=1) # read data and calculate mean/min/max for vn in varnms_vis_dn:
def ocn_modelvsobs(config, field, streamMap=None, variableMap=None): """ Plots a comparison of ACME/MPAS output to SST or MLD observations config is an instance of MpasAnalysisConfigParser containing configuration options. field is the name of a field to be analyize (currently one of 'mld' or 'sst') If present, streamMap is a dictionary of MPAS-O stream names that map to their mpas_analysis counterparts. If present, variableMap is a dictionary of MPAS-O variable names that map to their mpas_analysis counterparts. Authors: Luke Van Roekel, Milena Veneziani, Xylar Asay-Davis Modified: 12/08/2016 """ # read parameters from config file indir = config.get('paths', 'archive_dir_ocn') streams_filename = config.get('input', 'ocean_streams_filename') streams = StreamsFile(streams_filename, streamsdir=indir) # get a list of timeSeriesStats output files from the streams file, # reading only those that are between the start and end dates startDate = config.get('time', 'climo_start_date') endDate = config.get('time', 'climo_end_date') streamName = streams.find_stream(streamMap['timeSeriesStats']) infiles = streams.readpath(streamName, startDate=startDate, endDate=endDate) print 'Reading files {} through {}'.format(infiles[0], infiles[-1]) plots_dir = config.get('paths', 'plots_dir') obsdir = config.get('paths', 'obs_' + field + 'dir') casename = config.get('case', 'casename') meshfile = config.get('data', 'mpas_meshfile') climo_yr1 = config.getint('time', 'climo_yr1') climo_yr2 = config.getint('time', 'climo_yr2') yr_offset = config.getint('time', 'yr_offset') outputTimes = config.getExpression(field + '_modelvsobs', 'comparisonTimes') f = netcdf_dataset(meshfile, mode='r') lonCell = f.variables["lonCell"][:] latCell = f.variables["latCell"][:] varList = [field] if field == 'mld': selvals = None # Load MLD observational data obs_filename = "{}/holtetalley_mld_climatology.nc".format(obsdir) dsData = xr.open_mfdataset(obs_filename) # Increment month value to be consistent with the model output dsData.iMONTH.values += 1 # Rename the time dimension to be consistent with the SST dataset dsData.rename({'month': 'calmonth'}, inplace=True) dsData.rename({'iMONTH': 'month'}, inplace=True) obsFieldName = 'mld_dt_mean' # Reorder dataset for consistence dsData = dsData.transpose('month', 'iLON', 'iLAT') # Set appropriate MLD figure labels obsTitleLabel = "Observations (HolteTalley density threshold MLD)" fileOutLabel = "mldHolteTalleyARGO" unitsLabel = 'm' elif field == 'sst': selvals = {'nVertLevels': 0} obs_filename = \ "{}/MODEL.SST.HAD187001-198110.OI198111-201203.nc".format(obsdir) dsData = xr.open_mfdataset(obs_filename) # Select years for averaging (pre-industrial or present-day) # This seems fragile as definitions can change if yr_offset < 1900: time_start = datetime.datetime(1870, 1, 1) time_end = datetime.datetime(1900, 12, 31) preIndustrial_txt = "pre-industrial 1870-1900" else: time_start = datetime.datetime(1990, 1, 1) time_end = datetime.datetime(2011, 12, 31) preIndustrial_txt = "present-day 1990-2011" ds_tslice = dsData.sel(time=slice(time_start, time_end)) monthly_clim_data = ds_tslice.groupby('time.month').mean('time') # Rename the observation data for code compactness dsData = monthly_clim_data.transpose('month', 'lon', 'lat') obsFieldName = 'SST' # Set appropriate figure labels for SST obsTitleLabel = \ "Observations (Hadley/OI, {})".format(preIndustrial_txt) fileOutLabel = "sstHADOI" unitsLabel = r'$^o$C' elif field == 'sss': selvals = {'nVertLevels': 0} obs_filename = "{}/Aquarius_V3_SSS_Monthly.nc".format(obsdir) dsData = xr.open_mfdataset(obs_filename) time_start = datetime.datetime(2011, 8, 1) time_end = datetime.datetime(2014, 12, 31) ds_tslice = dsData.sel(time=slice(time_start, time_end)) # The following line converts from DASK to numpy to supress an odd # warning that doesn't influence the figure output ds_tslice.SSS.values monthly_clim_data = ds_tslice.groupby('time.month').mean('time') # Rename the observation data for code compactness dsData = monthly_clim_data.transpose('month', 'lon', 'lat') obsFieldName = 'SSS' # Set appropriate figure labels for SSS preIndustrial_txt = "2011-2014" obsTitleLabel = "Observations (Aquarius, {})".format(preIndustrial_txt) fileOutLabel = 'sssAquarius' unitsLabel = 'PSU' ds = xr.open_mfdataset( infiles, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset, timestr='Time', onlyvars=varList, selvals=selvals, varmap=variableMap)) ds = remove_repeated_time_index(ds) time_start = datetime.datetime(yr_offset+climo_yr1, 1, 1) time_end = datetime.datetime(yr_offset+climo_yr2, 12, 31) ds_tslice = ds.sel(Time=slice(time_start, time_end)) monthly_clim = ds_tslice.groupby('Time.month').mean('Time') latData, lonData = np.meshgrid(dsData.lat.values, dsData.lon.values) latData = latData.flatten() lonData = lonData.flatten() daysarray = np.ones((12, dsData[obsFieldName].values.shape[1], dsData[obsFieldName].values.shape[2])) for i, dval in enumerate(constants.dinmonth): daysarray[i, :, :] = dval inds = np.where(np.isnan(dsData[obsFieldName][i, :, :].values)) daysarray[i, inds[0], inds[1]] = np.NaN # initialize interpolation variables d2, inds2, lonTarg, latTarg = init_tree(np.rad2deg(lonCell), np.rad2deg(latCell), constants.lonmin, constants.lonmax, constants.latmin, constants.latmax, constants.dLongitude, constants.dLatitude) d, inds, lonTargD, latTargD = init_tree(lonData, latData, constants.lonmin, constants.lonmax, constants.latmin, constants.latmax, constants.dLongitude, constants.dLatitude) nLon = lonTarg.shape[0] nLat = lonTarg.shape[1] modelOutput = np.zeros((len(outputTimes), nLon, nLat)) observations = np.zeros((len(outputTimes), nLon, nLat)) bias = np.zeros((len(outputTimes), nLon, nLat)) # Interpolate and compute biases for i, timestring in enumerate(outputTimes): monthsvalue = constants.monthdictionary[timestring] if isinstance(monthsvalue, (int, long)): modeldata = monthly_clim.sel(month=monthsvalue)[field].values obsdata = dsData.sel(month=monthsvalue)[obsFieldName].values else: modeldata = (np.sum( constants.dinmonth[monthsvalue-1] * monthly_clim.sel(month=monthsvalue)[field].values.T, axis=1) / np.sum(constants.dinmonth[monthsvalue-1])) obsdata = (np.nansum( daysarray[monthsvalue-1, :, :] * dsData.sel(month=monthsvalue)[obsFieldName].values, axis=0) / np.nansum(daysarray[monthsvalue-1, :, :], axis=0)) modelOutput[i, :, :] = interp_fields(modeldata, d2, inds2, lonTarg) observations[i, :, :] = interp_fields(obsdata.flatten(), d, inds, lonTargD) for i in range(len(outputTimes)): bias[i, :, :] = modelOutput[i, :, :] - observations[i, :, :] clevsModelObs = config.getExpression(field + '_modelvsobs', 'clevsModelObs') cmap = plt.get_cmap(config.get(field + '_modelvsobs', 'cmapModelObs')) cmapIndices = config.getExpression(field + '_modelvsobs', 'cmapIndicesModelObs') cmapModelObs = cols.ListedColormap(cmap(cmapIndices), "cmapModelObs") clevsDiff = config.getExpression(field + '_modelvsobs', 'clevsDiff') cmap = plt.get_cmap(config.get(field + '_modelvsobs', 'cmapDiff')) cmapIndices = config.getExpression(field + '_modelvsobs', 'cmapIndicesDiff') cmapDiff = cols.ListedColormap(cmap(cmapIndices), "cmapDiff") for i in range(len(outputTimes)): fileout = "{}/{}_{}_{}_years{:04d}-{:04d}.png".format( plots_dir, fileOutLabel, casename, outputTimes[i], climo_yr1, climo_yr2) title = "{} ({}, years {:04d}-{:04d})".format( field.upper(), outputTimes[i], climo_yr1, climo_yr2) plot_global_comparison(config, lonTarg, latTarg, modelOutput[i, :, :], observations[i, :, :], bias[i, :, :], cmapModelObs, clevsModelObs, cmapDiff, clevsDiff, fileout=fileout, title=title, modelTitle="{}".format(casename), obsTitle=obsTitleLabel, diffTitle="Model-Observations", cbarlabel=unitsLabel)
def load_level4_granule(variable, datasetId='', name=''): '''Loads a Level4 gridded Dataset from PODAAC :param variable: The name of the variable to read from the dataset. :type variable: :mod:`string` :param datasetId: dataset persistent ID. datasetId or \ shortName is required for a granule search. Example: \ PODAAC-ASOP2-25X01 :type datasetId: :mod:`string` :param shortName: the shorter name for a dataset. \ Either shortName or datasetId is required for a \ granule search. Example: ASCATA-L2-25km :type shortName: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :returns: A :class:`dataset.Dataset` containing the dataset pointed to by the OpenDAP URL. :raises: ServerError ''' # Downloading the dataset using podaac toolkit podaac = Podaac() path = os.path.dirname(os.path.abspath(__file__)) granuleName = podaac.extract_l4_granule( dataset_id=datasetId, path=path) path = path + '/' + granuleName d = netcdf_dataset(path, mode='r') dataset = d.variables[variable] # By convention, but not by standard, if the dimensions exist, they will be in the order: # time (t), altitude (z), latitude (y), longitude (x) # but conventions aren't always followed and all dimensions aren't always present so # see if we can make some educated deductions before defaulting to just pulling the first three # columns. temp_dimensions = list(map(lambda x: x.lower(), dataset.dimensions)) dataset_dimensions = dataset.dimensions time = dataset_dimensions[temp_dimensions.index( 'time') if 'time' in temp_dimensions else 0] lat = dataset_dimensions[temp_dimensions.index( 'lat') if 'lat' in temp_dimensions else 1] lon = dataset_dimensions[temp_dimensions.index( 'lon') if 'lon' in temp_dimensions else 2] # Time is given to us in some units since an epoch. We need to convert # these values to datetime objects. Note that we use the main object's # time object and not the dataset specific reference to it. We need to # grab the 'units' from it and it fails on the dataset specific object. times = np.array(convert_times_to_datetime(d[time])) lats = np.array(d.variables[lat][:]) lons = np.array(d.variables[lon][:]) values = np.array(dataset[:]) origin = { 'source': 'PO.DAAC', 'url': 'podaac.jpl.nasa.gov/ws' } # Removing the downloaded temporary granule before creating the OCW # dataset. d.close() path = os.path.join(os.path.dirname(__file__), granuleName) os.remove(path) return Dataset(lats, lons, times, values, variable, name=name, origin=origin)
import numpy as np # parameters from get_parameters import get_area_mean_min_max #def lon_lat_contour_model_vs_model(varnm,season,scale_ctl,scale_exp,table): # data path ctl_name = "CTL" #os.environ["ctl_name"] exp_name = "TSIS" #os.environ["exp_name"] fpath_ctl = '/raid00/xianwen/data/cesm211_solar_exp/solar_CTL_cesm211_ETEST-f19_g17-ens_mean_2010-2019/climo/' fpath_exp = '/raid00/xianwen/data/cesm211_solar_exp/solar_TSIS_cesm211_ETEST-f19_g17-ens_mean_2010-2019/climo/' f1 = fpath_ctl + "solar_CTL_cesm211_ETEST-f19_g17-ens_mean_2010-2019_climo_ANN.nc" f2 = fpath_exp + "solar_TSIS_cesm211_ETEST-f19_g17-ens_mean_2010-2019_climo_ANN.nc" # open data file file_ctl = netcdf_dataset(f1, "r") file_exp = netcdf_dataset(f2, "r") # read lat and lon lat = file_ctl.variables["lat"] lon = file_ctl.variables["lon"] lev = file_ctl.variables["lev"] nlat = len(lat) nlon = len(lon) nlev = len(lev) print(nlat, nlon, nlev) #varnm="FSSDCLRS14" varnm = "CLDLOW" # #varnm_off="FLUTC_OFF" #offline computation #units=r"W/m$^2$"
means_yby_exp_sfc = np.zeros( (years.size, varnms_sfc.size)) #year by year mean for each variable means_ctl_sfc = np.zeros((varnms_sfc.size)) #multi-year mean for each variable means_exp_sfc = np.zeros((varnms_sfc.size)) #multi-year mean for each variable diffs_sfc = np.zeros( (varnms_sfc.size)) #multi-year exp-ctl diff for each variable pvals_sfc = np.zeros((varnms_sfc.size)) #pvalues of ttest for iy in range(0, years.size): # open data file fctl = fpath_ctl + ctl_pref + "_ANN_" + str(years[iy]) + ".nc" fexp = fpath_exp + exp_pref + "_ANN_" + str(years[iy]) + ".nc" fctl_fssd = fpath_ctl_fssd + ctl_fssd_pref + "_climo_ANN.nc" fexp_fssd = fpath_exp_fssd + exp_fssd_pref + "_climo_ANN.nc" file_ctl = netcdf_dataset(fctl, "r") file_exp = netcdf_dataset(fexp, "r") file_ctl_fssd = netcdf_dataset(fctl_fssd, "r") file_exp_fssd = netcdf_dataset(fexp_fssd, "r") # read lat and lon lat = file_ctl.variables["lat"] lon = file_ctl.variables["lon"] # read data and calculate mean/min/max for iv in range(0, varnms_toa.size): dtctl_toa=file_ctl_fssd.variables[varnms_toa[iv]][:,:,:] - \ file_ctl.variables[varnms_sub_toa[iv]][:,:,:] dtexp_toa=file_exp_fssd.variables[varnms_toa[iv]][:,:,:] - \ file_exp.variables[varnms_sub_toa[iv]][:,:,:] dtctl_sfc=file_ctl.variables[varnms_sfc[iv]][:,:,:] - \
from timeit import default_timer as timer from datetime import date, datetime, timedelta start = timer() filelist = sorted(os.listdir('/home/scarani/Desktop/data/goes/001/')) channel = 2 for i in range(1200, 1501, 1): print(i) file = str('/home/scarani/Desktop/data/goes/001/' + filelist[i]) filename = os.path.join(os.path.dirname(ccrs.__file__), 'data', 'netcdf', file) nc = netcdf_dataset(filename) sat_height = nc.variables[ 'goes_imager_projection'].perspective_point_height x = nc.variables['x'][:].data * sat_height y = nc.variables['y'][:].data * sat_height c = nc.variables['Rad'][:] data = nc.variables['Rad'] satvar = nc.variables.keys() time = nc['t'] proj_var = nc.variables[nc.variables['Rad'].grid_mapping] globe = ccrs.Globe(ellipse='sphere', semimajor_axis=proj_var.semi_major_axis,
from cartopy import config import cartopy.crs as ccrs from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER ################# #open and read the dataset, save an iterable range of times #note to zinka: time iterable is fricken stupid as it stands ################# #create filepath to save png files to filepath = 'PNG_files/' if os.path.isdir(filepath) == False: os.mkdir(filepath) dataset = netcdf_dataset('noaacwBLENDEDsstDaily_8423_ee32_0effevan.nc') time = dataset.variables['analysed_sst'][:, 0, 0] time = len(time) time = np.arange(time) time = np.asarray(time) print(time) ################# # Getting date labels to put in the images: ################# time_label = dataset.variables[ 'time'][:] #Time is in epoch time, need to convert it to human readable time #Use datetime to convert to human readable: time_list = [] for x in time: lab = datetime.datetime.fromtimestamp(
import os import matplotlib.pyplot as plt from netCDF4 import Dataset as netcdf_dataset import cartopy.crs as ccrs DIRBIN = os.path.dirname(os.path.abspath(__file__)) infile = os.path.join( DIRBIN, "b.e11.BRCP85C5CNBDRD.f09_g16.021.cice.h.hi_nh.208101-210012.nc" ) dataset = netcdf_dataset(infile) sst = dataset.variables['hi'][0, :, :] lats = dataset.variables['TLAT'][:] lons = dataset.variables['TLON'][:] ax = plt.axes(projection=ccrs.PlateCarree()) plt.contourf(lons, lats, sst, 60, transform=ccrs.PlateCarree()) ax.coastlines() plt.show()
import numpy as np # parameters from get_parameters import get_area_mean_min_max #def lon_lat_contour_model_vs_model(varnm,season,scale_ctl,scale_exp,table): # data path ctl_name = "CTL" #os.environ["ctl_name"] exp_name = "TSIS" #os.environ["exp_name"] fpath_ctl = '/Volumes/WD4T_1/cesm2_solar_exp/solar_CTL_cesm211_ETEST-f19_g17-ens_mean_2010-2019/climo/' #fpath_exp='/Volumes/WD4T_1/cesm2_solar_exp/solar_TSIS_cesm211_ETEST-f19_g17-ens_mean_2010-2019/climo/' f1 = fpath_ctl + "solar_CTL_cesm211_ETEST-f19_g17-ens_mean_2010-2019_climo_ANN.nc" #f2=fpath_exp+"solar_TSIS_cesm211_ETEST-f19_g17-ens_mean_2010-2019_climo_ANN.nc" # open data file file_ctl = netcdf_dataset(f1, "r") #file_exp=netcdf_dataset(f2,"r") # read lat and lon lat = file_ctl.variables["lat"] lon = file_ctl.variables["lon"] lev = file_ctl.variables["lev"] nlat = len(lat) nlon = len(lon) nlev = len(lev) #varnm="FSSDCLRS14" varnm = "FSSUS10" # FSSUS10, FSSUS08 varnm2 = "FSSDS10" # FSSDS10, FSSDS08 varnm3 = "FSSUS08" # FSSUS10, FSSUS08
def plotsat(): nc = netcdf_dataset(localfile.name) sat_height = nc.variables[ 'goes_imager_projection'].perspective_point_height x = nc.variables['x'][:].data * sat_height y = nc.variables['y'][:].data * sat_height c = nc.variables['CMI_C02'][:] data = nc.variables['CMI_C02'] satvar = nc.variables.keys() time = nc['t'] proj_var = nc.variables[nc.variables['CMI_C13'].grid_mapping] globe = ccrs.Globe(ellipse='sphere', semimajor_axis=proj_var.semi_major_axis, semiminor_axis=proj_var.semi_minor_axis) proj = ccrs.Geostationary(central_longitude=-75, sweep_axis='x', satellite_height=sat_height, globe=globe) north = y.max() south = y.min() east = x.max() west = x.min() x, y = np.meshgrid(x, y) fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(1, 1, 1, projection=proj) ax.set_xlim(west, east) ax.set_ylim(south, north) # vmin = 190 #13 # vmax = 305 #13 # colormap = 'Greys' #13 vmin = 0.03 #02 vmax = 1.2 #02 colormap = 'Greys_r' #02 im = ax.pcolormesh(x, y, c, cmap=colormap, vmin=vmin, vmax=vmax) ax.add_feature(cfeature.STATES, linewidth=2, edgecolor='black') ax.coastlines(resolution='10m', linewidth=1, edgecolor='black') ax.add_feature(cfeature.BORDERS, linewidth=1, edgecolor='black') cbar_ticks = np.arange(vmin, vmax, round(((abs(vmax) - abs(vmin)) / 6), 2)) cbar = fig.colorbar(im, ticks=cbar_ticks, orientation='horizontal', shrink=0.6, pad=.02) cbar.ax.set_yticklabels(str(cbar_ticks)) cbar.set_label(data.units, rotation=0, labelpad=5, fontsize=13) # Figure Text txt = open( '/home/scarani/Desktop/precipitation-onset/GOES/channel_title.txt', "r") titles = txt.readlines() channel = data.ancillary_variables[-2:] ch = titles[int(channel)][:-1] #times = str(str(.time.values)[:-10]+ 'UTC') orbital_slot = nc.orbital_slot fmt = '%Y%m%d_%H%M%S' epoch = '20000101_120000' sd = datetime.strptime(epoch, fmt) times = sd + timedelta(seconds=int(time[0].data)) time_title = times.strftime('%Y-%m-%dT%H:%M:%SUTC') savetime = times.strftime('%Y%m%d%H%M%S') sector = '***sector error***' if nc.dataset_name[15:-58] == 'M1': sector = 'Mesoscale-1' if nc.dataset_name[15:-58] == 'M2': sector = 'Mesoscale-2' if nc.dataset_name[15:-58] == 'F': sector = 'Disk' if nc.dataset_name[15:-58] == 'C': sector = 'CONUS' ax.set_title(orbital_slot + ': ' + sector + '\n' + ch + '\n' + str(time_title), fontsize=17) plt.savefig(savelocation + str(savetime) + '_' + str(scan) + '_' + str(channel) + '.png', bbox_inches='tight', dpi=300) plt.show() plt.close() gc.collect() mid = timer() print(round((mid - start) / 60, 2))
'clevsDiff_conc_{}'.format(season)) cmap = plt.get_cmap(config.get('seaice_modelvsobs', 'cmapDiff')) cmapIndices = config.getExpression('seaice_modelvsobs', 'cmapIndicesDiff') cmapDiff = cols.ListedColormap(cmap(cmapIndices), "cmapDiff") lon0 = config.getfloat('seaice_modelvsobs', 'lon0_{}'.format(hemisphere)) latmin = config.getfloat('seaice_modelvsobs', 'latmin_{}'.format(hemisphere)) # Load in sea-ice data # Model... # ice concentrations fileName = "{}/{}".format(climodir_regridded, climofiles[climName]) f = netcdf_dataset(fileName, mode='r') iceconc = f.variables["iceAreaCell"][:] if (first): lons = f.variables["lon"][:] lats = f.variables["lat"][:] print "Min lon: ", np.amin(lons), "Max lon: ", np.amax(lons) print "Min lat: ", np.amin(lats), "Max lat: ", np.amax(lats) Lons, Lats = np.meshgrid(lons, lats) first = False f.close() # ...and observations # ice concentrations from NASATeam (or Bootstrap) algorithm for obsName in ['NASATeam', 'Bootstrap']: fileName = obs_iceconc_filenames['{}_{}'.format(climName, obsName)]
def getData(dataset_id, parameter, qbounds, time): #print(qbounds) #print(parameter) #print(time) lat1 = qbounds[0] lat2 = qbounds[1] lon1 = qbounds[2] lon2 = qbounds[3] lat_order = qbounds[4] altf = qbounds[5] sub = qbounds[6] time1 = time # using just one timestamp for this example time2 = time if altf == 1: #write query with altitude, haven't tested this alt1 = float(qbounds[7]) alt2 = float(qbounds[8]) altsub = 1 query = parameter+'[(%s):%s:(%s)][(%f):%s:(%f)][(%f):%s:(%f)][(%f):%s:(%f)]' % (time1,sub,time2,alt1, altsub, alt2, lat1,sub,lat2,lon1,sub,lon2) else: #write query without altitude query = parameter+'[(%s):%s:(%s)][(%f):%s:(%f)][(%f):%s:(%f)]' % (time1,sub,time2,lat1,sub,lat2,lon1,sub,lon2) base_url = 'http://polarwatch.noaa.gov/erddap/griddap/'+ dataset_id +'.nc?' url = base_url + query print(url) datasetObj = { "url" : url } file = 'dataset.nc' http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) try: r = http.request('GET', url, preload_content=False) except urllib3.exceptions.HTTPError as he: # Update config file to show that the dataset is not valid datasetObj["inERDDAP"] = 0 datasetObj["inERDDAPReason"] = "Could not access actual data when trying to make plots" entry["validDatasets"][subEntry] = 0 logger.error(' * The server couldnt fullfill the request. Adding a not valid flag and not updating graphics today.') except urllib3.exceptions.URLError as e: logger.error(' * Failed to reach erddap server. Stopping.') logger.error(' * reason: ', e.reason) datasetObj["inERDDAP"] = 0 datasetObj["inERDDAPReason"] = "Failed to reach ERDDAP when trying to make plots" entry["validDatasets"][subEntry] = 0 except Exception: logger.error('generic exception: ' + traceback.format_exc()) datasetObj["inERDDAP"] = 0 datasetObj["inERDDAPReason"] = "General Error when trying to make plots (likely timeout)" else: # update the file with the new data with open(file, 'wb') as out: while True: data = r.read(1024*1024) if not data: break out.write(data) r.release_conn() print ('Satellite Data File Retrieved') #Check netcdf response try: data_netcdf_dataset = netcdf_dataset(file) except Exception: print(' an unusual error has occured') datasetObj["inERDDAP"] = 0 datasetObj["inERDDAPReason"] = "Valid NetCDF not returned in make images" getDataError = 'generic exception: ' + traceback.format_exc() datasetObj["goodFile"] = 0 datasetObj["error"] = getDataError datasetObj["data_netcdf_dataset"] = 0 else: print('retrieved valid netcdf') datasetObj["data_netcdf_dataset"] = data_netcdf_dataset datasetObj["goodFile"] = 1 datasetObj["inERDDAP"] = 1 return datasetObj
def getClosestIceMask(logger, date_str): print('Date of data: '+ date_str) date_datetime= dateutil.parser.parse(date_str) date_datetime.replace(tzinfo=timezone('UTC')) searchStartDate_datetime = date_datetime - timedelta(days=33) searchStartDate_str = searchStartDate_datetime.strftime("%Y-%m-%dT%I:%M:%SZ") searchEndDate_datetime = date_datetime + timedelta(days=33) searchEndDate_str = searchEndDate_datetime.strftime("%Y-%m-%dT%I:%M:%SZ") # See if we need to modify these times based # on the extent of the ice mask dataset time range # because erddap throws an error if the date is too # far into the future outside of the range of the dataset maskInfoUrl = 'https://polarwatch.noaa.gov/erddap/info/jplMURSST41/index.csv' http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) try: r = http.request('GET', maskInfoUrl, preload_content=False, timeout=20) except: logger.error(" * Error: Retrieving ice mask info file") else: reader = csv.reader(io.StringIO(r.read().decode('utf-8')), delimiter=',') for row in reader: if row[0] == "attribute" and row[2] == "time_coverage_end": maxIceMaskDate_datetime = dateutil.parser.parse(row[4]) maxIceMaskDate_datetime.replace(tzinfo=timezone('UTC')) # compare times, replace if needed if maxIceMaskDate_datetime < searchEndDate_datetime: print('replacing') searchEndDate_str = maxIceMaskDate_datetime.strftime("%Y-%m-%dT%I:%M:%SZ") # create a plus or minus two months on that date to grab a limited section of the times # get ice mask times from erddap so we can search and find the closest one iceMaskTimesUrl = 'https://polarwatch.noaa.gov/erddap/griddap/jplMURSST41.json?time[(' + searchStartDate_str + '):1:(' + searchEndDate_str+ ')]' try: response = http.request('GET', iceMaskTimesUrl, timeout=15) except urllib3.exceptions.HTTPError as e: logger.error('The server couldnt fullfill the request.') logger.error('Error Code: ', e.code) except urllib3.exceptions.URLError as e: logger.error('Failed to reach erddap server for: '+ iceMaskTimesUrl) logger.error('reason: ', e.reason) else: timesResponse = json.loads(response.data.decode('utf-8')) try: timesResponse = json.loads(response.data.decode('utf-8')) except: print('couldn''t read json') gotTimes=0 else: timeList_datetime = [] for val in timesResponse['table']['rows']: val_datetime = dateutil.parser.parse(val[0]) val_datetime.replace(tzinfo=timezone('UTC')) timeList_datetime.append(val_datetime) gotTimes =1 # find the closest time if gotTimes == 1 : timeList_datetime = np.array(timeList_datetime) timeList_datetime.sort datematch = np.searchsorted(timeList_datetime, date_datetime ) closest_date_str = timeList_datetime[datematch-1].strftime("%Y-%m-%dT%I:%M:%SZ") print('Date of ice mask request: '+ closest_date_str) #now request that mask maskUrl = 'https://polarwatch.noaa.gov/erddap/griddap/jplMURSST41.nc?mask[('+closest_date_str+'):1:('+closest_date_str+')][(-89.990000):35:(89.990000)][(-179.990000):71:(180.000000)]' print(maskUrl) try: r = http.request('GET', maskUrl, preload_content=False, timeout=70) except urllib3.exceptions.HTTPError as he: print('HTTP ERROR: the server couldnt fulfill the request') print(he) except urllib3.exceptions.URLError as e: logger.error(' * Failed to reach erddap server. Stopping.') logger.error(' * reason: ', e.reason) except Exception: logger.error('generic exception: ' + traceback.format_exc()) else: with open('icemask.nc', 'wb') as out: while True: icedata = r.read(1024*1024) if not icedata: break out.write(icedata) r.release_conn() # Try opening the returned mask data file to confirm we retrieved valid data maskNetCDF = netcdf_dataset('icemask.nc') return maskNetCDF
import os import matplotlib.pyplot as plt from netCDF4 import Dataset as netcdf_dataset import numpy as np from cartopy import config import cartopy.crs as ccrs dataset = netcdf_dataset('data.nc') dimensions = dataset.variables.keys() sst = dataset.variables['u10'][0, :, :] lats = dataset.variables['latitude'][:] lons = dataset.variables['longitude'][:] print(sst, lats, lons) ax = plt.axes(projection=ccrs.PlateCarree()) plt.contourf(lons, lats, sst, 60, transform=ccrs.PlateCarree()) ax.coastlines() plt.show()
def ohc_timeseries(config, streamMap=None, variableMap=None): """ Performs analysis of ocean heat content (OHC) from time-series output. config is an instance of an MpasAnalysisConfigParser containing configuration options. config is an instance of MpasAnalysisConfigParser containing configuration options. If present, streamMap is a dictionary of MPAS-O stream names that map to their mpas_analysis counterparts. If present, variableMap is a dictionary of MPAS-O variable names that map to their mpas_analysis counterparts. Author: Xylar Asay-Davis, Milena Veneziani Last Modified: 12/04/2016 """ # read parameters from config file indir = config.get('paths', 'archive_dir_ocn') namelist_filename = config.get('input', 'ocean_namelist_filename') namelist = NameList(namelist_filename, path=indir) streams_filename = config.get('input', 'ocean_streams_filename') streams = StreamsFile(streams_filename, streamsdir=indir) # Note: input file, not a mesh file because we need dycore specific fields # such as refBottomDepth and namelist fields such as config_density0 that # are not guaranteed to be in the mesh file. inputfile = streams.readpath('input')[0] # get a list of timeSeriesStats output files from the streams file, # reading only those that are between the start and end dates startDate = config.get('time', 'timeseries_start_date') endDate = config.get('time', 'timeseries_end_date') streamName = streams.find_stream(streamMap['timeSeriesStats']) infiles = streams.readpath(streamName, startDate=startDate, endDate=endDate) print 'Reading files {} through {}'.format(infiles[0], infiles[-1]) casename = config.get('case', 'casename') ref_casename_v0 = config.get('case', 'ref_casename_v0') indir_v0data = config.get('paths', 'ref_archive_v0_ocndir') compare_with_obs = config.getboolean('ohc_timeseries', 'compare_with_obs') plots_dir = config.get('paths', 'plots_dir') yr_offset = config.getint('time', 'yr_offset') N_movavg = config.getint('ohc_timeseries', 'N_movavg') regions = config.getExpression('regions', 'regions') plot_titles = config.getExpression('regions', 'plot_titles') iregions = config.getExpression('ohc_timeseries', 'regionIndicesToPlot') # Define/read in general variables print " Read in depth and compute specific depth indexes..." f = netcdf_dataset(inputfile, mode='r') # reference depth [m] depth = f.variables["refBottomDepth"][:] # specific heat [J/(kg*degC)] cp = namelist.getfloat('config_specific_heat_sea_water') # [kg/m3] rho = namelist.getfloat('config_density0') fac = 1e-22*rho*cp k700m = np.where(depth > 700.)[0][0] - 1 k2000m = np.where(depth > 2000.)[0][0] - 1 kbtm = len(depth)-1 # Load data print " Load ocean data..." varList = ['avgLayerTemperature', 'sumLayerMaskValue', 'avgLayerArea', 'avgLayerThickness'] ds = xr.open_mfdataset( infiles, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset, timestr='Time', onlyvars=varList, varmap=variableMap)) ds = remove_repeated_time_index(ds) # convert the start and end dates to datetime objects using # the Date class, which ensures the results are within the # supported range time_start = Date(startDate).to_datetime(yr_offset) time_end = Date(endDate).to_datetime(yr_offset) # select only the data in the specified range of years ds = ds.sel(Time=slice(time_start, time_end)) # Select year-1 data and average it (for later computing anomalies) time_start = datetime.datetime(time_start.year, 1, 1) time_end = datetime.datetime(time_start.year, 12, 31) ds_yr1 = ds.sel(Time=slice(time_start, time_end)) mean_yr1 = ds_yr1.mean('Time') print " Compute temperature anomalies..." avgLayerTemperature = ds.avgLayerTemperature avgLayerTemperature_yr1 = mean_yr1.avgLayerTemperature avgLayTemp_anomaly = avgLayerTemperature - avgLayerTemperature_yr1 year_start = (pd.to_datetime(ds.Time.min().values)).year year_end = (pd.to_datetime(ds.Time.max().values)).year time_start = datetime.datetime(year_start, 1, 1) time_end = datetime.datetime(year_end, 12, 31) if ref_casename_v0 != "None": print " Load in OHC for ACMEv0 case..." infiles_v0data = "{}/OHC.{}.year*.nc".format( indir_v0data, ref_casename_v0) ds_v0 = xr.open_mfdataset( infiles_v0data, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset)) ds_v0 = remove_repeated_time_index(ds_v0) ds_v0_tslice = ds_v0.sel(Time=slice(time_start, time_end)) sumLayerMaskValue = ds.sumLayerMaskValue avgLayerArea = ds.avgLayerArea avgLayerThickness = ds.avgLayerThickness print " Compute OHC and make plots..." for index in range(len(iregions)): iregion = iregions[index] # Compute volume of each layer in the region: layerArea = sumLayerMaskValue[:, iregion, :] * \ avgLayerArea[:, iregion, :] layerVolume = layerArea * avgLayerThickness[:, iregion, :] # Compute OHC: ohc = layerVolume * avgLayTemp_anomaly[:, iregion, :] # OHC over 0-bottom depth range: ohc_tot = ohc.sum('nVertLevels') ohc_tot = fac*ohc_tot # OHC over 0-700m depth range: ohc_700m = fac*ohc[:, 0:k700m].sum('nVertLevels') # OHC over 700m-2000m depth range: ohc_2000m = fac*ohc[:, k700m+1:k2000m].sum('nVertLevels') # OHC over 2000m-bottom depth range: ohc_btm = ohc[:, k2000m+1:kbtm].sum('nVertLevels') ohc_btm = fac*ohc_btm title = 'OHC, {}, 0-bottom (thick-), 0-700m (thin-), 700-2000m (--),' \ ' 2000m-bottom (-.) \n {}'.format(plot_titles[iregion], casename) xlabel = "Time [years]" ylabel = "[x$10^{22}$ J]" if ref_casename_v0 != "None": figname = "{}/ohc_{}_{}_{}.png".format(plots_dir, regions[iregion], casename, ref_casename_v0) ohc_v0_tot = ds_v0_tslice.ohc_tot ohc_v0_700m = ds_v0_tslice.ohc_700m ohc_v0_2000m = ds_v0_tslice.ohc_2000m ohc_v0_btm = ds_v0_tslice.ohc_btm title = "{} (r), {} (b)".format(title, ref_casename_v0) timeseries_analysis_plot(config, [ohc_tot, ohc_700m, ohc_2000m, ohc_btm, ohc_v0_tot, ohc_v0_700m, ohc_v0_2000m, ohc_v0_btm], N_movavg, title, xlabel, ylabel, figname, lineStyles=['r-', 'r-', 'r--', 'r-.', 'b-', 'b-', 'b--', 'b-.'], lineWidths=[2, 1, 1.5, 1.5, 2, 1, 1.5, 1.5]) if not compare_with_obs and ref_casename_v0 == "None": figname = "{}/ohc_{}_{}.png".format(plots_dir, regions[iregion], casename) timeseries_analysis_plot(config, [ohc_tot, ohc_700m, ohc_2000m, ohc_btm], N_movavg, title, xlabel, ylabel, figname, lineStyles=['r-', 'r-', 'r--', 'r-.'], lineWidths=[2, 1, 1.5, 1.5])
def ohc_timeseries(config, streamMap=None, variableMap=None): """ Performs analysis of ocean heat content (OHC) from time-series output. config is an instance of an MpasAnalysisConfigParser containing configuration options. config is an instance of MpasAnalysisConfigParser containing configuration options. If present, streamMap is a dictionary of MPAS-O stream names that map to their mpas_analysis counterparts. If present, variableMap is a dictionary of MPAS-O variable names that map to their mpas_analysis counterparts. Author: Xylar Asay-Davis, Milena Veneziani Last Modified: 01/07/2017 """ # read parameters from config file casename = config.get('case', 'casename') ref_casename_v0 = config.get('case', 'ref_casename_v0') indir_v0data = config.get('paths', 'ref_archive_v0_ocndir') compare_with_obs = config.getboolean('ohc_timeseries', 'compare_with_obs') plots_dir = config.get('paths', 'plots_dir') yr_offset = config.getint('time', 'yr_offset') N_movavg = config.getint('ohc_timeseries', 'N_movavg') regions = config.getExpression('regions', 'regions') plot_titles = config.getExpression('regions', 'plot_titles') iregions = config.getExpression('ohc_timeseries', 'regionIndicesToPlot') indir = config.get('paths', 'archive_dir_ocn') namelist_filename = config.get('input', 'ocean_namelist_filename') namelist = NameList(namelist_filename, path=indir) streams_filename = config.get('input', 'ocean_streams_filename') streams = StreamsFile(streams_filename, streamsdir=indir) # Note: input file, not a mesh file because we need dycore specific fields # such as refBottomDepth and namelist fields such as config_density0, as # well as simulationStartTime, that are not guaranteed to be in the mesh file. try: inputfile = streams.readpath('restart')[0] except ValueError: raise IOError( 'No MPAS-O restart file found: need at least one restart file for OHC calculation' ) # get a list of timeSeriesStats output files from the streams file, # reading only those that are between the start and end dates startDate = config.get('time', 'timeseries_start_date') endDate = config.get('time', 'timeseries_end_date') streamName = streams.find_stream(streamMap['timeSeriesStats']) infiles = streams.readpath(streamName, startDate=startDate, endDate=endDate) print 'Reading files {} through {}'.format(infiles[0], infiles[-1]) # Define/read in general variables print ' Read in depth and compute specific depth indexes...' f = netcdf_dataset(inputfile, mode='r') # reference depth [m] depth = f.variables['refBottomDepth'][:] # simulation start time simStartTime = netCDF4.chartostring(f.variables['simulationStartTime'][:]) simStartTime = str(simStartTime) f.close() # specific heat [J/(kg*degC)] cp = namelist.getfloat('config_specific_heat_sea_water') # [kg/m3] rho = namelist.getfloat('config_density0') fac = 1e-22 * rho * cp k700m = np.where(depth > 700.)[0][0] - 1 k2000m = np.where(depth > 2000.)[0][0] - 1 kbtm = len(depth) - 1 # Load data print ' Load ocean data...' varList = [ 'avgLayerTemperature', 'sumLayerMaskValue', 'avgLayerArea', 'avgLayerThickness' ] ds = xr.open_mfdataset( infiles, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset, timestr='Time', onlyvars=varList, varmap=variableMap)) ds = remove_repeated_time_index(ds) # convert the start and end dates to datetime objects using # the Date class, which ensures the results are within the # supported range time_start = Date(startDate).to_datetime(yr_offset) time_end = Date(endDate).to_datetime(yr_offset) # select only the data in the specified range of years ds = ds.sel(Time=slice(time_start, time_end)) # Select year-1 data and average it (for later computing anomalies) time_start_yr1 = Date(simStartTime).to_datetime(yr_offset) if time_start_yr1 < time_start: startDate_yr1 = simStartTime endDate_yr1 = startDate_yr1[0:5] + '12-31' + startDate_yr1[10:] infiles_yr1 = streams.readpath(streamName, startDate=startDate_yr1, endDate=endDate_yr1) ds_yr1 = xr.open_mfdataset( infiles_yr1, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset, timestr='Time', onlyvars=varList, varmap=variableMap)) ds_yr1 = remove_repeated_time_index(ds_yr1) else: time_start = datetime.datetime(time_start.year, 1, 1) time_end = datetime.datetime(time_start.year, 12, 31) ds_yr1 = ds.sel(Time=slice(time_start, time_end)) mean_yr1 = ds_yr1.mean('Time') print ' Compute temperature anomalies...' avgLayerTemperature = ds.avgLayerTemperature avgLayerTemperature_yr1 = mean_yr1.avgLayerTemperature avgLayTemp_anomaly = avgLayerTemperature - avgLayerTemperature_yr1 year_start = (pd.to_datetime(ds.Time.min().values)).year year_end = (pd.to_datetime(ds.Time.max().values)).year time_start = datetime.datetime(year_start, 1, 1) time_end = datetime.datetime(year_end, 12, 31) if ref_casename_v0 != 'None': print ' Load in OHC for ACMEv0 case...' infiles_v0data = '{}/OHC.{}.year*.nc'.format(indir_v0data, ref_casename_v0) ds_v0 = xr.open_mfdataset( infiles_v0data, preprocess=lambda x: preprocess_mpas(x, yearoffset=yr_offset)) ds_v0 = remove_repeated_time_index(ds_v0) year_end_v0 = (pd.to_datetime(ds_v0.Time.max().values)).year if year_start <= year_end_v0: ds_v0_tslice = ds_v0.sel(Time=slice(time_start, time_end)) else: print ' Warning: v0 time series lies outside current bounds of v1 time series. Skipping it.' ref_casename_v0 = 'None' sumLayerMaskValue = ds.sumLayerMaskValue avgLayerArea = ds.avgLayerArea avgLayerThickness = ds.avgLayerThickness print ' Compute OHC and make plots...' for index in range(len(iregions)): iregion = iregions[index] # Compute volume of each layer in the region: layerArea = sumLayerMaskValue[:, iregion, :] * \ avgLayerArea[:, iregion, :] layerVolume = layerArea * avgLayerThickness[:, iregion, :] # Compute OHC: ohc = layerVolume * avgLayTemp_anomaly[:, iregion, :] # OHC over 0-bottom depth range: ohc_tot = ohc.sum('nVertLevels') ohc_tot = fac * ohc_tot # OHC over 0-700m depth range: ohc_700m = fac * ohc[:, 0:k700m].sum('nVertLevels') # OHC over 700m-2000m depth range: ohc_2000m = fac * ohc[:, k700m + 1:k2000m].sum('nVertLevels') # OHC over 2000m-bottom depth range: ohc_btm = ohc[:, k2000m + 1:kbtm].sum('nVertLevels') ohc_btm = fac * ohc_btm title = 'OHC, {}, 0-bottom (thick-), 0-700m (thin-), 700-2000m (--),' \ ' 2000m-bottom (-.) \n {}'.format(plot_titles[iregion], casename) xlabel = 'Time [years]' ylabel = '[x$10^{22}$ J]' if ref_casename_v0 != 'None': figname = '{}/ohc_{}_{}_{}.png'.format(plots_dir, regions[iregion], casename, ref_casename_v0) ohc_v0_tot = ds_v0_tslice.ohc_tot ohc_v0_700m = ds_v0_tslice.ohc_700m ohc_v0_2000m = ds_v0_tslice.ohc_2000m ohc_v0_btm = ds_v0_tslice.ohc_btm title = '{} (r), {} (b)'.format(title, ref_casename_v0) timeseries_analysis_plot( config, [ ohc_tot, ohc_700m, ohc_2000m, ohc_btm, ohc_v0_tot, ohc_v0_700m, ohc_v0_2000m, ohc_v0_btm ], N_movavg, title, xlabel, ylabel, figname, lineStyles=[ 'r-', 'r-', 'r--', 'r-.', 'b-', 'b-', 'b--', 'b-.' ], lineWidths=[2, 1, 1.5, 1.5, 2, 1, 1.5, 1.5]) if not compare_with_obs and ref_casename_v0 == 'None': figname = '{}/ohc_{}_{}.png'.format(plots_dir, regions[iregion], casename) timeseries_analysis_plot(config, [ohc_tot, ohc_700m, ohc_2000m, ohc_btm], N_movavg, title, xlabel, ylabel, figname, lineStyles=['r-', 'r-', 'r--', 'r-.'], lineWidths=[2, 1, 1.5, 1.5])
#from get_parameters import * # scipy from scipy import stats # data path ctl_name = "CESM2" #os.environ["ctl_name"] exp_name = "TSIS-1" #os.environ["exp_name"] ctl_pref = "solar_CTL_cesm211_ETEST-f19_g17-ens_mean_2010-2019" #exp_pref="solar_TSIS_cesm211_ETEST-f19_g17-ens_mean_2010-2019" fpath_ctl = "/Volumes/WD4T_1/cesm2_solar_exp/" + ctl_pref + "/climo/" #fpath_exp="/raid00/xianwen/data/cesm211_solar_exp/"+exp_pref+"/climo/" fctl = fpath_ctl + "solar_CTL_cesm211_ETEST-f19_g17-ens_mean_2010-2019_climo_ANN.nc" file_ctl = netcdf_dataset(fctl, "r") #years=np.arange(2010,2020) #months_all=["01","02","03","04","05","06","07","08","09","10","11","12"] #varnm="ICEFRAC" varnm = "FSSUS10" # FSSUS10, FSSUS08 varnm2 = "FSSDS10" # FSSDS10, FSSDS08 varnm3 = "FSSUS08" # FSSUS10, FSSUS08 varnm4 = "FSSDS08" # FSSDS10, FSSDS08 #varnm_off="FLUTC_OFF" #offline computation #units=r"W/m$^2$" units = "" figure_name = "lat_lon_albedo_VIS" dtctl = file_ctl.variables[varnm][0, :, :] / file_ctl.variables[varnm2][
nlev = 32 means_yby_ctl = np.zeros( (years.size, nlat, nlon)) #year by year mean for each variable means_yby_exp = np.zeros( (years.size, nlat, nlon)) #year by year mean for each variable means_ctl = np.zeros((nlat, nlon)) #year by year mean for each variable means_exp = np.zeros((nlat, nlon)) #year by year mean for each variable diffs = np.zeros((nlat, nlon)) #multi-year exp-ctl diff for each variable pvals = np.zeros((nlat, nlon)) #pvalues of ttest for iy in range(0, years.size): # open data file fctl = fpath_ctl + ctl_pref + "_" + season + "_" + str(years[iy]) + ".nc" fexp = fpath_exp + exp_pref + "_" + season + "_" + str(years[iy]) + ".nc" file_ctl = netcdf_dataset(fctl, "r") file_exp = netcdf_dataset(fexp, "r") # read lat and lon lat = file_ctl.variables["lat"] lon = file_ctl.variables["lon"] means_yby_ctl[iy, :, :] = file_ctl.variables[varnm][0, :, :] means_yby_exp[iy, :, :] = file_exp.variables[varnm][0, :, :] means_ctl[:, :] = np.mean(means_yby_ctl, axis=0) means_exp[:, :] = np.mean(means_yby_exp, axis=0) diffs = means_exp - means_ctl if pole == "N": latbound1 = np.min(np.where(lat[:] > 55))
# You can reduce the resolution of the data returned from the server # This can be helpful during testing if the dataset is very large # Set this to one for full resolution, two for half, and so on req["sub"] = '4' print(req["sub"]) # In[8]: # GET DATA # m0 = datetime.now() #dataset = getData(dataset_id, parameter, qbounds, '', req["sub"],basemapImageDir) file = basemapImageDir + '/' + parameter + '.nc' datafile = netcdf_dataset(file) dataset = {"data": datafile} dataObj = dataset['data'] # check data file for expected altitude dimension response # pull out parameter data and x, y if len(np.shape(dataObj.variables[parameter])) == 3: data = dataObj.variables[parameter][0, :, :] elif len(np.shape(dataObj.variables[parameter])) == 4: data = dataObj.variables[parameter][0, 0, :, :] elif len(np.shape(dataObj.variables[parameter])) == 2: data = dataObj.variables[parameter][:, :] xgrid = dataObj.variables[xcoord_dimension["name"]][:] ygrid = dataObj.variables[ycoord_dimension["name"]][:]
import os import matplotlib.pyplot as plt from netCDF4 import Dataset as netcdf_dataset import numpy as np import cartopy.crs as ccrs # get the path of the file. It can be found in the data directory, conveniently # at the same level as the cartopy/crs.py file. fname = os.path.join(os.path.dirname(ccrs.__file__), 'data', 'netcdf', 'HadISST1_SST_update.nc' ) dataset = netcdf_dataset(fname) sst = dataset.variables['sst'][0, :, :] lats = dataset.variables['lat'][:] lons = dataset.variables['lon'][:] ax = plt.axes(projection=ccrs.PlateCarree()) plt.contourf(lons, lats, sst, 60, transform=ccrs.PlateCarree()) ax.coastlines() plt.show()
'clevsDiff_conc_{}'.format(season)) cmap = plt.get_cmap(config.get('seaice_modelvsobs', 'cmapDiff')) cmapIndices = config.getExpression('seaice_modelvsobs', 'cmapIndicesDiff') cmapDiff = cols.ListedColormap(cmap(cmapIndices), "cmapDiff") lon0 = config.getfloat('seaice_modelvsobs', 'lon0_{}'.format(hemisphere)) latmin = config.getfloat('seaice_modelvsobs', 'latmin_{}'.format(hemisphere)) # Load in sea-ice data # Model... # ice concentrations fileName = "{}/{}".format(climodir_regridded, climofiles[climName]) f = netcdf_dataset(fileName, mode='r') iceconc = f.variables["iceAreaCell"][:] if(first): lons = f.variables["lon"][:] lats = f.variables["lat"][:] print "Min lon: ", np.amin(lons), "Max lon: ", np.amax(lons) print "Min lat: ", np.amin(lats), "Max lat: ", np.amax(lats) Lons, Lats = np.meshgrid(lons, lats) first = False f.close() # ...and observations # ice concentrations from NASATeam (or Bootstrap) algorithm for obsName in ['NASATeam', 'Bootstrap']: fileName = obs_iceconc_filenames['{}_{}'.format(climName, obsName)]