def test_input_tmax_and_tmin(): p_min = "/snow3/huziy/Daymet_daily/daymet_v3_tmin_1988_na.nc4" p_max = "/snow3/huziy/Daymet_daily/daymet_v3_tmax_1988_na.nc4" tmx = xarray.open_dataset(p_max)["tmax"][0, :, :] tmn = xarray.open_dataset(p_min)["tmin"][0, :, :] tmn_msk = tmn.to_masked_array() tmx_msk = tmx.to_masked_array() diff = tmx_msk - tmn_msk print(diff.shape) plt.figure() suspect = (diff < 0.) & ~np.isnan(diff) & ~diff.mask if np.any(suspect): print(diff[suspect], len(suspect), diff[suspect].min(), "...", diff[suspect].max()) # print(tmx_msk[suspect]) # print(tavg_msk[suspect]) # print(tmn_msk[suspect]) diff[suspect] = -100 im = plt.pcolormesh(tmn.x, tmn.y, diff, cmap=cm.get_cmap("coolwarm", 20), vmin=-2, vmax=2) plt.colorbar(im) plt.show()
def from_netcdf(netcdf_path): """Load serialized TimeSeries from netCDF file.""" with netCDF4.Dataset(netcdf_path) as ds: channels = list(ds.groups) # First channel group stores time series metadata with xr.open_dataset(netcdf_path, group=channels[0]) as ds: t = [ds.time.values] m = [ds.measurement.values] e = [ds.error.values] target = ds.attrs.get('target') meta_features = ds.meta_features.to_series() name = ds.attrs.get('name') path = ds.attrs.get('path') for channel in channels[1:]: with xr.open_dataset(netcdf_path, group=channel) as ds: m.append(ds.measurement.values) if 'time' in ds: t.append(ds.time.values) if 'error' in ds: e.append(ds.error.values) return TimeSeries(_make_array(t), _make_array(m), _make_array(e), target, meta_features, name, path)
def xrload(file_name, engine="h5netcdf", load_to_mem=True, create_new=False): """ Loads a xarray dataset. Parameters ---------- file_name: name of file engine: engine used to load file load_to_mem: once opened, load from disk to memory create_new: if no file exists make a blank one Returns ------- ds: loaded Dataset """ file_name = auto_add_extension(file_name, engine) try: try: ds = xr.open_dataset(file_name, engine=engine) except AttributeError as e1: if "object has no attribute" in str(e1): ds = xr.open_dataset(file_name, engine="netcdf4") else: raise e1 if load_to_mem: ds.load() ds.close() except (RuntimeError, OSError) as e2: if "o such" in str(e2) and create_new: ds = xr.Dataset() else: raise e2 return ds
def __init__(self,config_name='config.ini',init=False,*args,**kwargs): #Shared parameters as attributes super(FAPAR,self).__init__(config_name,init,*args,**kwargs) #FAPAR parameters, super(FAPAR,self).import_config(config_name,'FAPAR') #Load observation file and rename/reorder dimensions self.obs=xr.open_dataset(self.fnobs,decode_times=False).FAPAR self.obs=self.obs.rename({'TAX':'TIME'}) self.obs=self.obs.transpose('TIME','LATITUDE','LONGITUDE') #Overwrite timeaxis with datetime object self.timeaxis=pd.date_range('1997-09-01',freq='m',periods=106) self.obs['TIME']=self.timeaxis self.obs_mean=self.obs.mean(dim='TIME') #Monthly mean self.obs_monthly=self.obs.groupby('TIME.month').mean(dim='TIME') #Load par from control run self.par=xr.open_dataset(self.fnctrl_m).par.isel(grid_only=0).drop('grid_only') self.tconstraint=(self.par.TIME>1997.65) * (self.par.TIME<2006.5) self.par=self.par[self.tconstraint] self.par['TIME']=self.timeaxis self.name='fapar' #Load area from control run, stack it for seasonal mean self.area=xr.open_dataset(self.fnctrl).area self.area_monthly=xr.concat([self.area]*12,dim='month') self.area_monthly.month.values=range(1,13)
def test_qc(self): nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed', 'nutnr_a_sample') ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed', 'ctdpf_sbe43_sample') nutnr_fn = 'nutnr_a_sample.nc' ctdpf_fn = 'ctdpf_sbe43_sample.nc' cals = json.load(open(os.path.join(DATA_DIR, 'cals.json'))) qc = json.load(open(os.path.join(DATA_DIR, 'qc.json'))) tr = TimeRange(3.65342400e+09, 3.65351040e+09) coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': cals[k], 'deployment': 1}] for k in cals} sr = StreamRequest(nutnr_sk, [2443], coefficients, tr, {}, qc_parameters=qc, request_id='UNIT') nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn), decode_times=False) ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn), decode_times=False) ctdpf_ds = ctdpf_ds[self.base_params + [p.name for p in sr.stream_parameters[ctdpf_sk]]] nutnr_ds = nutnr_ds[self.base_params + [p.name for p in sr.stream_parameters[nutnr_sk]]] sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients, sr.uflags, [nutnr_sk], sr.request_id) sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients, sr.uflags, [ctdpf_sk], sr.request_id) sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds) sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds) sr.calculate_derived_products() expected_parameters = ['temp_sal_corrected_nitrate_qc_executed', 'temp_sal_corrected_nitrate_qc_results'] self.assert_parameters_in_datasets(sr.datasets[nutnr_sk].datasets, expected_parameters)
def test_coordinates_encoding(self): def equals_latlon(obj): return obj == 'lat lon' or obj == 'lon lat' original = Dataset({'temp': ('x', [0, 1]), 'precip': ('x', [0, -1])}, {'lat': ('x', [2, 3]), 'lon': ('x', [4, 5])}) with self.roundtrip(original) as actual: self.assertDatasetIdentical(actual, original) with create_tmp_file() as tmp_file: original.to_netcdf(tmp_file) with open_dataset(tmp_file, decode_coords=False) as ds: self.assertTrue(equals_latlon(ds['temp'].attrs['coordinates'])) self.assertTrue(equals_latlon(ds['precip'].attrs['coordinates'])) self.assertNotIn('coordinates', ds.attrs) self.assertNotIn('coordinates', ds['lat'].attrs) self.assertNotIn('coordinates', ds['lon'].attrs) modified = original.drop(['temp', 'precip']) with self.roundtrip(modified) as actual: self.assertDatasetIdentical(actual, modified) with create_tmp_file() as tmp_file: modified.to_netcdf(tmp_file) with open_dataset(tmp_file, decode_coords=False) as ds: self.assertTrue(equals_latlon(ds.attrs['coordinates'])) self.assertNotIn('coordinates', ds['lat'].attrs) self.assertNotIn('coordinates', ds['lon'].attrs)
def create_metbk_hourly_sr(self): metbk_fn = 'metbk_a_dcl_instrument_recovered.nc' metbk_ds = xr.open_dataset(os.path.join(DATA_DIR, metbk_fn), decode_times=False) vel_fn = 'velpt_ab_dcl_instrument_recovered.nc' vel_ds = xr.open_dataset(os.path.join(DATA_DIR, vel_fn), decode_times=False) # both of these datasets are labeled deployment 3 but the times are squarely in deployment 1. Fix. metbk_ds.deployment.values[:] = 1 vel_ds.deployment.values[:] = 1 tr = TimeRange(metbk_ds.time.values[0], metbk_ds.time.values[-1]) sr = StreamRequest(self.hourly_sk, [], tr, {}, request_id='UNIT') metbk_ds = metbk_ds[self.base_params + [p.name for p in sr.stream_parameters[self.met_sk]]] vel_ds = vel_ds[self.base_params + [p.name for p in sr.stream_parameters[self.vel_sk]]] sr.datasets[self.met_sk] = StreamDataset(self.met_sk, sr.uflags, [self.hourly_sk, self.vel_sk], sr.request_id) sr.datasets[self.hourly_sk] = StreamDataset(self.hourly_sk, sr.uflags, [self.met_sk, self.vel_sk], sr.request_id) sr.datasets[self.vel_sk] = StreamDataset(self.vel_sk, sr.uflags, [self.hourly_sk, self.met_sk], sr.request_id) sr.datasets[self.hourly_sk].events = self.met_events sr.datasets[self.met_sk].events = self.met_events sr.datasets[self.vel_sk].events = self.vel_events sr.datasets[self.met_sk]._insert_dataset(metbk_ds) sr.datasets[self.vel_sk]._insert_dataset(vel_ds) return sr
def test_csv(self): nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed', 'nutnr_a_sample') ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed', 'ctdpf_sbe43_sample') nutnr_fn = 'nutnr_a_sample.nc' ctdpf_fn = 'ctdpf_sbe43_sample.nc' cals = json.load(open(os.path.join(DATA_DIR, 'cals.json'))) tr = TimeRange(3.65342400e+09, 3.65351040e+09) coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': cals[k], 'deployment': 1}] for k in cals} sr = StreamRequest(nutnr_sk, [2443], coefficients, tr, {}, request_id='UNIT') nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn), decode_times=False) ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn), decode_times=False) nutnr_ds = nutnr_ds[self.base_params + [p.name for p in sr.stream_parameters[nutnr_sk]]] ctdpf_ds = ctdpf_ds[self.base_params + [p.name for p in sr.stream_parameters[ctdpf_sk]]] sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients, sr.uflags, [nutnr_sk], sr.request_id) sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients, sr.uflags, [ctdpf_sk], sr.request_id) sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds) sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds) sr.calculate_derived_products() csv = CsvGenerator(sr, ',').to_csv() self.assertTrue(csv)
def test_calculate_external_single_deployment(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels', 'frame_type', 'nutnr_dark_value_used_for_fit']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT') nut_stream_dataset.events = self.nut_events nut_stream_dataset._insert_dataset(nut_ds) nut_stream_dataset.calculate_all() nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset}) nut_stream_dataset.calculate_all() expected_params = ['ctdpf_sbe43_sample-seawater_temperature', 'ctdpf_sbe43_sample-practical_salinity', 'salinity_corrected_nitrate'] self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def woa_get_ts(llon, llat, plot=0): import xarray as xr tempfile = '/Users/gunnar/Data/world_ocean_atlas/woa13_decav_t00_04v2.nc' saltfile = '/Users/gunnar/Data/world_ocean_atlas/woa13_decav_s00_04v2.nc' dt = xr.open_dataset(tempfile, decode_times=False) a = dt.isel(time=0) a.reset_coords(drop=True) t = a['t_mn'] T = t.sel(lon=llon, lat=llat, method='nearest').values ds = xr.open_dataset(saltfile, decode_times=False) a = ds.isel(time=0) a.reset_coords(drop=True) s = a['s_mn'] S = s.sel(lon=llon, lat=llat, method='nearest').values depth = s['depth'].data if plot: # import gvfigure as gvf import matplotlib.pyplot as plt # fig,ax = gvf.newfig(3,5) # plt.plot(T,depth) # ax.invert_yaxis() f, (ax1, ax2) = plt.subplots(1, 2, sharey=True) ax1.plot(T, depth, 'k') ax1.set_xlabel('Temperature') ax1.set_ylabel('Depth [m]') ax2.plot(S, depth, 'k') ax2.set_xlabel('Salinity') ax1.invert_yaxis() f.set_figwidth(5) f.set_figheight(5) return T, S, depth
def test_calculate_external_multiple_deployments(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels', 'frame_type', 'nutnr_dark_value_used_for_fit']] # remap times to make this two separate deployments dep1_start = self.ctd_events.deps[1].ntp_start dep2_stop = self.ctd_events.deps[2].ntp_start + 864000 ctd_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=ctd_ds.time.shape[0]) nut_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=nut_ds.time.shape[0]) ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT') nut_stream_dataset.events = self.nut_events nut_stream_dataset._insert_dataset(nut_ds) nut_stream_dataset.calculate_all() nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset}) nut_stream_dataset.calculate_all() expected_params = ['ctdpf_sbe43_sample-seawater_temperature', 'ctdpf_sbe43_sample-practical_salinity', 'salinity_corrected_nitrate'] self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def test_distribute_climate_cru(self): hef_file = get_demo_file('Hintereisferner.shp') entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0] gdirs = [] gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir) gis.define_glacier_region(gdir, entity=entity) gdirs.append(gdir) gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir_cru) gis.define_glacier_region(gdir, entity=entity) gdirs.append(gdir) climate.distribute_climate_data([gdirs[0]]) cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc') cru_dir = os.path.dirname(cru_dir) cfg.PATHS['climate_file'] = '~' cfg.PATHS['cru_dir'] = cru_dir climate.distribute_climate_data([gdirs[1]]) cfg.PATHS['cru_dir'] = '~' cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc') gdh = gdirs[0] gdc = gdirs[1] with xr.open_dataset(os.path.join(gdh.dir, 'climate_monthly.nc')) as nc_h: with xr.open_dataset(os.path.join(gdc.dir, 'climate_monthly.nc')) as nc_c: # put on the same altitude # (using default gradient because better) temp_cor = nc_c.temp -0.0065 * (nc_h.ref_hgt - nc_c.ref_hgt) totest = temp_cor - nc_h.temp self.assertTrue(totest.mean() < 0.5) # precip totest = nc_c.prcp - nc_h.prcp self.assertTrue(totest.mean() < 100)
def test_add_externals(self): nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed', 'nutnr_a_sample') ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed', 'ctdpf_sbe43_sample') nutnr_fn = 'nutnr_a_sample.nc' ctdpf_fn = 'ctdpf_sbe43_sample.nc' cals = json.load(open(os.path.join(DATA_DIR, 'cals.json'))) tr = TimeRange(3.65342400e+09, 3.65351040e+09) coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': cals[k], 'deployment': 1}] for k in cals} sr = StreamRequest(nutnr_sk, [2443], coefficients, tr, {}, request_id='UNIT') nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn), decode_times=False) ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn), decode_times=False) ctdpf_ds = ctdpf_ds[self.base_params + [p.name for p in sr.stream_parameters[ctdpf_sk]]] nutnr_ds = nutnr_ds[self.base_params + [p.name for p in sr.stream_parameters[nutnr_sk]]] sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients, sr.uflags, [nutnr_sk], sr.request_id) sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients, sr.uflags, [ctdpf_sk], sr.request_id) sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds) sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds) sr.calculate_derived_products() sr.import_extra_externals() self.assertIn('ctdpf_sbe43_sample-seawater_pressure', sr.datasets[nutnr_sk].datasets[0]) self.assertNotIn('ctdpf_sbe43_sample-seawater_pressure', sr.datasets[ctdpf_sk].datasets[0]) data = json.loads(JsonResponse(sr).json()) for each in data: self.assertIn('int_ctd_pressure', each)
def test_calculate_external_single_deployment(self): tr = TimeRange(3.65342400e+09, 3.65351040e+09) coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': v, 'deployment': 1}] for k, v in self.ctd_nutnr_cals.iteritems()} coefficients = CalibrationCoefficientStore(coefficients, 'UNIT') ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels', 'frame_type', 'nutnr_dark_value_used_for_fit']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, coefficients, {}, [], 'UNIT') ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_internal() nut_stream_dataset = StreamDataset(self.nutnr_sk, coefficients, {}, [self.ctdpf_sk], 'UNIT') nut_stream_dataset._insert_dataset(nut_ds) nut_stream_dataset.calculate_internal() nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset}) nut_stream_dataset.calculate_external() expected_params = ['ctdpf_sbe43_sample-seawater_temperature', 'ctdpf_sbe43_sample-practical_salinity', 'temp_sal_corrected_nitrate'] self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def main(in_file: Path, target_grid_file: Path, out_dir: Path=None): if out_dir is not None: out_dir.mkdir(exist_ok=True) out_file = out_dir / (in_file.name + "_interpolated") else: out_file = in_file.parent / (in_file.name + "_interpolated") if out_file.exists(): print(f"Skipping {in_file}, output already exists ({out_file})") return with xarray.open_dataset(target_grid_file) as ds_grid: lons, lats = ds_grid["lon"][:].values, ds_grid["lat"][:].values xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons.flatten(), lats.flatten()) with xarray.open_dataset(in_file) as ds_in: lons_s, lats_s = ds_in["lon"][:].values, ds_in["lat"][:].values xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_s.flatten(), lats_s.flatten()) ktree = KDTree(list(zip(xs, ys, zs))) dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1) # resample to daily ds_in_r = ds_in.resample(t="1D", keep_attrs=True).mean() ds_out = xarray.Dataset() for vname, var in ds_grid.variables.items(): ds_out[vname] = var[:] ds_out["t"] = ds_in_r["t"][:] for vname, var in ds_in_r.variables.items(): assert isinstance(var, xarray.Variable) var = var.squeeze() # only interested in (t, x, y) fields if var.ndim != 3: print(f"skipping {vname}") continue if vname.lower() not in ["t", "time", "lon", "lat"]: print(f"Processing {vname}") var_interpolated = [var[ti].values.flatten()[inds].reshape(lons.shape) for ti in range(var.shape[0])] ds_out[vname] = xarray.DataArray( var_interpolated, dims=("t", "x", "y"), attrs=var.attrs, ) ds_out.to_netcdf(out_file)
def get_xarray(self): try: self.dataset = xr.open_dataset(self.dataset_path) except ValueError as ve: print(datetime_error, ve) self.dataset = xr.open_dataset(self.dataset_path, decode_times=False) print(success_msg) return self.dataset
def get_xarray_datasets_from_filenames(filenames): if isinstance(filenames, string_types): datasets = [xarray.open_dataset(filenames)] else: datasets = [] for filename in filenames: datasets.append(xarray.open_dataset(filename)) return datasets
def convert(self): import xarray as xr try: # try OpenDAP url return xr.open_dataset(self.url) except IOError: # download the file return xr.open_dataset(self.file)
def test_image_driver_no_output_file_nans(fnames, domain_file): ''' Test that all VIC image driver output files have the same nan structure as the domain file ''' ds_domain = xr.open_dataset(domain_file) for fname in fnames: ds_output = xr.open_dataset(fname) assert_nan_equal(ds_domain, ds_output)
def delta_mm( fn, mean_fn, variable, mean_variable='tas' ): ''' simple way to compute extreme - mean deltas as native model resolution and write to NetCDF4 on disk ''' ds = xr.open_dataset( fn )[ variable ] ds_mean = xr.open_dataset( mean_fn )[ mean_variable ] delta = ds - ds_mean return delta.to_dataset( name=variable )
def read(self, file_path): self.file_path = file_path try: self.dataset = xr.open_dataset(self.file_path) except ValueError as ve: print(datetime_error, ve) self.dataset = xr.open_dataset(self.file_path, decode_times=False) print(success_msg) self.var_names = self.get_var_names(self.dataset)
def get_sim(self,memberid): "returns mean simulated soilcarbon between 1982 and 2005 (As specified in Carvalhais, last page)" fnmember=self.path2cdf+memberid+'.cdf' #Open Dataset sim=xr.open_dataset(fnmember).soilcarbon luarea=xr.open_dataset(fnmember).lu_area #Time constraint tconstraint=(sim.TIME>1982.1) * (sim.TIME<2005.9) sum_sim=(sim[tconstraint]*luarea[tconstraint]).sum(dim='landuse',skipna=False).mean(dim='TIME',skipna=False) return sum_sim
def test_write_groups(self): data1 = create_test_data() data2 = data1 * 2 with create_tmp_file() as tmp_file: data1.to_netcdf(tmp_file, group='data/1') data2.to_netcdf(tmp_file, group='data/2', mode='a') with open_dataset(tmp_file, group='data/1') as actual1: self.assertDatasetIdentical(data1, actual1) with open_dataset(tmp_file, group='data/2') as actual2: self.assertDatasetIdentical(data2, actual2)
def concat_simulations(fp_files, variables=['spec001_mr', 'WD_spec001', 'DD_spec001']): """ concatentate a series of flexpart simulations works with a list of netcdf input files from FP > v10 and a list of variable names. Only those variables will be returned in the concatenated dataset. Inputs: fp_files = list of flexpart netcdf files variables = list of variable names to concat Assumptions: Uses a function `reduce_dataset` which extracts the variables, it also takes a finite difference for all the deposition variables (cumulative) Requires all runs have the same: loutstep, dxout, dyout and dimesions """ # The following could work, but is too memory intensive # Need to do some profiling, to see where speedup/opt is possible # with one year of runs (1 yearly GAINS, 12 monthly GFED) this method # is 3 minutes, vs. 4 minutes as implemented, but memory cons. higher if False: dsets = [xr.open_dataset(f) for f in fp_files] nds = [reduce_dataset(d, variables) for d in dsets] nds = concat_runs(nds) clsd = [d.close() for d in dsets] ds = nds for i, f in enumerate(fp_files): print(f) with xr.open_dataset(f) as dset: dset = xr.open_dataset(f) nds = reduce_dataset(dset, variables) if i == 0: ds = nds continue ds = concat_runs([ds, nds]) dset.close() _end_datetime = {'iedate': dset.attrs['iedate'], 'ietime': dset.attrs['ietime']} # provide some further attribute information ds.attrs.update(_end_datetime) ds.attrs['history'] = """ Concatentation of: {0} using: {1}""".format(fp_files, inspect.stack()[0][3]) ds.attrs['institution'] = "UiO" ds.attrs['contact'] = "John F. Burkhart <*****@*****.**>" return ds.squeeze() # we return and get rid of nageclass/pointspec dims
def get_previous_hours(date): ehour = 12 if (date.hour) <= 16: print('Nighttime') edate = date - pd.Timedelta('1 days') edate = edate.replace(hour=ehour) else: edate = date edate = edate.replace(hour=ehour) t1 = edate t2 = edate + pd.Timedelta('3 hours') file = constants.ERA5 try: cmm = xr.open_dataset(file + 'ERA5_'+str(date.year)+'_pls.nc') except: return None pl_clim = xr.open_dataset(file + 'CLIM/ERA5_2008-2010_CLIM_'+str(edate.month)+'-'+str(ehour)+'_pls.nc') cmm = cmm.sel(time=t1) cm = cmm['t'] - pl_clim['t'] cm = cm.to_dataset() vwind_srfc = cmm['v'] - pl_clim['v'] uwind_srfc = cmm['u'] - pl_clim['u'] div = cmm['d'] - pl_clim['d'] q = cmm['q'] - pl_clim['q'] cm['u950'] = uwind_srfc cm['v950'] = vwind_srfc cm['div'] = div *1000 cm['q'] = q cm['qclim'] = pl_clim['q'] cm['divclim'] = pl_clim['d'] cm['bclim'] = pl_clim['v'] cm['uclim'] = pl_clim['u'] pl_clim.close() return cm
def __init__(self,config_name='config.ini',init=False,*args,**kwargs): #Shared parameters as attributes super(SoilCMap,self).__init__(config_name,init,*args,**kwargs) #FAPAR parameters, super(SoilCMap,self).import_config(config_name,'SoilCMap') #Load observation file and rename/reorder dimensions self.obs=xr.open_dataset(self.fnobs).cSoilTotal #Load area from control run, stack it for seasonal mean self.area=xr.open_dataset(self.fnctrl).area #Name self.name='SoilCMap'
def test_dask_roundtrip(self): with create_tmp_file() as tmp: data = create_test_data() data.to_netcdf(tmp) chunks = {'dim1': 4, 'dim2': 4, 'dim3': 4, 'time': 10} with open_dataset(tmp, chunks=chunks) as dask_ds: self.assertDatasetIdentical(data, dask_ds) with create_tmp_file() as tmp2: dask_ds.to_netcdf(tmp2) with open_dataset(tmp2) as on_disk: self.assertDatasetIdentical(data, on_disk)
def get_sim(self,memberid): "Get mean evapotranspiration from 1989-2005" fnmember=self.path2cdf+memberid+'.cdf' #Open Datasets evap=xr.open_dataset(fnmember).evap transp=xr.open_dataset(fnmember).transp etransp=evap+transp luarea=xr.open_dataset(fnmember).lu_area #Time constraint (+0.6 to count endyear as well..) tconstraint=(evap.TIME>float(self.startyear)) * (evap.TIME<float(self.endyear)+0.6) sum_etransp=(etransp[tconstraint]*luarea[tconstraint]).sum(dim='landuse',skipna=False).mean(dim='TIME',skipna=False) return sum_etransp
def __init__(self, dataset_path): self.dataset_path = dataset_path try: try: self.dataset = xr.open_dataset(self.dataset_path) except ValueError as ve: print(datetime_error, ve) self.dataset = xr.open_dataset(self.dataset_path, decode_times=False) print(success_msg) except OSError: print('File not found.') exit()
def delta_mm( fn, mean_fn, variable, mean_variable, output_filename ): ''' simple way to compute extreme - mean deltas as native model resolution and write to NetCDF4 on disk ''' import xarray as xr ds = xr.open_dataset( fn )[ variable ] ds_mean = xr.open_dataset( mean_fn )[ mean_variable ] delta = ds - ds_mean delta_ds = delta.to_dataset( name=variable ) delta_ds.to_netcdf( output_filename ) return output_filename
def main(plot_dir): # layer thickness zse = np.array([.022, .058, .154, .409, 1.085, 2.872]) plc_rf_all = np.zeros(0) plc_wsf_all = np.zeros(0) plc_dsf_all = np.zeros(0) plc_grw_all = np.zeros(0) plc_saw_all = np.zeros(0) sw_rf_all = np.zeros(0) sw_wsf_all = np.zeros(0) sw_dsf_all = np.zeros(0) sw_grw_all = np.zeros(0) sw_saw_all = np.zeros(0) start_yr = 2000 end_yr = 2010 nyears = (end_yr - start_yr) + 1 nmonths = 12 fdir = "outputs" fname = os.path.join(fdir, "cable_out_2000.nc") ds = xr.open_dataset(fname) iveg = ds["iveg"][:, :].values idx_rf = np.argwhere(iveg == 18.0) idx_wsf = np.argwhere(iveg == 19.0) idx_dsf = np.argwhere(iveg == 20.0) idx_grw = np.argwhere(iveg == 21.0) idx_saw = np.argwhere(iveg == 22.0) plc_rf_all = np.zeros((nyears * nmonths, len(idx_rf))) plc_wsf_all = np.zeros((nyears * nmonths, len(idx_wsf))) plc_dsf_all = np.zeros((nyears * nmonths, len(idx_dsf))) plc_grw_all = np.zeros((nyears * nmonths, len(idx_grw))) plc_saw_all = np.zeros((nyears * nmonths, len(idx_saw))) sw_rf_all = np.zeros((nyears * nmonths, len(idx_rf))) sw_wsf_all = np.zeros((nyears * nmonths, len(idx_wsf))) sw_dsf_all = np.zeros((nyears * nmonths, len(idx_dsf))) sw_grw_all = np.zeros((nyears * nmonths, len(idx_grw))) sw_saw_all = np.zeros((nyears * nmonths, len(idx_saw))) nyear = 0 cnt = 0 for year in np.arange(start_yr, end_yr): print(year) fdir = "outputs" fname = os.path.join(fdir, "cable_out_%d.nc" % (year)) ds = xr.open_dataset(fname) plc_vals = ds["plc"][:, 0, :, :].values """ SoilMoist1 = ds["SoilMoist"][:,0,:,:].values * zse[0] SoilMoist2 = ds["SoilMoist"][:,1,:,:].values * zse[1] SoilMoist3 = ds["SoilMoist"][:,2,:,:].values * zse[2] SoilMoist4 = ds["SoilMoist"][:,3,:,:].values * zse[3] SoilMoist5 = ds["SoilMoist"][:,4,:,:].values * zse[4] SoilMoist6 = ds["SoilMoist"][:,5,:,:].values * zse[5] sw = (SoilMoist1 + SoilMoist2 + SoilMoist3 + \ SoilMoist4 + SoilMoist5 + SoilMoist6 ) / np.sum(zse) """ SoilMoist1 = ds["SoilMoist"][:, 0, :, :].values * zse[0] SoilMoist2 = ds["SoilMoist"][:, 1, :, :].values * zse[1] SoilMoist3 = ds["SoilMoist"][:, 2, :, :].values * zse[2] SoilMoist4 = ds["SoilMoist"][:, 3, :, :].values * zse[3] sw = (SoilMoist1 + SoilMoist2 + \ SoilMoist3 + SoilMoist4) / np.sum(zse[0:4]) idx = nyear + cnt plc_rf = np.zeros((12, len(idx_rf))) sw_rf = np.zeros((12, len(idx_rf))) for i in range(len(idx_rf)): (row, col) = idx_rf[i] plc_rf[:, i] = plc_vals[:, row, col] sw_rf[:, i] = sw[:, row, col] plc_rf_all[idx:(idx + 12), :] = plc_rf sw_rf_all[idx:(idx + 12), :] = sw_rf plc_wsf = np.zeros((12, len(idx_wsf))) sw_wsf = np.zeros((12, len(idx_wsf))) for i in range(len(idx_wsf)): (row, col) = idx_wsf[i] plc_wsf[:, i] = plc_vals[:, row, col] sw_wsf[:, i] = sw[:, row, col] plc_wsf_all[idx:(idx + 12), :] = plc_wsf sw_wsf_all[idx:(idx + 12), :] = sw_wsf plc_dsf = np.zeros((12, len(idx_dsf))) sw_dsf = np.zeros((12, len(idx_dsf))) for i in range(len(idx_dsf)): (row, col) = idx_dsf[i] plc_dsf[:, i] = plc_vals[:, row, col] sw_dsf[:, i] = sw[:, row, col] plc_dsf_all[idx:(idx + 12), :] = plc_dsf sw_dsf_all[idx:(idx + 12), :] = sw_dsf plc_grw = np.zeros((12, len(idx_grw))) sw_grw = np.zeros((12, len(idx_grw))) for i in range(len(idx_grw)): (row, col) = idx_grw[i] plc_grw[:, i] = plc_vals[:, row, col] sw_grw[:, i] = sw[:, row, col] plc_grw_all[idx:(idx + 12), :] = plc_grw sw_grw_all[idx:(idx + 12), :] = sw_grw plc_saw = np.zeros((12, len(idx_saw))) sw_saw = np.zeros((12, len(idx_saw))) for i in range(len(idx_saw)): (row, col) = idx_saw[i] plc_saw[:, i] = plc_vals[:, row, col] sw_saw[:, i] = sw[:, row, col] plc_saw_all[idx:(idx + 12), :] = plc_saw sw_saw_all[idx:(idx + 12), :] = sw_saw nyear += 1 cnt += 12 #from matplotlib.pyplot import cm #colours = cm.Set2(np.linspace(0, 1, 5)) #colours = cm.get_cmap('Set2') labels = label_generator('lower', start="(", end=")") import seaborn as sns sns.set_style("ticks") colours = sns.color_palette("Set2", 8) fig = plt.figure(figsize=(20, 9)) fig.subplots_adjust(hspace=0.1) fig.subplots_adjust(wspace=0.15) plt.rcParams['text.usetex'] = False plt.rcParams['font.family'] = "sans-serif" plt.rcParams['font.sans-serif'] = "Helvetica" plt.rcParams['axes.labelsize'] = 20 plt.rcParams['font.size'] = 20 plt.rcParams['legend.fontsize'] = 20 plt.rcParams['xtick.labelsize'] = 20 plt.rcParams['ytick.labelsize'] = 20 ax1 = fig.add_subplot(231) ax2 = fig.add_subplot(232) ax3 = fig.add_subplot(233) ax4 = fig.add_subplot(234) ax5 = fig.add_subplot(235) ax1.scatter(sw_rf_all, plc_rf_all, marker=".", s=0.05, alpha=0.08, color=colours[0], rasterized=True) ax2.scatter(sw_wsf_all, plc_wsf_all, marker=".", s=0.05, alpha=0.08, color=colours[0], rasterized=True) ax3.scatter(sw_dsf_all, plc_dsf_all, marker=".", s=0.05, alpha=0.08, color=colours[0], rasterized=True) ax4.scatter(sw_grw_all, plc_grw_all, marker=".", s=0.05, alpha=0.08, color=colours[0], rasterized=True) ax5.scatter(sw_saw_all, plc_saw_all, marker=".", s=0.05, alpha=0.08, color=colours[0], rasterized=True) props = dict(boxstyle='round', facecolor='white', alpha=1.0, ec="white") fig_label = "%s %s" % (next(labels), "RF") ax1.text(0.8, 0.8, fig_label, transform=ax1.transAxes, fontsize=14, verticalalignment='top', bbox=props) fig_label = "%s %s" % (next(labels), "WSF") ax2.text(0.8, 0.8, fig_label, transform=ax2.transAxes, fontsize=14, verticalalignment='top', bbox=props) fig_label = "%s %s" % (next(labels), "DSF") ax3.text(0.8, 0.8, fig_label, transform=ax3.transAxes, fontsize=14, verticalalignment='top', bbox=props) fig_label = "%s %s" % (next(labels), "GRW") ax4.text(0.8, 0.8, fig_label, transform=ax4.transAxes, fontsize=14, verticalalignment='top', bbox=props) fig_label = "%s %s" % (next(labels), "SAW") ax5.text(0.8, 0.8, fig_label, transform=ax5.transAxes, fontsize=14, verticalalignment='top', bbox=props) ax1.axhline(y=88.0, ls="--", lw=1, color="black", label="$\Psi$$_{crit}$") ax2.axhline(y=88.0, ls="--", lw=1, color="black") ax3.axhline(y=88.0, ls="--", lw=1, color="black") ax4.axhline(y=88.0, ls="--", lw=1, color="black") ax5.axhline(y=88.0, ls="--", lw=1, color="black") ax1.legend(numpoints=1, loc=(0.67, 0.6), ncol=1, frameon=False) ax1.set_ylim(-5, 100) ax2.set_ylim(-5, 100) ax3.set_ylim(-5, 100) ax4.set_ylim(-5, 100) ax5.set_ylim(-5, 100) ax1.set_xlim(0.1, 0.3) ax2.set_xlim(0.1, 0.3) ax3.set_xlim(0.1, 0.3) ax4.set_xlim(0.1, 0.3) ax5.set_xlim(0.1, 0.3) ax1.set_ylabel("Loss of hydraulic conductivity (%)", position=(0.5, 0.0)) ax5.set_xlabel(r"$\theta$ (m$^{3}$ m$^{-3}$)") #ax.legend(numpoints=1, loc=(0.01, 0.65), ncol=1, frameon=False) plt.setp(ax1.get_xticklabels(), visible=False) plt.setp(ax2.get_xticklabels(), visible=False) plt.setp(ax2.get_yticklabels(), visible=False) plt.setp(ax3.get_yticklabels(), visible=False) plt.setp(ax5.get_yticklabels(), visible=False) from matplotlib.ticker import MaxNLocator ax1.xaxis.set_major_locator(MaxNLocator(4)) ax2.xaxis.set_major_locator(MaxNLocator(4)) ax3.xaxis.set_major_locator(MaxNLocator(4)) ax4.xaxis.set_major_locator(MaxNLocator(4)) ax5.xaxis.set_major_locator(MaxNLocator(4)) ax1.yaxis.set_major_locator(MaxNLocator(4)) ax2.yaxis.set_major_locator(MaxNLocator(4)) ax3.yaxis.set_major_locator(MaxNLocator(4)) ax4.yaxis.set_major_locator(MaxNLocator(4)) ax5.xaxis.set_major_locator(MaxNLocator(4)) plt.savefig(os.path.join(plot_dir, "plc_vs_sw_all.pdf"), bbox_inches='tight', pad_inches=0.1)
OutSR.ImportFromEPSG(102009) # WGS84 UTM Zone 56 South OutSR.SetProjParm("Central_Meridian", -100.0) OutSR.SetProjParm("Latitude_Of_Origin", 42.5) OutSR.SetProjParm("Standard_Parallel_1", 25.0) OutSR.SetProjParm("Standard_Parallel_2", 60.0) Point = ogr.Geometry(ogr.wkbPoint) Point.AddPoint(loni,lati) # use your coordinates here Point.AssignSpatialReference(InSR) # tell the point what coordinates it's in Point.TransformTo(OutSR) # project it to the out spatial reference yi = Point.GetY() xi = Point.GetX() ds = xr.open_dataset(Daymet_file) dsloc = ds.sel(x=xi, y=yi, method='nearest') Tmax = dsloc['tmax'].values time = dsloc['time'].values plt.figure(figsize=(16,10)) plt.plot(time, Tmax) plt.title('Daily Maximum Air Temperature at Wyoming Big Sage Site from Daymet') plt.xlabel('Time') plt.ylabel('$T_{max} {}^{\circ}C$') plt.show()
WORKPLACE = r"C:\Users\z3439910\Documents\Kien\1_Projects\2_Msc\1_E1\5_GIS_project" IRDIR = WORKPLACE + r"\IRimages2012" BTDIR = WORKPLACE + r"\2_IBTrACSfiles" os.chdir(IRDIR) CHOSEN_YEAR = "2012" BASIN = "NI" IMAG_RES = 4 #km r = 500 DEG_TO_KM = 111 #ratio if BASIN == "WP": LAT_BOUND = [-20, 60] #WP Basin LON_BOUND = [60, 180] #WP Basin B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.WP.v04r00.nc") elif BASIN == "EP": LAT_BOUND = [-20, 60] #WP Basin LON_BOUND = [-180, -60] #WP Basin B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.EP.v04r00.nc") elif BASIN == "NI": LAT_BOUND = [-20, 60] #WP Basin LON_BOUND = [0, 120] #WP Basin B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.NI.v04r00.nc") elif BASIN == "SI": LAT_BOUND = [-60, 20] #WP Basin LON_BOUND = [0, 120] #WP Basin B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.SI.v04r00.nc") B_TC_serials_decode = [x.decode("utf-8") for x in B_tracks['sid'].values]
################################################################################ # 1a ################################################################################ # Run MC print("Running Monte Carlo simulation for Exercise 1 a.") subprocess.call(["./exercise_1_a"]) # Load matplotlib style print("Plotting results.") path = os.path.dirname(os.path.abspath(__file__)) plt.style.use(os.path.join(path, "..", "misc", "matplotlib_style.rc")) results = [] for i in range(3): data = xarray.open_dataset("results_1_a_{}_cuda.nc".format(i + 1)) results += [np.array(data.irradiance)] # Results of Beer-Lambert law x = np.linspace(0, 10e3, 1001) sigma = np.zeros(x.shape) sigma[:] = 1e-4 tau = cumtrapz(sigma, x = x) f = plt.figure(figsize = (10, 5)) gs = GridSpec(1, 2, width_ratios = [1.0, 1.0]) ax = plt.subplot(gs[0]) handles = [] handles += ax.plot(x[1:] / 1e3, np.exp(-tau), ls = "--", c = "k", label = "Beer-Lambert Law")
csvdir = './data/' nwanews = lgn.news2roms(grdfile) nwanews.read_database(csvdir) nwanews.extract_domain() # do this if you want to see locations or rivers mouth #nwanews.plot_river_mouth(db='domain') nwanews.move_rivermouth2roms() # you can pass a list of rivers to check, will show spreading performed #no3_conc = nwanews.create_rivers_input('NO3_CONC',river_checklist=['Susquehanna']) no3_conc = nwanews.create_rivers_input('NO3_CONC', plot_result=False) po4_conc = nwanews.create_rivers_input('PO4_CONC', plot_result=False) # you can do the same for all other nutrients # create the output file with xarray grd = xr.open_dataset(grdfile) ds = xr.Dataset( { 'no3_conc': (['eta_rho', 'xi_rho'], no3_conc), 'po4_conc': (['eta_rho', 'xi_rho'], po4_conc), # add other nutrient to dataset }, coords={ 'lon_rho': (['eta_rho', 'xi_rho'], grd.lon_rho), 'lat_rho': (['eta_rho', 'xi_rho'], grd.lat_rho) }) ds.to_netcdf('nutrients_conc_NWA.nc')
'bufr' : '/raid60/scratch/federico/Harvested_Databases/bufr' } for d,p in zip( [db], [data_directories[db] ] ): print(' I am here') dic_all = {} dic = '' print('Processing the dataset: ' , d , ' ' , p ) files = [ k for k in os.listdir(p) if '.nc' in k ] ### edit here for f in files: print(files.index(f) ) try: File = xr.open_dataset( p + '/' + f , engine = 'h5netcdf' ) rts = File['recordtimestamp'].data except: continue x = pd.to_datetime(rts) try: dic = dic.append(x) except: dic = x dic_all[d] = dic np.save('NEW_NUMPYS/records_' + d , dic_all, allow_pickle=True ) print('Finished ***', d )
mask_data = Dataset(mask_file) climate_data = Dataset(climate_file) print('Mask file dimensions:', mask_data.dimensions) print('Climate file dimensions:',climate_data.dimensions) """ Identify the variables stored within each dataset: """ print('Mask file variables:', mask_data.variables) print('Climate file variables:', climate_data.variables) """ 3.1) Firstly, we will use xarray to assign the land cover percentage data to a new object. """ mask_dset = xr.open_dataset(mask_file) #Use xarray to open the mask dataset land_perc = mask_dset['sftlf'] # assign the land percentage variable to a new object # sftlf is the standardised term for land percentage cover print('Land_perc:', land_perc) print('Max land area:', land_perc.data.max(), '%') # check that max land area is 100 % print('Min land area:', land_perc.data.min(), '%') # check that min land area is 0 % """ 3.2) Second we will load in the climate data and assign temperature (tas) to a new object. """ climate_dset = xr.open_dataset(climate_file) clim = climate_dset['tas'].mean('time', keep_attrs=True) # create a new object with the temperature data for each grid cell # keep_attrs maintains attribute data (e.g. units, variable names etc) in new object """
SW=SW) field.make_base() y_predict = self.predict(field._X) outfield = MakeBigImage(y_predict, ind=(field._indouty, field._indoutx)) return outfield if __name__ == "__main__": name = '../data/model_upar.pkl' nn = loadmymodel(name) # nn = mymodel(model=nntmp._model, # moyX = nntmp._moyX,etX=nntmp._etX, # moyY = nntmp._moyY,etY=nntmp._etY) testfile = '../data/test_image.nc' tdata = xr.open_dataset(testfile) infield = ['uphy', 'hphy'] outfield = 'uparam' forcfield = ['taux'] y = nn.predictfield(tdata, outfield=outfield, infield=infield, forcfield=forcfield) fig, (ax1, ax2, ax3) = plt.subplots(ncols=3) p1 = ax1.imshow(tdata['uparam'][1, :, :]) p2 = ax2.imshow(y) p3 = ax3.imshow(y - tdata['uparam'][1, :, :], cmap=plt.get_cmap('bwr')) fig.colorbar(p1, ax=ax1) fig.colorbar(p2, ax=ax2) fig.colorbar(p3, ax=ax3)
psd_files = sorted([ i for i in glob.glob( '/home/disk/eos9/jkcm/Data/particle/psd/compressed/UW_particle_classifications.1hz.rf*_psd.nc' ) if not 'rf15' in i ]) flight_files = sorted([ i for i in glob.glob('/home/disk/eos9/jfinlon/socrates/*/*.PNI.nc') if not 'RF15' in i ]) psds = xr.open_mfdataset(psd_files, combine='by_coords') ds = [] for f in sorted(flight_files): flights = xr.open_dataset(f) flights = flights.drop('PSTFC', errors='ignore') ds.append(flights) flights = xr.concat(ds, dim='Time', data_vars=['ATX'], compat="no_conflicts") flights = flights.rename({'Time': 'time'}) fig, axg = plt.subplots(figsize=(15, 10), nrows=2, ncols=3, sharex=True, sharey=True) axl = axg.flatten() all_cloud = psds.count_darea_all.sum(dim='size_bin') > 10 temp_ranges = [-40, -20, -5, 0, 5, 40, 100]
def main(fname, plot_dir): ds = xr.open_dataset(fname, decode_times=False) bottom, top = np.min(ds.latitude).values, np.max(ds.latitude).values left, right = np.min(ds.longitude).values, np.max(ds.longitude).values ntime, nrows, ncols = ds.VOD.shape # Get baseline period # 1993-1999 nyears = 7 vod_pre = np.zeros((nyears, nrows, ncols)) vals = np.zeros((3, nrows, ncols)) yr_count = 0 count = 0 for year in np.arange(1993, 2000): for month in np.arange(1, 13): if month == 12: #vals = ds.VOD[count,:,:].values #vals = np.where(np.isnan(vals), 0, vals) vod_pre[yr_count, :, :] += ds.VOD[count, :, :] vod_pre[yr_count, :, :] += ds.VOD[count + 1, :, :] vod_pre[yr_count, :, :] += ds.VOD[count + 2, :, :] vod_pre[yr_count, :, :] /= 3 count += 1 yr_count += 1 vod_pre = np.mean(vod_pre, axis=0) vod_pre = np.flipud(vod_pre) # We will have incremented the counter by one too many on the final # iteration, fix this as we need to start at the right point for 2000 count = count - 1 # 2000-2009 nyears = 10 vod_dur = np.zeros((nyears, nrows, ncols)) yr_count = 0 for year in np.arange(2000, 2010): for month in np.arange(1, 13): if month == 12: vod_dur[yr_count, :, :] += ds.VOD[count, :, :] vod_dur[yr_count, :, :] += ds.VOD[count + 1, :, :] vod_dur[yr_count, :, :] += ds.VOD[count + 2, :, :] vod_dur[yr_count, :, :] /= 3 count += 1 yr_count += 1 vod_dur = np.mean(vod_dur, axis=0) vod_dur = np.flipud(vod_dur) chg = ((vod_dur - vod_pre) / vod_pre) * 100.0 fig = plt.figure(figsize=(9, 6)) plt.rcParams['font.family'] = "sans-serif" plt.rcParams['font.size'] = "14" plt.rcParams['font.sans-serif'] = "Helvetica" cmap = plt.cm.get_cmap('BrBG', 10) # discrete colour map projection = ccrs.PlateCarree() axes_class = (GeoAxes, dict(map_projection=projection)) rows = 1 cols = 1 axgr = AxesGrid(fig, 111, axes_class=axes_class, nrows_ncols=(rows, cols), axes_pad=0.2, cbar_location='right', cbar_mode='single', cbar_pad=0.5, cbar_size='5%', label_mode='') # note the empty label_mode for i, ax in enumerate(axgr): # add a subplot into the array of plots #ax = fig.add_subplot(rows, cols, i+1, projection=ccrs.PlateCarree()) plims = plot_map(ax, chg, cmap, i, top, bottom, left, right) import cartopy.feature as cfeature states = cfeature.NaturalEarthFeature( category='cultural', name='admin_1_states_provinces_lines', scale='10m', facecolor='none') # plot state border SOURCE = 'Natural Earth' LICENSE = 'public domain' ax.add_feature(states, edgecolor='black', lw=0.5) cbar = axgr.cbar_axes[0].colorbar(plims) cbar.ax.set_title("% Difference", fontsize=16, pad=10) #cbar.ax.set_yticklabels([' ', '-30', '-15', '0', '15', '<=70']) ofname = os.path.join(plot_dir, "vod.png") fig.savefig(ofname, dpi=300, bbox_inches='tight', pad_inches=0.1)
def t_ws_func(t_c, t_dew, dim="time"): return xr.apply_ufunc( t_ws_gufunc, t_c, t_dew, input_core_dims=[[], []], dask="parallelized", output_dtypes=[output_dtype], ) #%% =============== LOAD DATA ================== try: # if file exists load the data data = xr.open_dataset(save_dict["savefolder"] + save_dict["savename"]) print("=======\nload data done") except: #%% ==================== CALCULATION OF SWBGT ================ print("=======\nfile not found") folderpath = r"I:\Bachlor_thesis\Data" st = time.time() temp = xr.open_dataset( folderpath + r"\t2m_era20c_1900-2010.nc") #, chunks={'time': 12*4*365}) print("load temp took {:.2f}".format(time.time() - st)) st = time.time() temp_dew = xr.open_dataset( folderpath + r"\td2m_era20c_1900-2010.nc") #, chunks={'time': 12*4*365}) print("load dew took {:.2f}".format(time.time() - st))
import cmocean #import proplot as plot #import seaborn as sns #import proplot as plot #fin = '/Users/cpatrizio/data/MERRA2/' #fin = '/Users/cpatrizio/data/ECMWF/' #fin = '/Users/cpatrizio/data/OAFlux/' #fout = '/Volumes/GoogleDrive/My Drive/PhD/figures/ocean-atmosphere/localSST_global/' fin = '/Volumes/GoogleDrive/My Drive/data_drive/MERRA2/' fout = '/Users/cpatrizio/figures_arc/' #MERRA-2 fsstM2 = xr.open_dataset(fin + 'MERRA2_SST_ocnthf_monthly1980to2017.nc') fthf = xr.open_dataset(fin + 'MERRA2_thf_monthly1980to2017.nc') fSLP = xr.open_dataset(fin + 'MERRA2_SLP_monthly1980to2017.nc') radfile = xr.open_dataset(fin + 'MERRA2_rad_monthly1980to2017.nc') cffile = xr.open_dataset(fin + 'MERRA2_modis_cldfrac_monthly1980to2017.nc') #frad = cdms2.open(fin + 'MERRA2_tsps_monthly1980to2017.nc') #fuv = cdms2.open(fin + 'MERRA2_uv_monthly1980to2017.nc') #fRH = cdms2.open(fin + 'MERRA2_qv10m_monthly1980to2017.nc') #fcE = cdms2.open(fin + 'MERRA2_cE_monthly1980to2017.nc') #fcD = cdms2.open(fin + 'MERRA2_cD_monthly1980to2017.nc') ftau = xr.open_dataset(fin + 'MERRA2_tau_monthly1980to2019.nc') fssh = xr.open_dataset(fin + 'ncep.ssh.198001-201912.nc') fseaice = xr.open_dataset(fin + 'MERRA2_seaice_monthly1980to2019.nc') #fh = xr.open_dataset(fin + 'ncep.mixedlayerdepth.198001-201712.nc')
#!/usr/bin/env python2 # -*- coding: utf-8 -*- """ Created on Mon Mar 20 19:44:28 2017 author: Craig Risien ([email protected]) reference: http://journals.ametsoc.org/doi/abs/10.1175/2008JPO3881.1 """ import xarray as xr import numpy as np from matplotlib import pyplot as plt from mpl_toolkits.basemap import Basemap ds = xr.open_dataset( 'http://chapman.ceoas.oregonstate.edu/thredds/dodsC/CIOSS/HadISST/Monthly/HadISST_sst.nc' ) ds.info() sst = ds.sst.data lat = ds.latitude.data lon = ds.longitude.data time = ds.time.data del ds #clear ds #subset the data for the period 1900-1999 sst = sst[360:1560, :, :] sst[sst == -1000] = np.nan #reshape the sst matrix tmp = np.zeros((180, 360, 1200)) for i in range(0, 1200):
hour = original_hour / 3 if hour % 3 != int(hour % 3): print(hour % 3) print("Invalid Hour: Rounding to the closest time") hour = round(hour, 0) if hour > 384: print("Time out of 384hr bounds. Defaulting to 3 ") hour = 1 hour = int(hour) lat_north = 59 lat_south = -10 lon_west = 360 lon_east = 220 data = xr.open_dataset(dataset) data = data.sel(lon=slice(lon_east, lon_west), lat=slice(lat_south, lat_north)) lon = data["lon"] lat = data["lat"] mslp = data["msletmsl"][hour] / 100 zr = data["cfrzrsfc"][hour] ip = data["cicepsfc"][hour] sn = data["csnowsfc"][hour] rate = data["prateavesfc"][hour] * 1500 #Heights Geo_500 = data["hgtprs"][hour].sel(lev=500) / 10 Geo_1000 = data["hgtprs"][hour].sel(lev=1000) / 10 remain = Geo_500 - Geo_1000
- Same grid (spatial) - same time step (temporal) - Subset ROI - assign latitude, longitude, time """ import numpy as np import pandas as pd import matplotlib.pyplot as plt import xarray as xr from pathlib import Path data_dir = Path("/Volumes/Lees_Extend/data/ecmwf_sowc/data") data_dir = Path("data") ds = xr.open_dataset(data_dir / "raw" / "seasonal" / "fcmean2018.grb", engine="cfgrib") # rename the variables to more reasonable names ds = ds.rename({ "p146.172": "surface_sensible_heat_flux", "p147.172": "surface_latent_heat_flux" }) # rename the coords to more reasonable coords ds = ds.rename({"time": "initialisation_date", "step": "forecast_horizon"}) # total precip rate m/s d = ds.tprate fig, ax = plt.subplots() d.mean(dim=["initialisation_date", "forecast_horizon", "number"]).plot(ax=ax)
def addHistoricalData(station, CTDConfig, work_dir): if station.name in ['VT69']: ds = xr.open_dataset( '/Users/trondkr/Dropbox/NIVA/OKOKYST/Historical_oekokyst_data/VT69_2013_2016.nc' ) elif station.name in ['VT70']: ds = xr.open_dataset( '/Users/trondkr/Dropbox/NIVA/OKOKYST/Historical_oekokyst_data/VT70_2013_2016.nc' ) else: raise Exception( "Should not call addHistoricalData unless station is VT69 or VT70 due to available timeseries" ) df = ds.to_dataframe().groupby(['time', 'depth']).mean() first = True for date_depth_index, row in df.iterrows(): if first: old_date = date_depth_index[0] temperature = [] depth = [] salinity = [] oxsat = [] oxygen = [] ftu = [] first = False if old_date == date_depth_index[0]: temperature.append(row['temp']) salinity.append(row['salt']) oxsat.append(row['O2sat']) oxygen.append(row['O2vol']) depth.append(float(date_depth_index[1])) ftu.append(row['O2vol'] * np.nan) else: print("Storing data for time {}".format(date_depth_index[0])) depth = pd.Series(depth) temperature = pd.Series(temperature, index=depth) salinity = pd.Series(salinity, index=depth) oxygen = pd.Series(oxygen, index=depth) oxsat = pd.Series(oxsat, index=depth) ftu = pd.Series(ftu, index=depth) df_new = pd.DataFrame(columns=[ "Depth", "Temperature", "Salinity", "Oxygen", "Oxsat", "FTU" ]) df_new.set_index('Depth', inplace=True, drop=False) df_new["Depth"] = depth df_new["Temperature"] = temperature df_new["Salinity"] = salinity df_new["Oxygen"] = oxygen df_new["Oxsat"] = oxsat # Add data to station object for later station.addData( salinity, temperature, oxygen, oxsat, ftu, salinity.index, date2num(date_depth_index[0], CTDConfig.refdate, calendar="standard")) temperature = [] depth = [] salinity = [] oxsat = [] oxygen = [] ftu = [] old_date = date_depth_index[0] temperature.append(row['temp']) salinity.append(row['temp']) oxsat.append(row['O2sat']) oxygen.append(row['O2vol']) depth.append(float(date_depth_index[1])) ftu.append(row['O2vol'] * np.nan)
lat = [line.rstrip('\n') for line in open(os.path.join(path, 'grid', 'lat.csv'))] lon = [line.rstrip('\n') for line in open(os.path.join(path, 'grid', 'lon.csv'))] grid = [] i = 1 for lat_ in lat: for lon_ in lon: grid.append([lat_, lon_, i]) i += 1 files = glob.glob1(os.path.join(path,'Data'), '*.nc') for file in files: print(file) name = file[-7:][:4] d = xr.open_dataset(os.path.join(path,'Data', file)) d.precip.to_dataframe().to_csv(os.path.join(path, 'CSV', '{}.csv'.format(name))) columns = ['date', 'lat', 'lon', 'pre'] df = pd.read_csv(os.path.join(path, 'CSV', '{}.csv'.format(name)), skiprows=1, names=columns) df.loc[df['pre'] == -99, ['pre']] = np.nan df_l = pd.DataFrame(grid, columns=['lat', 'lon', 'index']) df_l['lat'] = df['lat'].astype(float) df_l['lon'] = df['lon'].astype(float) new_df = pd.merge(df, df_l, how='left', left_on=['lat', 'lon'], right_on=['lat', 'lon']) new_df.drop(['lat', 'lon'], axis=1, inplace=True) number_of_chunks = 20 [new_df.to_csv(os.path.join(path, 'CSV', '{name}_{id}.csv'.format(name=name, id=id)), index=False) for id, new_df in
# -------------------- general setup -------------------- t_main_start = time.time() if args.verbose: print("Running", sys.argv[0]) print(" -> verbose output = True") print() # -------------------- read basin mask -------------------- if args.verbose: print("... reading basin mask from file '" + args.basin_file + "'") t_read_start = time.time() t_read_bmask_start = time.time() try: ds_basins = xr.open_dataset(args.basin_file) except: print("BASIN_FILE '", args.basin_file, "' can't be found! Exiting.") sys.exit(1) basins = ds_basins['basins'] # mask values: # 0 = grounded ice or open ocean # 1 = continental shelf but ignored # 2 = continental shelf as used in PICO contshelf_mask = ds_basins['pico_contshelf_mask'] # check dimensions basin_ndim = len(basins.dims) if basin_ndim != 2 and basin_ndim != 3: raise ValueError( str("basin field has dimensions " + \
# Date created: 09 Sept 2020 # Description: Script imports gappy mooring timeseries, bins it to daily temperatures, and fits successively longer predicting LSTM models to fill in the gaps # import packages import pandas as pd import numpy as np import tensorflow as tf from tensorflow import keras from tensorflow.keras import layers f'Tensorflow version: {tf.__version__}' import xarray as xr import os import pickle # open raw netcdf file temp = xr.open_dataset('/srv/scratch/z3289452/OceanDataScience/Mooring/PH100/IMOS_ANMN-NSW_TZ_20091029_PH100_FV01_TEMP-aggregated-timeseries_END-20190612_C-20190819.nc') # bin data at various pressure depths bins = [0,11,19,20,28,106,114,116] labs = [5.5,15,19.5,24,(106-28)/2,110,115] # create pd dataframe from scratch data = {'TIME': temp.TIME.values, 'TEMP': temp.TEMP.values, 'PRES': temp.PRES_REL.values} temp_df= pd.DataFrame(data, columns = ['TIME','TEMP','PRES']) # set TIME as the indexing variable temp_df = temp_df.set_index('TIME') # create a column with pressure bin labels temp_df['PRES_BIN'] = pd.cut(temp_df.PRES, bins = bins, labels = labs, include_lowest=True) # First groupby PRES_BIN column, then resample each group on hourly intervals and mean each resulting bin
init_data, init_data1 = 0, 0 map_sum, map_cnt, map_max = np.zeros([imap_lats, imap_lons]), np.zeros( [imap_lats, imap_lons]), np.zeros([imap_lats, imap_lons]) map_sum_recov, map_cnt_recov = np.zeros([imap_lats, imap_lons]), np.zeros( [imap_lats, imap_lons]) map_sum_mld, map_cnt_mld = np.zeros([imap_lats, imap_lons ]), np.zeros([imap_lats, imap_lons]) for inum_storm in range(0, 110): # for inum_storm in range(16,17): filename = dir_out + str(iyr_storm) + '/' + str(inum_storm).zfill( 3) + '_interpolated_track.nc' exists = os.path.isfile(filename) if not exists: continue print(filename) ds_storm_info = xr.open_dataset(filename) ds_storm_info = ds_storm_info.sel(j2=0) ds_storm_info.close() filename = dir_out + str(iyr_storm) + '/' + str(inum_storm).zfill( 3) + '_combined_data.nc' ds_all = xr.open_dataset(filename) ds_all['spd'] = np.sqrt(ds_all.uwnd**2 + ds_all.vwnd**2) ds_all.close() filename = dir_out + str(iyr_storm) + '/' + str(inum_storm).zfill( 3) + '_MLD_data_v2.nc' ds_all2 = xr.open_dataset(filename) ds_all2.close() if abs(ds_all.lon[-1] - ds_all.lon[0]) > 180: ds_all.coords['lon'] = np.mod(ds_all['lon'], 360) ds_storm_info['lon'] = np.mod(ds_storm_info['lon'], 360) max_lat = ds_storm_info.lat.max()
# --- Read in data --- # # Set directories and file names dailyDir = '/Users/meganfowler/Documents/NCAR/Analysis/Coupling_initial/data/day/' middleName = '_day_CESM2_amip_r10i1p1f1_gn_' varNames = ['hfls', 'hfss', 'mrso', 'mrsos', 'tas', 'hurs', 'ps'] # Set time period portion of fileNames timeName_flx = [ '19700101-19791231', '19800101-19891231', '19900101-19991231', '20000101-20091231', '20100101-20150101' ] # Read in test file to get lat/lon fileName = dailyDir + varNames[0] + middleName + timeName_flx[0] + '.nc' testDF = xr.open_dataset(fileName, decode_times=True) # Get lat and lon lat = testDF.lat.values lon = testDF.lon.values # Read in surface properties (T, RH, P) for iT in range(len(timeName_flx)): # Sfc Temp tFile = dailyDir + varNames[4] + middleName + timeName_flx[ iT] + '.nc' # File name TsfcDF = xr.open_dataset(tFile, decode_times=True) TsfcDF['time'] = TsfcDF.indexes['time'].to_datetimeindex()
# =========================== # Load the Data # ========================== sns.set_context('paper') # AVHRR def preprocess(ds): data = ds.sel(lat=slice(-90, -40)) return data # Load all the MAR data MAR = xr.open_dataset( '/uio/lagringshotell/geofag/projects/miphclac/shofer/MAR/wBS/mon-CC-MAR_ERA5-1981-2018.nc' ) MAR_noBS = xr.open_dataset( '/uio/lagringshotell/geofag/projects/miphclac/shofer/MAR/noBS/mon-CC-MAR_ERA5-1980-2018.nc' ) MAR_grid = xr.open_dataset( '/uio/lagringshotell/geofag/projects/miphclac/shofer/MAR/wBS/MARcst-AN35km-176x148.cdf' ) ds_grid = xr.Dataset( { 'RIGNOT': (['y', 'x'], MAR_grid.RIGNOT.values), 'SH': (['y', 'x'], MAR_grid.SH.values), 'LAT': (['y', 'x'], MAR_grid.LAT.values), 'LON': (['y', 'x'], MAR_grid.LON.values) }, coords={
def lagranto_plotting(traj_variable_name, start_time, end_time, trajs_bunch_level='all', subset=False, save=False): '''This function plots the chosen variables for the trajectories calculated with Lagranto. Supported variables for plotting procedure are water_vapor, updraft and height.''' ### Predefine some variables ### traj_data_dir = '/scratch3/thomasl/work/retrospective_part/lagranto/' 'traj_baden_10000_every1000_area_1200.ll' save_dir = '/scratch3/thomasl/work/retrospective_part' '/casestudy_baden/lagranto/' start_locations = 'area' # distinction between single and multiple # starting points of trajectories number_trajs_plot = 1 # Location of Initiation lat = 47.25 lon = 7.85 initiation = CoordPair(lat, lon) # Change extent of plot subset_extent = [7, 9, 47, 48] # Variables for getting PBL height of WRF model data date = '2018-05-30' time = '16:10' wrf_filename = '/scratch3/thomasl/work/data/casestudy_baden/' 'wrfout_d02_{}_{}:00'.format( date, time) # Variables: if traj_variable_name == 'water_vapor': traj_variable_name = 'QVAPOR' title_name = 'Trajectories of Water Vapor' colorbar_label_trajs = 'Water Vapor Mixing Ratio [$g$ $kg^{-1}$]' save_name = 'trajectory_water_vapor_{}_{}'.format(start_time, end_time) traj_variable_min = 0 traj_variable_max = 15 elif traj_variable_name == 'updraft': traj_variable_name = 'W_UP_MAX' title_name = 'Trajectories of Updraft' colorbar_label_trajs = 'Max Z-Wind Updraft [$m$ $s^-$$^1$]' save_name = 'trajectory_updraft_{}_{}'.format(start_time, end_time) traj_variable_min = 0 traj_variable_max = 10 elif traj_variable_name == 'height': traj_variable_name = 'z' title_name = 'Height of Trajectories' colorbar_label_trajs = 'Height of Trajectories [$m$]' save_name = 'trajectory_height_{}_{}'.format(start_time, end_time) traj_variable_min = 0 traj_variable_max = 10000 ### Plotting procedure ### trajs = Tra() trajs.load_ascii(traj_data_dir) # Get PBL out of WRF model data ncfile = Dataset(wrf_filename) wrf_file = xr.open_dataset(wrf_filename) data = wrf_file.PBLH location = ll_to_xy(ncfile, lat, lon) data_point = data.sel(west_east=location[0], south_north=location[1]) pbl = data_point.values # Separate trajectories in 3 vertical bunches # Trajectories of pbl (according to pbl height of WRF model data) trajspbl = [] for t in trajs: if (t['z'][0] <= pbl): trajspbl.append(t) # Trajectories between pbl and 5000 m trajs5 = [] for t in trajs: if (t['z'][0] > pbl and t['z'][0] < 5000): trajs5.append(t) # Trajectories between 5000 m and 10000 m trajs10 = [] for t in trajs: if (t['z'][0] > 5000 and t['z'][0] <= 10000): trajs10.append(t) if trajs_bunch_level == 'pbl': trajs_bunch = trajspbl elif trajs_bunch_level == '5': trajs_bunch = trajs5 elif trajs_bunch_level == '10': trajs_bunch = trajs10 elif trajs_bunch_level == 'all': trajs_bunch = trajs # Get terrain height terrain_height = getvar(ncfile, 'HGT') / 1000 # change to km # Define cart projection lats, lons = latlon_coords(terrain_height) cart_proj = ccrs.LambertConformal(central_longitude=8.722206, central_latitude=46.73585) # Create figure fig = plt.figure(figsize=(15, 10)) ax = plt.axes(projection=cart_proj) ### Set map extent ### domain_extent = [3.701088, 13.814863, 43.85472, 49.49499] if subset == True: ax.set_extent([ subset_extent[0], subset_extent[1], subset_extent[2], subset_extent[3] ], ccrs.PlateCarree()) else: ax.set_extent([ domain_extent[0] + 0.7, domain_extent[1] - 0.7, domain_extent[2] + 0.1, domain_extent[3] - 0.1 ], ccrs.PlateCarree()) # Plot trajectories levels_trajs = np.linspace(traj_variable_min, traj_variable_max, 11) rounded_levels_trajs = [round(elem, 1) for elem in levels_trajs] cmap = ListedColormap(sns.cubehelix_palette( 10, start=.5, rot=-.75, )) plt_trajs = plot_trajs(ax, trajs_bunch[0::number_trajs_plot], traj_variable_name, linewidth=2, levels=rounded_levels_trajs, cmap=cmap) # Plot the terrain height with colorbar levels = np.linspace(0, 4, 21) terrain = plt.contourf(to_np(lons), to_np(lats), to_np(terrain_height), levels=levels, transform=ccrs.PlateCarree(), cmap=get_cmap('Greys'), alpha=0.75) cbar = mpu.colorbar(terrain, ax, orientation='horizontal', aspect=40, shrink=.05, pad=0.075) cbar.set_label('Terrain Height [$km$]', fontsize=15) cbar.set_ticks(levels) # Make only every second color bar tick label visible for label in cbar.ax.xaxis.get_ticklabels()[1::2]: label.set_visible(False) # Add color bar for trajectory variable if traj_variable_name == 'QVAPOR': extend = 'both' else: extend = 'max' cbar_trajs = mpu.colorbar(plt_trajs, ax, orientation='vertical', aspect=40, shrink=.05, pad=0.05, extend=extend) cbar_trajs.set_label(colorbar_label_trajs, fontsize=15) cbar_trajs.set_ticks(rounded_levels_trajs) # Add borders and coastlines ax.add_feature(cfeature.BORDERS.with_scale('10m'), linewidth=0.8) ax.add_feature(cfeature.COASTLINE.with_scale('10m'), linewidth=0.8) # Add cross for initiation location for t in trajs: ax.plot(t['lon'][0], t['lat'][0], 'kx', transform=ccrs.PlateCarree()) # Add gridlines lon = np.arange(0, 20, 1) lat = np.arange(40, 60, 1) gl = ax.gridlines(xlocs=lon, ylocs=lat, zorder=3) # Add tick labels mpu.yticklabels(lat, ax=ax, fontsize=12.5) mpu.xticklabels(lon, ax=ax, fontsize=12.5) # Set title ax.set_title(title_name, loc='left', fontsize=20) ax.set_title('Time Range: {} - {} UTC'.format(start_time, end_time), loc='right', fontsize=15) plt.show() ### Save figure ### if save == True: if subset == True: if trajs_bunch_level == 'all': fig.savefig('{}{}_subset_{}.png'.format( save_dir, save_name, start_locations), bbox_inches='tight', dpi=300) else: fig.savefig('{}{}_subset_{}_{}.png'.format( save_dir, save_name, trajs_bunch_level, start_locations), bbox_inches='tight', dpi=300) else: if trajs_bunch_level == 'all': fig.savefig('{}{}_{}.png'.format(save_dir, save_name, start_locations), bbox_inches='tight', dpi=300) else: fig.savefig('{}{}_{}_{}.png'.format(save_dir, save_name, trajs_bunch_level, start_locations), bbox_inches='tight', dpi=300)
#sort el ninio events import numpy as np import xarray as xr import pandas as pd import plots import os os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' RUTA='~/datos/data/' #abro el archivo de geopotencial y junto la coordenada year y numbre ds = xr.open_dataset(RUTA + 'monthly_hgt200_aug_feb.nc') ds = ds - ds.mean(dim='longitude') ninio34 = xr.open_dataset(RUTA + 'ninio34_index.nc') month = ['Aug', 'Sep', 'Oct', 'Nov','Dec', 'Jan', 'Feb'] seas = ['ASO', 'SON', 'OND', 'NDJ', 'DJF'] index_monthly_upper = ninio34.ninio34_mon >= ninio34.ninio34_mon.quantile(0.90, dim='dim_0', interpolation='linear') index_monthly_lower = ninio34.ninio34_mon <= ninio34.ninio34_mon.quantile(0.10, dim='dim_0', interpolation='linear') index_monthly_normal = np.logical_and(ninio34.ninio34_mon < ninio34.ninio34_mon.quantile(0.90, dim='dim_0', interpolation='linear'), ninio34.ninio34_mon > ninio34.ninio34_mon.quantile(0.10, dim='dim_0', interpolation='linear')) for i in np.arange(0,7): var = np.mean(ds.z.values[i, index_monthly_normal.values, :, :], axis=0) tit = 'S4 Z* 200hPa Neutral ENSO years - ' + month[i] filename = './new_figures_decile/z200_neutral_NINIO_' + month[i] +'.png' plots.PlotCompDiff(var, ds.latitude, ds.longitude, tit, filename) for i in np.arange(0,5): var = ds.isel(month=range(i, i + 3)).mean(dim='month') var = np.mean(var.z[index_monthly_normal.values, :, :], axis=0) tit = 'S4 Z* 200hPa Neutral ENSO years - ' + seas[i] filename = './new_figures_decile/z200_neutral_NINIO_' + seas[i] +'.png'
type=str, help="output filename of the plot to be generated -- PNG format") # unpack args args = parser.parse_args() fubu_fn = args.fubu_fn points_fn = args.points_fn out_fn = args.out_fn # # # TESTING # fubu_fn = '/workspace/Shared/Tech_Projects/SeaIce_NOAA_Indicators/project_data/nsidc_0051/outputs/NetCDF/nsidc_0051_sic_nasateam_1979-2013_Alaska_hann_smoothed_fubu_dates.nc' # points_fn = '/workspace/Shared/Tech_Projects/SeaIce_NOAA_Indicators/project_data/nsidc_0051/selection_points/chuckchi_points.shp' # out_fn = '/workspace/Shared/Tech_Projects/SeaIce_NOAA_Indicators/project_data/nsidc_0051/outputs/png/chuckchi_avg_fig7.png' # # # # # ds = xr.open_dataset(fubu_fn, autoclose=True) a = Affine(*eval( ds.affine_transform)[:6]) # make an affine transform for lookups # make barrow points and get their row/col locs points = gpd.read_file(points_fn).geometry.apply(lambda x: (x.x, x.y)).tolist() colrows = [~a * pt for pt in points] colrows = [(int(c), int(r)) for c, r in colrows] cols = [c for c, r in colrows] rows = [r for c, r in colrows] # make a dataframe of the FUBU dates metrics = ['freezeup_start', 'freezeup_end', 'breakup_start', 'breakup_end'] out = [] for metric in metrics: hold = [ds[metric][:, r, c].values for c, r in colrows]
import glob import datetime import pandas as pd import sys today = datetime.date.today() import sys sys.path.insert(0, '/home/albert7a/git/xscale') import xscale ## data location and gridfile data_dir = '/store/CT1/hmg2840/lbrodeau/eNATL60/eNATL60-BLBT02-S/' gridfile = '/store/CT1/hmg2840/lbrodeau/eNATL60/eNATL60-I/mesh_mask_eNATL60_3.6_lev1.nc4' dsgrid = xr.open_dataset(gridfile, chunks={'x': 1000, 'y': 1000}) ## box indices def read_csv(box): boxes = pd.read_csv('/home/albert7a/git/formation_ANNA/make_boxes/boxes_' + str(box) + '_1x1_eNATL60.csv', sep='\t', index_col=0) imin = boxes['imin'] imax = boxes['imax'] jmin = boxes['jmin'] jmax = boxes['jmax'] box_name = boxes.index return imin, imax, jmin, jmax, box_name
import xarray as xr import matplotlib.pyplot as plt import numpy as np # 解决出图中文和负号显示问题 # 指定默认字体 plt.rcParams['font.sans-serif'] = ['SimHei'] # 解决保存图像是负号'-'显示为方块的问题 plt.rcParams['axes.unicode_minus'] = False filepath = r"rhum.2016.nc" nc20 = xr.open_dataset(filepath) # 读取各量 lons = nc20.lon lats = nc20.lat levs = nc20.level times = nc20.time rhums = nc20.rhum #数据切片及相关处理 # 选择点位,经纬度(105,30),时间2016.6.14.08-20时 lon = lons[42] lat = lats[24] t = times[660:689] s = np.arange(660, 689) rhum = rhums.isel(lon=42, lat=24, time=s, level=[0, 1, 2, 3, 4, 5, 6, 7]) # 矩阵转置 rh = rhum.T # 画图 fig = plt.figure(figsize=(10, 8)) ax = fig.subplots()
def get_usgs_data(station_id, start_date, end_date, parameter="00060", cache_dir=None): """Get river discharge data from the USGS REST web service. See `U.S. Geological Survey Water Services <https://waterservices.usgs.gov/>`_ (USGS) Parameters ---------- station_id : str The station id to get start_date : str String for start date in the format: 'YYYY-MM-dd', e.g. '1980-01-01' end_date : str String for start date in the format: 'YYYY-MM-dd', e.g. '2018-12-31' parameter : str The parameter code to get, e.g. ('00060') discharge, cubic feet per second cache_dir : str Directory where files retrieved from the web service are cached. If set to None then USGS_DATA_HOME env var will be used as cache directory. Examples -------- >>> from ewatercycle.observation.usgs import get_usgs_data >>> data = get_usgs_data('03109500', '2000-01-01', '2000-12-31', cache_dir='.') >>> data <xarray.Dataset> Dimensions: (time: 8032) Coordinates: * time (time) datetime64[ns] 2000-01-04T05:00:00 ... 2000-12-23T04:00:00 Data variables: Streamflow (time) float32 8.296758 10.420501 ... 10.647034 11.694747 Attributes: title: USGS Data from streamflow data station: Little Beaver Creek near East Liverpool OH stationid: 03109500 location: (40.6758974, -80.5406244) """ # noqa: E501 if cache_dir is None: cache_dir = os.environ["USGS_DATA_HOME"] # Check if we have the netcdf data netcdf = os.path.join( cache_dir, "USGS_" + station_id + "_" + parameter + "_" + start_date + "_" + end_date + ".nc", ) if os.path.exists(netcdf): return xr.open_dataset(netcdf) # Download the data if needed out = os.path.join( cache_dir, "USGS_" + station_id + "_" + parameter + "_" + start_date + "_" + end_date + ".wml", ) if not os.path.exists(out): collector = UsgsRest() collector.filter( start=datetime.strptime(start_date, "%Y-%m-%d"), end=datetime.strptime(end_date, "%Y-%m-%d"), variables=[parameter], features=[station_id], ) data = collector.raw() with open(out, "w") as file: file.write(data) collector.clear() else: with open(out, "r") as file: data = file.read() # Convert the raw data to an xarray data = WaterML11ToPaegan(data).feature # We expect only 1 station if len(data.elements) == 0: raise ValueError("Data does not contain any station data") else: station = data.elements[0] # Unit conversion from cubic feet to cubic meter per second values = np.array( [ float(point.members[0]["value"]) / 35.315 for point in station.elements ], dtype=np.float32, ) times = [point.time for point in station.elements] attrs = { "units": "cubic meters per second", } # Create the xarray dataset ds = xr.Dataset({"streamflow": (["time"], values, attrs)}, coords={"time": times}) # Set some nice attributes ds.attrs["title"] = "USGS Data from streamflow data" ds.attrs["station"] = station.name ds.attrs["stationid"] = station.get_uid() ds.attrs["location"] = (station.location.y, station.location.x) ds.to_netcdf(netcdf) return ds
df = e.to_pandas( parse_dates=True, skiprows=(1, ) # units information can be dropped. ).dropna() argo_ids = np.asarray(df['platform_number']) argo_times = np.asarray(df['time (UTC)']) argo_press = np.asarray(df['pres (decibar)']) argo_lons = np.asarray(df['longitude (degrees_east)']) argo_lats = np.asarray(df['latitude (degrees_north)']) argo_temps = np.asarray(df['temp (degree_Celsius)']) argo_salts = np.asarray(df['psal (PSU)']) #%% GOFS 3.1 GOFS_ts = xr.open_dataset(url_GOFS_ts, decode_times=False) GOFS_uv = xr.open_dataset(url_GOFS_uv, decode_times=False) lt_GOFS = np.asarray(GOFS_ts['lat'][:]) ln_GOFS = np.asarray(GOFS_ts['lon'][:]) tt = GOFS_ts['time'] t_GOFS = netCDF4.num2date(tt[:], tt.units) depth_GOFS = np.asarray(GOFS_ts['depth'][:]) # Conversion from glider longitude and latitude to GOFS convention lon_limG = np.empty((len(lon_lim), )) lon_limG[:] = np.nan for i in range(len(lon_lim)): if lon_lim[i] < 0: lon_limG[i] = 360 + lon_lim[i]
# Input directories in_dir = "/data/ABOVE/MODIS/MYD21A2/" shp_dir = "/data/home/hamiddashti/mnt/nasa_above/working/modis_analyses/Study_area/" out_dir = "/data/ABOVE/MODIS/MYD21A2/MYD_QC_Night_Tiles/" chunks = {"lat": 2692, "lon": 8089} def myclip(tileID): import xarray as xr import rioxarray print(" Cliping QC Night tile ID:" + str(tileID)) tile_shp = geodf[geodf["OBJECTID"] == tileID] da.rio.set_crs(4326) da_clip = da.rio.clip(tile_shp.geometry.apply(mapping), tile_shp.crs) da_clip.to_netcdf(out_dir + "MYD_QC_Night_Tile_" + str(tileID) + ".nc") # Read shapefile geodf = gpd.read_file(shp_dir + "Above_180km_Clip_Geographic_Core.shp") IDs = np.arange(1, len(geodf) + 1) # Read the nc file ds = xr.open_dataset(in_dir + "MYD_QC_Night.nc", chunks=chunks) # ds = xr.open_dataset('subqc.nc',chunks=chunks) ds = ds.rename({"lat": "y", "lon": "x"}) da = ds["QC_Night"] for tileID in IDs: myclip(tileID) t2 = time.time() print(t2 - t1)