def test_input_tmax_and_tmin():
p_min = "/snow3/huziy/Daymet_daily/daymet_v3_tmin_1988_na.nc4"
p_max = "/snow3/huziy/Daymet_daily/daymet_v3_tmax_1988_na.nc4"
tmx = xarray.open_dataset(p_max)["tmax"][0, :, :]
tmn = xarray.open_dataset(p_min)["tmin"][0, :, :]
tmn_msk = tmn.to_masked_array()
tmx_msk = tmx.to_masked_array()
diff = tmx_msk - tmn_msk
print(diff.shape)
plt.figure()
suspect = (diff < 0.) & ~np.isnan(diff) & ~diff.mask
if np.any(suspect):
print(diff[suspect], len(suspect), diff[suspect].min(), "...", diff[suspect].max())
# print(tmx_msk[suspect])
# print(tavg_msk[suspect])
# print(tmn_msk[suspect])
diff[suspect] = -100
im = plt.pcolormesh(tmn.x, tmn.y, diff, cmap=cm.get_cmap("coolwarm", 20), vmin=-2, vmax=2)
plt.colorbar(im)
plt.show()
def from_netcdf(netcdf_path):
"""Load serialized TimeSeries from netCDF file."""
with netCDF4.Dataset(netcdf_path) as ds:
channels = list(ds.groups)
# First channel group stores time series metadata
with xr.open_dataset(netcdf_path, group=channels[0]) as ds:
t = [ds.time.values]
m = [ds.measurement.values]
e = [ds.error.values]
target = ds.attrs.get('target')
meta_features = ds.meta_features.to_series()
name = ds.attrs.get('name')
path = ds.attrs.get('path')
for channel in channels[1:]:
with xr.open_dataset(netcdf_path, group=channel) as ds:
m.append(ds.measurement.values)
if 'time' in ds:
t.append(ds.time.values)
if 'error' in ds:
e.append(ds.error.values)
return TimeSeries(_make_array(t), _make_array(m), _make_array(e), target,
meta_features, name, path)
def xrload(file_name, engine="h5netcdf", load_to_mem=True, create_new=False):
"""
Loads a xarray dataset.
Parameters
----------
file_name: name of file
engine: engine used to load file
load_to_mem: once opened, load from disk to memory
create_new: if no file exists make a blank one
Returns
-------
ds: loaded Dataset
"""
file_name = auto_add_extension(file_name, engine)
try:
try:
ds = xr.open_dataset(file_name, engine=engine)
except AttributeError as e1:
if "object has no attribute" in str(e1):
ds = xr.open_dataset(file_name, engine="netcdf4")
else:
raise e1
if load_to_mem:
ds.load()
ds.close()
except (RuntimeError, OSError) as e2:
if "o such" in str(e2) and create_new:
ds = xr.Dataset()
else:
raise e2
return ds
def __init__(self,config_name='config.ini',init=False,*args,**kwargs):
#Shared parameters as attributes
super(FAPAR,self).__init__(config_name,init,*args,**kwargs)
#FAPAR parameters,
super(FAPAR,self).import_config(config_name,'FAPAR')
#Load observation file and rename/reorder dimensions
self.obs=xr.open_dataset(self.fnobs,decode_times=False).FAPAR
self.obs=self.obs.rename({'TAX':'TIME'})
self.obs=self.obs.transpose('TIME','LATITUDE','LONGITUDE')
#Overwrite timeaxis with datetime object
self.timeaxis=pd.date_range('1997-09-01',freq='m',periods=106)
self.obs['TIME']=self.timeaxis
self.obs_mean=self.obs.mean(dim='TIME')
#Monthly mean
self.obs_monthly=self.obs.groupby('TIME.month').mean(dim='TIME')
#Load par from control run
self.par=xr.open_dataset(self.fnctrl_m).par.isel(grid_only=0).drop('grid_only')
self.tconstraint=(self.par.TIME>1997.65) * (self.par.TIME<2006.5)
self.par=self.par[self.tconstraint]
self.par['TIME']=self.timeaxis
self.name='fapar'
#Load area from control run, stack it for seasonal mean
self.area=xr.open_dataset(self.fnctrl).area
self.area_monthly=xr.concat([self.area]*12,dim='month')
self.area_monthly.month.values=range(1,13)
def test_qc(self):
nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed', 'nutnr_a_sample')
ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed', 'ctdpf_sbe43_sample')
nutnr_fn = 'nutnr_a_sample.nc'
ctdpf_fn = 'ctdpf_sbe43_sample.nc'
cals = json.load(open(os.path.join(DATA_DIR, 'cals.json')))
qc = json.load(open(os.path.join(DATA_DIR, 'qc.json')))
tr = TimeRange(3.65342400e+09, 3.65351040e+09)
coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': cals[k], 'deployment': 1}] for k in cals}
sr = StreamRequest(nutnr_sk, [2443], coefficients, tr, {}, qc_parameters=qc, request_id='UNIT')
nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn), decode_times=False)
ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn), decode_times=False)
ctdpf_ds = ctdpf_ds[self.base_params + [p.name for p in sr.stream_parameters[ctdpf_sk]]]
nutnr_ds = nutnr_ds[self.base_params + [p.name for p in sr.stream_parameters[nutnr_sk]]]
sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients, sr.uflags, [nutnr_sk], sr.request_id)
sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients, sr.uflags, [ctdpf_sk], sr.request_id)
sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds)
sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds)
sr.calculate_derived_products()
expected_parameters = ['temp_sal_corrected_nitrate_qc_executed',
'temp_sal_corrected_nitrate_qc_results']
self.assert_parameters_in_datasets(sr.datasets[nutnr_sk].datasets, expected_parameters)
def test_coordinates_encoding(self):
def equals_latlon(obj):
return obj == 'lat lon' or obj == 'lon lat'
original = Dataset({'temp': ('x', [0, 1]), 'precip': ('x', [0, -1])},
{'lat': ('x', [2, 3]), 'lon': ('x', [4, 5])})
with self.roundtrip(original) as actual:
self.assertDatasetIdentical(actual, original)
with create_tmp_file() as tmp_file:
original.to_netcdf(tmp_file)
with open_dataset(tmp_file, decode_coords=False) as ds:
self.assertTrue(equals_latlon(ds['temp'].attrs['coordinates']))
self.assertTrue(equals_latlon(ds['precip'].attrs['coordinates']))
self.assertNotIn('coordinates', ds.attrs)
self.assertNotIn('coordinates', ds['lat'].attrs)
self.assertNotIn('coordinates', ds['lon'].attrs)
modified = original.drop(['temp', 'precip'])
with self.roundtrip(modified) as actual:
self.assertDatasetIdentical(actual, modified)
with create_tmp_file() as tmp_file:
modified.to_netcdf(tmp_file)
with open_dataset(tmp_file, decode_coords=False) as ds:
self.assertTrue(equals_latlon(ds.attrs['coordinates']))
self.assertNotIn('coordinates', ds['lat'].attrs)
self.assertNotIn('coordinates', ds['lon'].attrs)
def create_metbk_hourly_sr(self):
metbk_fn = 'metbk_a_dcl_instrument_recovered.nc'
metbk_ds = xr.open_dataset(os.path.join(DATA_DIR, metbk_fn), decode_times=False)
vel_fn = 'velpt_ab_dcl_instrument_recovered.nc'
vel_ds = xr.open_dataset(os.path.join(DATA_DIR, vel_fn), decode_times=False)
# both of these datasets are labeled deployment 3 but the times are squarely in deployment 1. Fix.
metbk_ds.deployment.values[:] = 1
vel_ds.deployment.values[:] = 1
tr = TimeRange(metbk_ds.time.values[0], metbk_ds.time.values[-1])
sr = StreamRequest(self.hourly_sk, [], tr, {}, request_id='UNIT')
metbk_ds = metbk_ds[self.base_params + [p.name for p in sr.stream_parameters[self.met_sk]]]
vel_ds = vel_ds[self.base_params + [p.name for p in sr.stream_parameters[self.vel_sk]]]
sr.datasets[self.met_sk] = StreamDataset(self.met_sk, sr.uflags, [self.hourly_sk, self.vel_sk], sr.request_id)
sr.datasets[self.hourly_sk] = StreamDataset(self.hourly_sk, sr.uflags, [self.met_sk, self.vel_sk], sr.request_id)
sr.datasets[self.vel_sk] = StreamDataset(self.vel_sk, sr.uflags, [self.hourly_sk, self.met_sk], sr.request_id)
sr.datasets[self.hourly_sk].events = self.met_events
sr.datasets[self.met_sk].events = self.met_events
sr.datasets[self.vel_sk].events = self.vel_events
sr.datasets[self.met_sk]._insert_dataset(metbk_ds)
sr.datasets[self.vel_sk]._insert_dataset(vel_ds)
return sr
def test_csv(self):
nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed', 'nutnr_a_sample')
ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed', 'ctdpf_sbe43_sample')
nutnr_fn = 'nutnr_a_sample.nc'
ctdpf_fn = 'ctdpf_sbe43_sample.nc'
cals = json.load(open(os.path.join(DATA_DIR, 'cals.json')))
tr = TimeRange(3.65342400e+09, 3.65351040e+09)
coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': cals[k], 'deployment': 1}] for k in cals}
sr = StreamRequest(nutnr_sk, [2443], coefficients, tr, {}, request_id='UNIT')
nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn), decode_times=False)
ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn), decode_times=False)
nutnr_ds = nutnr_ds[self.base_params + [p.name for p in sr.stream_parameters[nutnr_sk]]]
ctdpf_ds = ctdpf_ds[self.base_params + [p.name for p in sr.stream_parameters[ctdpf_sk]]]
sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients, sr.uflags, [nutnr_sk], sr.request_id)
sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients, sr.uflags, [ctdpf_sk], sr.request_id)
sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds)
sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds)
sr.calculate_derived_products()
csv = CsvGenerator(sr, ',').to_csv()
self.assertTrue(csv)
def test_calculate_external_single_deployment(self):
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False)
nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False)
ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0']]
nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels',
'frame_type', 'nutnr_dark_value_used_for_fit']]
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT')
ctd_stream_dataset.events = self.ctd_events
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_all()
nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT')
nut_stream_dataset.events = self.nut_events
nut_stream_dataset._insert_dataset(nut_ds)
nut_stream_dataset.calculate_all()
nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset})
nut_stream_dataset.calculate_all()
expected_params = ['ctdpf_sbe43_sample-seawater_temperature',
'ctdpf_sbe43_sample-practical_salinity',
'salinity_corrected_nitrate']
self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def woa_get_ts(llon, llat, plot=0):
import xarray as xr
tempfile = '/Users/gunnar/Data/world_ocean_atlas/woa13_decav_t00_04v2.nc'
saltfile = '/Users/gunnar/Data/world_ocean_atlas/woa13_decav_s00_04v2.nc'
dt = xr.open_dataset(tempfile, decode_times=False)
a = dt.isel(time=0)
a.reset_coords(drop=True)
t = a['t_mn']
T = t.sel(lon=llon, lat=llat, method='nearest').values
ds = xr.open_dataset(saltfile, decode_times=False)
a = ds.isel(time=0)
a.reset_coords(drop=True)
s = a['s_mn']
S = s.sel(lon=llon, lat=llat, method='nearest').values
depth = s['depth'].data
if plot:
# import gvfigure as gvf
import matplotlib.pyplot as plt
# fig,ax = gvf.newfig(3,5)
# plt.plot(T,depth)
# ax.invert_yaxis()
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)
ax1.plot(T, depth, 'k')
ax1.set_xlabel('Temperature')
ax1.set_ylabel('Depth [m]')
ax2.plot(S, depth, 'k')
ax2.set_xlabel('Salinity')
ax1.invert_yaxis()
f.set_figwidth(5)
f.set_figheight(5)
return T, S, depth
def test_calculate_external_multiple_deployments(self):
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False)
nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False)
ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0']]
nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels',
'frame_type', 'nutnr_dark_value_used_for_fit']]
# remap times to make this two separate deployments
dep1_start = self.ctd_events.deps[1].ntp_start
dep2_stop = self.ctd_events.deps[2].ntp_start + 864000
ctd_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=ctd_ds.time.shape[0])
nut_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=nut_ds.time.shape[0])
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT')
ctd_stream_dataset.events = self.ctd_events
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_all()
nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT')
nut_stream_dataset.events = self.nut_events
nut_stream_dataset._insert_dataset(nut_ds)
nut_stream_dataset.calculate_all()
nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset})
nut_stream_dataset.calculate_all()
expected_params = ['ctdpf_sbe43_sample-seawater_temperature',
'ctdpf_sbe43_sample-practical_salinity',
'salinity_corrected_nitrate']
self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def test_distribute_climate_cru(self):
hef_file = get_demo_file('Hintereisferner.shp')
entity = gpd.GeoDataFrame.from_file(hef_file).iloc[0]
gdirs = []
gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir)
gis.define_glacier_region(gdir, entity=entity)
gdirs.append(gdir)
gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir_cru)
gis.define_glacier_region(gdir, entity=entity)
gdirs.append(gdir)
climate.distribute_climate_data([gdirs[0]])
cru_dir = get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc')
cru_dir = os.path.dirname(cru_dir)
cfg.PATHS['climate_file'] = '~'
cfg.PATHS['cru_dir'] = cru_dir
climate.distribute_climate_data([gdirs[1]])
cfg.PATHS['cru_dir'] = '~'
cfg.PATHS['climate_file'] = get_demo_file('histalp_merged_hef.nc')
gdh = gdirs[0]
gdc = gdirs[1]
with xr.open_dataset(os.path.join(gdh.dir, 'climate_monthly.nc')) as nc_h:
with xr.open_dataset(os.path.join(gdc.dir, 'climate_monthly.nc')) as nc_c:
# put on the same altitude
# (using default gradient because better)
temp_cor = nc_c.temp -0.0065 * (nc_h.ref_hgt - nc_c.ref_hgt)
totest = temp_cor - nc_h.temp
self.assertTrue(totest.mean() < 0.5)
# precip
totest = nc_c.prcp - nc_h.prcp
self.assertTrue(totest.mean() < 100)
def test_add_externals(self):
nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed', 'nutnr_a_sample')
ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed', 'ctdpf_sbe43_sample')
nutnr_fn = 'nutnr_a_sample.nc'
ctdpf_fn = 'ctdpf_sbe43_sample.nc'
cals = json.load(open(os.path.join(DATA_DIR, 'cals.json')))
tr = TimeRange(3.65342400e+09, 3.65351040e+09)
coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': cals[k], 'deployment': 1}] for k in cals}
sr = StreamRequest(nutnr_sk, [2443], coefficients, tr, {}, request_id='UNIT')
nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn), decode_times=False)
ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn), decode_times=False)
ctdpf_ds = ctdpf_ds[self.base_params + [p.name for p in sr.stream_parameters[ctdpf_sk]]]
nutnr_ds = nutnr_ds[self.base_params + [p.name for p in sr.stream_parameters[nutnr_sk]]]
sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients, sr.uflags, [nutnr_sk], sr.request_id)
sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients, sr.uflags, [ctdpf_sk], sr.request_id)
sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds)
sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds)
sr.calculate_derived_products()
sr.import_extra_externals()
self.assertIn('ctdpf_sbe43_sample-seawater_pressure', sr.datasets[nutnr_sk].datasets[0])
self.assertNotIn('ctdpf_sbe43_sample-seawater_pressure', sr.datasets[ctdpf_sk].datasets[0])
data = json.loads(JsonResponse(sr).json())
for each in data:
self.assertIn('int_ctd_pressure', each)
def test_calculate_external_single_deployment(self):
tr = TimeRange(3.65342400e+09, 3.65351040e+09)
coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': v, 'deployment': 1}]
for k, v in self.ctd_nutnr_cals.iteritems()}
coefficients = CalibrationCoefficientStore(coefficients, 'UNIT')
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False)
nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False)
ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0']]
nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels',
'frame_type', 'nutnr_dark_value_used_for_fit']]
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, coefficients, {}, [], 'UNIT')
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_internal()
nut_stream_dataset = StreamDataset(self.nutnr_sk, coefficients, {}, [self.ctdpf_sk], 'UNIT')
nut_stream_dataset._insert_dataset(nut_ds)
nut_stream_dataset.calculate_internal()
nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset})
nut_stream_dataset.calculate_external()
expected_params = ['ctdpf_sbe43_sample-seawater_temperature',
'ctdpf_sbe43_sample-practical_salinity',
'temp_sal_corrected_nitrate']
self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def main(in_file: Path, target_grid_file: Path, out_dir: Path=None):
if out_dir is not None:
out_dir.mkdir(exist_ok=True)
out_file = out_dir / (in_file.name + "_interpolated")
else:
out_file = in_file.parent / (in_file.name + "_interpolated")
if out_file.exists():
print(f"Skipping {in_file}, output already exists ({out_file})")
return
with xarray.open_dataset(target_grid_file) as ds_grid:
lons, lats = ds_grid["lon"][:].values, ds_grid["lat"][:].values
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons.flatten(), lats.flatten())
with xarray.open_dataset(in_file) as ds_in:
lons_s, lats_s = ds_in["lon"][:].values, ds_in["lat"][:].values
xs, ys, zs = lat_lon.lon_lat_to_cartesian(lons_s.flatten(), lats_s.flatten())
ktree = KDTree(list(zip(xs, ys, zs)))
dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)
# resample to daily
ds_in_r = ds_in.resample(t="1D", keep_attrs=True).mean()
ds_out = xarray.Dataset()
for vname, var in ds_grid.variables.items():
ds_out[vname] = var[:]
ds_out["t"] = ds_in_r["t"][:]
for vname, var in ds_in_r.variables.items():
assert isinstance(var, xarray.Variable)
var = var.squeeze()
# only interested in (t, x, y) fields
if var.ndim != 3:
print(f"skipping {vname}")
continue
if vname.lower() not in ["t", "time", "lon", "lat"]:
print(f"Processing {vname}")
var_interpolated = [var[ti].values.flatten()[inds].reshape(lons.shape) for ti in range(var.shape[0])]
ds_out[vname] = xarray.DataArray(
var_interpolated, dims=("t", "x", "y"),
attrs=var.attrs,
)
ds_out.to_netcdf(out_file)
def get_xarray(self):
try:
self.dataset = xr.open_dataset(self.dataset_path)
except ValueError as ve:
print(datetime_error, ve)
self.dataset = xr.open_dataset(self.dataset_path, decode_times=False)
print(success_msg)
return self.dataset
def get_xarray_datasets_from_filenames(filenames):
if isinstance(filenames, string_types):
datasets = [xarray.open_dataset(filenames)]
else:
datasets = []
for filename in filenames:
datasets.append(xarray.open_dataset(filename))
return datasets
def convert(self):
import xarray as xr
try:
# try OpenDAP url
return xr.open_dataset(self.url)
except IOError:
# download the file
return xr.open_dataset(self.file)
def test_image_driver_no_output_file_nans(fnames, domain_file):
'''
Test that all VIC image driver output files have the same nan structure as
the domain file
'''
ds_domain = xr.open_dataset(domain_file)
for fname in fnames:
ds_output = xr.open_dataset(fname)
assert_nan_equal(ds_domain, ds_output)
def delta_mm( fn, mean_fn, variable, mean_variable='tas' ): ''' simple way to compute extreme - mean deltas as native model resolution and write to NetCDF4 on disk ''' ds = xr.open_dataset( fn )[ variable ] ds_mean = xr.open_dataset( mean_fn )[ mean_variable ] delta = ds - ds_mean return delta.to_dataset( name=variable )
def read(self, file_path):
self.file_path = file_path
try:
self.dataset = xr.open_dataset(self.file_path)
except ValueError as ve:
print(datetime_error, ve)
self.dataset = xr.open_dataset(self.file_path, decode_times=False)
print(success_msg)
self.var_names = self.get_var_names(self.dataset)
def get_sim(self,memberid):
"returns mean simulated soilcarbon between 1982 and 2005 (As specified in Carvalhais, last page)"
fnmember=self.path2cdf+memberid+'.cdf'
#Open Dataset
sim=xr.open_dataset(fnmember).soilcarbon
luarea=xr.open_dataset(fnmember).lu_area
#Time constraint
tconstraint=(sim.TIME>1982.1) * (sim.TIME<2005.9)
sum_sim=(sim[tconstraint]*luarea[tconstraint]).sum(dim='landuse',skipna=False).mean(dim='TIME',skipna=False)
return sum_sim
def test_write_groups(self):
data1 = create_test_data()
data2 = data1 * 2
with create_tmp_file() as tmp_file:
data1.to_netcdf(tmp_file, group='data/1')
data2.to_netcdf(tmp_file, group='data/2', mode='a')
with open_dataset(tmp_file, group='data/1') as actual1:
self.assertDatasetIdentical(data1, actual1)
with open_dataset(tmp_file, group='data/2') as actual2:
self.assertDatasetIdentical(data2, actual2)
def concat_simulations(fp_files, variables=['spec001_mr', 'WD_spec001', 'DD_spec001']):
""" concatentate a series of flexpart simulations
works with a list of netcdf input files from FP > v10 and a list of variable
names. Only those variables will be returned in the concatenated dataset.
Inputs:
fp_files = list of flexpart netcdf files
variables = list of variable names to concat
Assumptions:
Uses a function `reduce_dataset` which extracts the variables, it also
takes a finite difference for all the deposition variables (cumulative)
Requires all runs have the same:
loutstep, dxout, dyout and dimesions
"""
# The following could work, but is too memory intensive
# Need to do some profiling, to see where speedup/opt is possible
# with one year of runs (1 yearly GAINS, 12 monthly GFED) this method
# is 3 minutes, vs. 4 minutes as implemented, but memory cons. higher
if False:
dsets = [xr.open_dataset(f) for f in fp_files]
nds = [reduce_dataset(d, variables) for d in dsets]
nds = concat_runs(nds)
clsd = [d.close() for d in dsets]
ds = nds
for i, f in enumerate(fp_files):
print(f)
with xr.open_dataset(f) as dset:
dset = xr.open_dataset(f)
nds = reduce_dataset(dset, variables)
if i == 0:
ds = nds
continue
ds = concat_runs([ds, nds])
dset.close()
_end_datetime = {'iedate': dset.attrs['iedate'],
'ietime': dset.attrs['ietime']}
# provide some further attribute information
ds.attrs.update(_end_datetime)
ds.attrs['history'] = """ Concatentation of:
{0}
using: {1}""".format(fp_files, inspect.stack()[0][3])
ds.attrs['institution'] = "UiO"
ds.attrs['contact'] = "John F. Burkhart <*****@*****.**>"
return ds.squeeze() # we return and get rid of nageclass/pointspec dims
def get_previous_hours(date):
ehour = 12
if (date.hour) <= 16:
print('Nighttime')
edate = date - pd.Timedelta('1 days')
edate = edate.replace(hour=ehour)
else:
edate = date
edate = edate.replace(hour=ehour)
t1 = edate
t2 = edate + pd.Timedelta('3 hours')
file = constants.ERA5
try:
cmm = xr.open_dataset(file + 'ERA5_'+str(date.year)+'_pls.nc')
except:
return None
pl_clim = xr.open_dataset(file + 'CLIM/ERA5_2008-2010_CLIM_'+str(edate.month)+'-'+str(ehour)+'_pls.nc')
cmm = cmm.sel(time=t1)
cm = cmm['t'] - pl_clim['t']
cm = cm.to_dataset()
vwind_srfc = cmm['v'] - pl_clim['v']
uwind_srfc = cmm['u'] - pl_clim['u']
div = cmm['d'] - pl_clim['d']
q = cmm['q'] - pl_clim['q']
cm['u950'] = uwind_srfc
cm['v950'] = vwind_srfc
cm['div'] = div *1000
cm['q'] = q
cm['qclim'] = pl_clim['q']
cm['divclim'] = pl_clim['d']
cm['bclim'] = pl_clim['v']
cm['uclim'] = pl_clim['u']
pl_clim.close()
return cm
def __init__(self,config_name='config.ini',init=False,*args,**kwargs):
#Shared parameters as attributes
super(SoilCMap,self).__init__(config_name,init,*args,**kwargs)
#FAPAR parameters,
super(SoilCMap,self).import_config(config_name,'SoilCMap')
#Load observation file and rename/reorder dimensions
self.obs=xr.open_dataset(self.fnobs).cSoilTotal
#Load area from control run, stack it for seasonal mean
self.area=xr.open_dataset(self.fnctrl).area
#Name
self.name='SoilCMap'
def test_dask_roundtrip(self):
with create_tmp_file() as tmp:
data = create_test_data()
data.to_netcdf(tmp)
chunks = {'dim1': 4, 'dim2': 4, 'dim3': 4, 'time': 10}
with open_dataset(tmp, chunks=chunks) as dask_ds:
self.assertDatasetIdentical(data, dask_ds)
with create_tmp_file() as tmp2:
dask_ds.to_netcdf(tmp2)
with open_dataset(tmp2) as on_disk:
self.assertDatasetIdentical(data, on_disk)
def get_sim(self,memberid):
"Get mean evapotranspiration from 1989-2005"
fnmember=self.path2cdf+memberid+'.cdf'
#Open Datasets
evap=xr.open_dataset(fnmember).evap
transp=xr.open_dataset(fnmember).transp
etransp=evap+transp
luarea=xr.open_dataset(fnmember).lu_area
#Time constraint (+0.6 to count endyear as well..)
tconstraint=(evap.TIME>float(self.startyear)) * (evap.TIME<float(self.endyear)+0.6)
sum_etransp=(etransp[tconstraint]*luarea[tconstraint]).sum(dim='landuse',skipna=False).mean(dim='TIME',skipna=False)
return sum_etransp
def __init__(self, dataset_path):
self.dataset_path = dataset_path
try:
try:
self.dataset = xr.open_dataset(self.dataset_path)
except ValueError as ve:
print(datetime_error, ve)
self.dataset = xr.open_dataset(self.dataset_path, decode_times=False)
print(success_msg)
except OSError:
print('File not found.')
exit()
def delta_mm( fn, mean_fn, variable, mean_variable, output_filename ): ''' simple way to compute extreme - mean deltas as native model resolution and write to NetCDF4 on disk ''' import xarray as xr ds = xr.open_dataset( fn )[ variable ] ds_mean = xr.open_dataset( mean_fn )[ mean_variable ] delta = ds - ds_mean delta_ds = delta.to_dataset( name=variable ) delta_ds.to_netcdf( output_filename ) return output_filename
def main(plot_dir):
# layer thickness
zse = np.array([.022, .058, .154, .409, 1.085, 2.872])
plc_rf_all = np.zeros(0)
plc_wsf_all = np.zeros(0)
plc_dsf_all = np.zeros(0)
plc_grw_all = np.zeros(0)
plc_saw_all = np.zeros(0)
sw_rf_all = np.zeros(0)
sw_wsf_all = np.zeros(0)
sw_dsf_all = np.zeros(0)
sw_grw_all = np.zeros(0)
sw_saw_all = np.zeros(0)
start_yr = 2000
end_yr = 2010
nyears = (end_yr - start_yr) + 1
nmonths = 12
fdir = "outputs"
fname = os.path.join(fdir, "cable_out_2000.nc")
ds = xr.open_dataset(fname)
iveg = ds["iveg"][:, :].values
idx_rf = np.argwhere(iveg == 18.0)
idx_wsf = np.argwhere(iveg == 19.0)
idx_dsf = np.argwhere(iveg == 20.0)
idx_grw = np.argwhere(iveg == 21.0)
idx_saw = np.argwhere(iveg == 22.0)
plc_rf_all = np.zeros((nyears * nmonths, len(idx_rf)))
plc_wsf_all = np.zeros((nyears * nmonths, len(idx_wsf)))
plc_dsf_all = np.zeros((nyears * nmonths, len(idx_dsf)))
plc_grw_all = np.zeros((nyears * nmonths, len(idx_grw)))
plc_saw_all = np.zeros((nyears * nmonths, len(idx_saw)))
sw_rf_all = np.zeros((nyears * nmonths, len(idx_rf)))
sw_wsf_all = np.zeros((nyears * nmonths, len(idx_wsf)))
sw_dsf_all = np.zeros((nyears * nmonths, len(idx_dsf)))
sw_grw_all = np.zeros((nyears * nmonths, len(idx_grw)))
sw_saw_all = np.zeros((nyears * nmonths, len(idx_saw)))
nyear = 0
cnt = 0
for year in np.arange(start_yr, end_yr):
print(year)
fdir = "outputs"
fname = os.path.join(fdir, "cable_out_%d.nc" % (year))
ds = xr.open_dataset(fname)
plc_vals = ds["plc"][:, 0, :, :].values
"""
SoilMoist1 = ds["SoilMoist"][:,0,:,:].values * zse[0]
SoilMoist2 = ds["SoilMoist"][:,1,:,:].values * zse[1]
SoilMoist3 = ds["SoilMoist"][:,2,:,:].values * zse[2]
SoilMoist4 = ds["SoilMoist"][:,3,:,:].values * zse[3]
SoilMoist5 = ds["SoilMoist"][:,4,:,:].values * zse[4]
SoilMoist6 = ds["SoilMoist"][:,5,:,:].values * zse[5]
sw = (SoilMoist1 + SoilMoist2 + SoilMoist3 + \
SoilMoist4 + SoilMoist5 + SoilMoist6 ) / np.sum(zse)
"""
SoilMoist1 = ds["SoilMoist"][:, 0, :, :].values * zse[0]
SoilMoist2 = ds["SoilMoist"][:, 1, :, :].values * zse[1]
SoilMoist3 = ds["SoilMoist"][:, 2, :, :].values * zse[2]
SoilMoist4 = ds["SoilMoist"][:, 3, :, :].values * zse[3]
sw = (SoilMoist1 + SoilMoist2 + \
SoilMoist3 + SoilMoist4) / np.sum(zse[0:4])
idx = nyear + cnt
plc_rf = np.zeros((12, len(idx_rf)))
sw_rf = np.zeros((12, len(idx_rf)))
for i in range(len(idx_rf)):
(row, col) = idx_rf[i]
plc_rf[:, i] = plc_vals[:, row, col]
sw_rf[:, i] = sw[:, row, col]
plc_rf_all[idx:(idx + 12), :] = plc_rf
sw_rf_all[idx:(idx + 12), :] = sw_rf
plc_wsf = np.zeros((12, len(idx_wsf)))
sw_wsf = np.zeros((12, len(idx_wsf)))
for i in range(len(idx_wsf)):
(row, col) = idx_wsf[i]
plc_wsf[:, i] = plc_vals[:, row, col]
sw_wsf[:, i] = sw[:, row, col]
plc_wsf_all[idx:(idx + 12), :] = plc_wsf
sw_wsf_all[idx:(idx + 12), :] = sw_wsf
plc_dsf = np.zeros((12, len(idx_dsf)))
sw_dsf = np.zeros((12, len(idx_dsf)))
for i in range(len(idx_dsf)):
(row, col) = idx_dsf[i]
plc_dsf[:, i] = plc_vals[:, row, col]
sw_dsf[:, i] = sw[:, row, col]
plc_dsf_all[idx:(idx + 12), :] = plc_dsf
sw_dsf_all[idx:(idx + 12), :] = sw_dsf
plc_grw = np.zeros((12, len(idx_grw)))
sw_grw = np.zeros((12, len(idx_grw)))
for i in range(len(idx_grw)):
(row, col) = idx_grw[i]
plc_grw[:, i] = plc_vals[:, row, col]
sw_grw[:, i] = sw[:, row, col]
plc_grw_all[idx:(idx + 12), :] = plc_grw
sw_grw_all[idx:(idx + 12), :] = sw_grw
plc_saw = np.zeros((12, len(idx_saw)))
sw_saw = np.zeros((12, len(idx_saw)))
for i in range(len(idx_saw)):
(row, col) = idx_saw[i]
plc_saw[:, i] = plc_vals[:, row, col]
sw_saw[:, i] = sw[:, row, col]
plc_saw_all[idx:(idx + 12), :] = plc_saw
sw_saw_all[idx:(idx + 12), :] = sw_saw
nyear += 1
cnt += 12
#from matplotlib.pyplot import cm
#colours = cm.Set2(np.linspace(0, 1, 5))
#colours = cm.get_cmap('Set2')
labels = label_generator('lower', start="(", end=")")
import seaborn as sns
sns.set_style("ticks")
colours = sns.color_palette("Set2", 8)
fig = plt.figure(figsize=(20, 9))
fig.subplots_adjust(hspace=0.1)
fig.subplots_adjust(wspace=0.15)
plt.rcParams['text.usetex'] = False
plt.rcParams['font.family'] = "sans-serif"
plt.rcParams['font.sans-serif'] = "Helvetica"
plt.rcParams['axes.labelsize'] = 20
plt.rcParams['font.size'] = 20
plt.rcParams['legend.fontsize'] = 20
plt.rcParams['xtick.labelsize'] = 20
plt.rcParams['ytick.labelsize'] = 20
ax1 = fig.add_subplot(231)
ax2 = fig.add_subplot(232)
ax3 = fig.add_subplot(233)
ax4 = fig.add_subplot(234)
ax5 = fig.add_subplot(235)
ax1.scatter(sw_rf_all,
plc_rf_all,
marker=".",
s=0.05,
alpha=0.08,
color=colours[0],
rasterized=True)
ax2.scatter(sw_wsf_all,
plc_wsf_all,
marker=".",
s=0.05,
alpha=0.08,
color=colours[0],
rasterized=True)
ax3.scatter(sw_dsf_all,
plc_dsf_all,
marker=".",
s=0.05,
alpha=0.08,
color=colours[0],
rasterized=True)
ax4.scatter(sw_grw_all,
plc_grw_all,
marker=".",
s=0.05,
alpha=0.08,
color=colours[0],
rasterized=True)
ax5.scatter(sw_saw_all,
plc_saw_all,
marker=".",
s=0.05,
alpha=0.08,
color=colours[0],
rasterized=True)
props = dict(boxstyle='round', facecolor='white', alpha=1.0, ec="white")
fig_label = "%s %s" % (next(labels), "RF")
ax1.text(0.8,
0.8,
fig_label,
transform=ax1.transAxes,
fontsize=14,
verticalalignment='top',
bbox=props)
fig_label = "%s %s" % (next(labels), "WSF")
ax2.text(0.8,
0.8,
fig_label,
transform=ax2.transAxes,
fontsize=14,
verticalalignment='top',
bbox=props)
fig_label = "%s %s" % (next(labels), "DSF")
ax3.text(0.8,
0.8,
fig_label,
transform=ax3.transAxes,
fontsize=14,
verticalalignment='top',
bbox=props)
fig_label = "%s %s" % (next(labels), "GRW")
ax4.text(0.8,
0.8,
fig_label,
transform=ax4.transAxes,
fontsize=14,
verticalalignment='top',
bbox=props)
fig_label = "%s %s" % (next(labels), "SAW")
ax5.text(0.8,
0.8,
fig_label,
transform=ax5.transAxes,
fontsize=14,
verticalalignment='top',
bbox=props)
ax1.axhline(y=88.0, ls="--", lw=1, color="black", label="$\Psi$$_{crit}$")
ax2.axhline(y=88.0, ls="--", lw=1, color="black")
ax3.axhline(y=88.0, ls="--", lw=1, color="black")
ax4.axhline(y=88.0, ls="--", lw=1, color="black")
ax5.axhline(y=88.0, ls="--", lw=1, color="black")
ax1.legend(numpoints=1, loc=(0.67, 0.6), ncol=1, frameon=False)
ax1.set_ylim(-5, 100)
ax2.set_ylim(-5, 100)
ax3.set_ylim(-5, 100)
ax4.set_ylim(-5, 100)
ax5.set_ylim(-5, 100)
ax1.set_xlim(0.1, 0.3)
ax2.set_xlim(0.1, 0.3)
ax3.set_xlim(0.1, 0.3)
ax4.set_xlim(0.1, 0.3)
ax5.set_xlim(0.1, 0.3)
ax1.set_ylabel("Loss of hydraulic conductivity (%)", position=(0.5, 0.0))
ax5.set_xlabel(r"$\theta$ (m$^{3}$ m$^{-3}$)")
#ax.legend(numpoints=1, loc=(0.01, 0.65), ncol=1, frameon=False)
plt.setp(ax1.get_xticklabels(), visible=False)
plt.setp(ax2.get_xticklabels(), visible=False)
plt.setp(ax2.get_yticklabels(), visible=False)
plt.setp(ax3.get_yticklabels(), visible=False)
plt.setp(ax5.get_yticklabels(), visible=False)
from matplotlib.ticker import MaxNLocator
ax1.xaxis.set_major_locator(MaxNLocator(4))
ax2.xaxis.set_major_locator(MaxNLocator(4))
ax3.xaxis.set_major_locator(MaxNLocator(4))
ax4.xaxis.set_major_locator(MaxNLocator(4))
ax5.xaxis.set_major_locator(MaxNLocator(4))
ax1.yaxis.set_major_locator(MaxNLocator(4))
ax2.yaxis.set_major_locator(MaxNLocator(4))
ax3.yaxis.set_major_locator(MaxNLocator(4))
ax4.yaxis.set_major_locator(MaxNLocator(4))
ax5.xaxis.set_major_locator(MaxNLocator(4))
plt.savefig(os.path.join(plot_dir, "plc_vs_sw_all.pdf"),
bbox_inches='tight',
pad_inches=0.1)
OutSR.ImportFromEPSG(102009) # WGS84 UTM Zone 56 South
OutSR.SetProjParm("Central_Meridian", -100.0)
OutSR.SetProjParm("Latitude_Of_Origin", 42.5)
OutSR.SetProjParm("Standard_Parallel_1", 25.0)
OutSR.SetProjParm("Standard_Parallel_2", 60.0)
Point = ogr.Geometry(ogr.wkbPoint)
Point.AddPoint(loni,lati) # use your coordinates here
Point.AssignSpatialReference(InSR) # tell the point what coordinates it's in
Point.TransformTo(OutSR) # project it to the out spatial reference
yi = Point.GetY()
xi = Point.GetX()
ds = xr.open_dataset(Daymet_file)
dsloc = ds.sel(x=xi, y=yi, method='nearest')
Tmax = dsloc['tmax'].values
time = dsloc['time'].values
plt.figure(figsize=(16,10))
plt.plot(time, Tmax)
plt.title('Daily Maximum Air Temperature at Wyoming Big Sage Site from Daymet')
plt.xlabel('Time')
plt.ylabel('$T_{max} {}^{\circ}C$')
plt.show()
WORKPLACE = r"C:\Users\z3439910\Documents\Kien\1_Projects\2_Msc\1_E1\5_GIS_project"
IRDIR = WORKPLACE + r"\IRimages2012"
BTDIR = WORKPLACE + r"\2_IBTrACSfiles"
os.chdir(IRDIR)
CHOSEN_YEAR = "2012"
BASIN = "NI"
IMAG_RES = 4 #km
r = 500
DEG_TO_KM = 111 #ratio
if BASIN == "WP":
LAT_BOUND = [-20, 60] #WP Basin
LON_BOUND = [60, 180] #WP Basin
B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.WP.v04r00.nc")
elif BASIN == "EP":
LAT_BOUND = [-20, 60] #WP Basin
LON_BOUND = [-180, -60] #WP Basin
B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.EP.v04r00.nc")
elif BASIN == "NI":
LAT_BOUND = [-20, 60] #WP Basin
LON_BOUND = [0, 120] #WP Basin
B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.NI.v04r00.nc")
elif BASIN == "SI":
LAT_BOUND = [-60, 20] #WP Basin
LON_BOUND = [0, 120] #WP Basin
B_tracks = xr.open_dataset(BTDIR + "\\" + "IBTrACS.SI.v04r00.nc")
B_TC_serials_decode = [x.decode("utf-8") for x in B_tracks['sid'].values]
################################################################################
# 1a
################################################################################
# Run MC
print("Running Monte Carlo simulation for Exercise 1 a.")
subprocess.call(["./exercise_1_a"])
# Load matplotlib style
print("Plotting results.")
path = os.path.dirname(os.path.abspath(__file__))
plt.style.use(os.path.join(path, "..", "misc", "matplotlib_style.rc"))
results = []
for i in range(3):
data = xarray.open_dataset("results_1_a_{}_cuda.nc".format(i + 1))
results += [np.array(data.irradiance)]
# Results of Beer-Lambert law
x = np.linspace(0, 10e3, 1001)
sigma = np.zeros(x.shape)
sigma[:] = 1e-4
tau = cumtrapz(sigma, x = x)
f = plt.figure(figsize = (10, 5))
gs = GridSpec(1, 2, width_ratios = [1.0, 1.0])
ax = plt.subplot(gs[0])
handles = []
handles += ax.plot(x[1:] / 1e3, np.exp(-tau), ls = "--", c = "k", label = "Beer-Lambert Law")
csvdir = './data/'
nwanews = lgn.news2roms(grdfile)
nwanews.read_database(csvdir)
nwanews.extract_domain()
# do this if you want to see locations or rivers mouth
#nwanews.plot_river_mouth(db='domain')
nwanews.move_rivermouth2roms()
# you can pass a list of rivers to check, will show spreading performed
#no3_conc = nwanews.create_rivers_input('NO3_CONC',river_checklist=['Susquehanna'])
no3_conc = nwanews.create_rivers_input('NO3_CONC', plot_result=False)
po4_conc = nwanews.create_rivers_input('PO4_CONC', plot_result=False)
# you can do the same for all other nutrients
# create the output file with xarray
grd = xr.open_dataset(grdfile)
ds = xr.Dataset(
{
'no3_conc': (['eta_rho', 'xi_rho'], no3_conc),
'po4_conc': (['eta_rho', 'xi_rho'], po4_conc),
# add other nutrient to dataset
},
coords={
'lon_rho': (['eta_rho', 'xi_rho'], grd.lon_rho),
'lat_rho': (['eta_rho', 'xi_rho'], grd.lat_rho)
})
ds.to_netcdf('nutrients_conc_NWA.nc')
'bufr' : '/raid60/scratch/federico/Harvested_Databases/bufr' }
for d,p in zip( [db], [data_directories[db] ] ):
print(' I am here')
dic_all = {}
dic = ''
print('Processing the dataset: ' , d , ' ' , p )
files = [ k for k in os.listdir(p) if '.nc' in k ] ### edit here
for f in files:
print(files.index(f) )
try:
File = xr.open_dataset( p + '/' + f , engine = 'h5netcdf' )
rts = File['recordtimestamp'].data
except:
continue
x = pd.to_datetime(rts)
try:
dic = dic.append(x)
except:
dic = x
dic_all[d] = dic
np.save('NEW_NUMPYS/records_' + d , dic_all, allow_pickle=True )
print('Finished ***', d )
mask_data = Dataset(mask_file)
climate_data = Dataset(climate_file)
print('Mask file dimensions:', mask_data.dimensions)
print('Climate file dimensions:',climate_data.dimensions)
"""
Identify the variables stored within each dataset:
"""
print('Mask file variables:', mask_data.variables)
print('Climate file variables:', climate_data.variables)
"""
3.1) Firstly, we will use xarray to assign the land cover percentage data to a new object.
"""
mask_dset = xr.open_dataset(mask_file) #Use xarray to open the mask dataset
land_perc = mask_dset['sftlf'] # assign the land percentage variable to a new object
# sftlf is the standardised term for land percentage cover
print('Land_perc:', land_perc)
print('Max land area:', land_perc.data.max(), '%') # check that max land area is 100 %
print('Min land area:', land_perc.data.min(), '%') # check that min land area is 0 %
"""
3.2) Second we will load in the climate data and assign temperature (tas) to a new object.
"""
climate_dset = xr.open_dataset(climate_file)
clim = climate_dset['tas'].mean('time', keep_attrs=True) # create a new object with the temperature data for each grid cell
# keep_attrs maintains attribute data (e.g. units, variable names etc) in new object
"""
SW=SW)
field.make_base()
y_predict = self.predict(field._X)
outfield = MakeBigImage(y_predict,
ind=(field._indouty, field._indoutx))
return outfield
if __name__ == "__main__":
name = '../data/model_upar.pkl'
nn = loadmymodel(name)
# nn = mymodel(model=nntmp._model,
# moyX = nntmp._moyX,etX=nntmp._etX,
# moyY = nntmp._moyY,etY=nntmp._etY)
testfile = '../data/test_image.nc'
tdata = xr.open_dataset(testfile)
infield = ['uphy', 'hphy']
outfield = 'uparam'
forcfield = ['taux']
y = nn.predictfield(tdata,
outfield=outfield,
infield=infield,
forcfield=forcfield)
fig, (ax1, ax2, ax3) = plt.subplots(ncols=3)
p1 = ax1.imshow(tdata['uparam'][1, :, :])
p2 = ax2.imshow(y)
p3 = ax3.imshow(y - tdata['uparam'][1, :, :], cmap=plt.get_cmap('bwr'))
fig.colorbar(p1, ax=ax1)
fig.colorbar(p2, ax=ax2)
fig.colorbar(p3, ax=ax3)
psd_files = sorted([
i for i in glob.glob(
'/home/disk/eos9/jkcm/Data/particle/psd/compressed/UW_particle_classifications.1hz.rf*_psd.nc'
) if not 'rf15' in i
])
flight_files = sorted([
i for i in glob.glob('/home/disk/eos9/jfinlon/socrates/*/*.PNI.nc')
if not 'RF15' in i
])
psds = xr.open_mfdataset(psd_files, combine='by_coords')
ds = []
for f in sorted(flight_files):
flights = xr.open_dataset(f)
flights = flights.drop('PSTFC', errors='ignore')
ds.append(flights)
flights = xr.concat(ds, dim='Time', data_vars=['ATX'], compat="no_conflicts")
flights = flights.rename({'Time': 'time'})
fig, axg = plt.subplots(figsize=(15, 10),
nrows=2,
ncols=3,
sharex=True,
sharey=True)
axl = axg.flatten()
all_cloud = psds.count_darea_all.sum(dim='size_bin') > 10
temp_ranges = [-40, -20, -5, 0, 5, 40, 100]
def main(fname, plot_dir):
ds = xr.open_dataset(fname, decode_times=False)
bottom, top = np.min(ds.latitude).values, np.max(ds.latitude).values
left, right = np.min(ds.longitude).values, np.max(ds.longitude).values
ntime, nrows, ncols = ds.VOD.shape
# Get baseline period
# 1993-1999
nyears = 7
vod_pre = np.zeros((nyears, nrows, ncols))
vals = np.zeros((3, nrows, ncols))
yr_count = 0
count = 0
for year in np.arange(1993, 2000):
for month in np.arange(1, 13):
if month == 12:
#vals = ds.VOD[count,:,:].values
#vals = np.where(np.isnan(vals), 0, vals)
vod_pre[yr_count, :, :] += ds.VOD[count, :, :]
vod_pre[yr_count, :, :] += ds.VOD[count + 1, :, :]
vod_pre[yr_count, :, :] += ds.VOD[count + 2, :, :]
vod_pre[yr_count, :, :] /= 3
count += 1
yr_count += 1
vod_pre = np.mean(vod_pre, axis=0)
vod_pre = np.flipud(vod_pre)
# We will have incremented the counter by one too many on the final
# iteration, fix this as we need to start at the right point for 2000
count = count - 1
# 2000-2009
nyears = 10
vod_dur = np.zeros((nyears, nrows, ncols))
yr_count = 0
for year in np.arange(2000, 2010):
for month in np.arange(1, 13):
if month == 12:
vod_dur[yr_count, :, :] += ds.VOD[count, :, :]
vod_dur[yr_count, :, :] += ds.VOD[count + 1, :, :]
vod_dur[yr_count, :, :] += ds.VOD[count + 2, :, :]
vod_dur[yr_count, :, :] /= 3
count += 1
yr_count += 1
vod_dur = np.mean(vod_dur, axis=0)
vod_dur = np.flipud(vod_dur)
chg = ((vod_dur - vod_pre) / vod_pre) * 100.0
fig = plt.figure(figsize=(9, 6))
plt.rcParams['font.family'] = "sans-serif"
plt.rcParams['font.size'] = "14"
plt.rcParams['font.sans-serif'] = "Helvetica"
cmap = plt.cm.get_cmap('BrBG', 10) # discrete colour map
projection = ccrs.PlateCarree()
axes_class = (GeoAxes, dict(map_projection=projection))
rows = 1
cols = 1
axgr = AxesGrid(fig,
111,
axes_class=axes_class,
nrows_ncols=(rows, cols),
axes_pad=0.2,
cbar_location='right',
cbar_mode='single',
cbar_pad=0.5,
cbar_size='5%',
label_mode='') # note the empty label_mode
for i, ax in enumerate(axgr):
# add a subplot into the array of plots
#ax = fig.add_subplot(rows, cols, i+1, projection=ccrs.PlateCarree())
plims = plot_map(ax, chg, cmap, i, top, bottom, left, right)
import cartopy.feature as cfeature
states = cfeature.NaturalEarthFeature(
category='cultural',
name='admin_1_states_provinces_lines',
scale='10m',
facecolor='none')
# plot state border
SOURCE = 'Natural Earth'
LICENSE = 'public domain'
ax.add_feature(states, edgecolor='black', lw=0.5)
cbar = axgr.cbar_axes[0].colorbar(plims)
cbar.ax.set_title("% Difference", fontsize=16, pad=10)
#cbar.ax.set_yticklabels([' ', '-30', '-15', '0', '15', '<=70'])
ofname = os.path.join(plot_dir, "vod.png")
fig.savefig(ofname, dpi=300, bbox_inches='tight', pad_inches=0.1)
def t_ws_func(t_c, t_dew, dim="time"):
return xr.apply_ufunc(
t_ws_gufunc,
t_c,
t_dew,
input_core_dims=[[], []],
dask="parallelized",
output_dtypes=[output_dtype],
)
#%% =============== LOAD DATA ==================
try: # if file exists load the data
data = xr.open_dataset(save_dict["savefolder"] + save_dict["savename"])
print("=======\nload data done")
except: #%% ==================== CALCULATION OF SWBGT ================
print("=======\nfile not found")
folderpath = r"I:\Bachlor_thesis\Data"
st = time.time()
temp = xr.open_dataset(
folderpath +
r"\t2m_era20c_1900-2010.nc") #, chunks={'time': 12*4*365})
print("load temp took {:.2f}".format(time.time() - st))
st = time.time()
temp_dew = xr.open_dataset(
folderpath +
r"\td2m_era20c_1900-2010.nc") #, chunks={'time': 12*4*365})
print("load dew took {:.2f}".format(time.time() - st))
import cmocean #import proplot as plot #import seaborn as sns #import proplot as plot #fin = '/Users/cpatrizio/data/MERRA2/' #fin = '/Users/cpatrizio/data/ECMWF/' #fin = '/Users/cpatrizio/data/OAFlux/' #fout = '/Volumes/GoogleDrive/My Drive/PhD/figures/ocean-atmosphere/localSST_global/' fin = '/Volumes/GoogleDrive/My Drive/data_drive/MERRA2/' fout = '/Users/cpatrizio/figures_arc/' #MERRA-2 fsstM2 = xr.open_dataset(fin + 'MERRA2_SST_ocnthf_monthly1980to2017.nc') fthf = xr.open_dataset(fin + 'MERRA2_thf_monthly1980to2017.nc') fSLP = xr.open_dataset(fin + 'MERRA2_SLP_monthly1980to2017.nc') radfile = xr.open_dataset(fin + 'MERRA2_rad_monthly1980to2017.nc') cffile = xr.open_dataset(fin + 'MERRA2_modis_cldfrac_monthly1980to2017.nc') #frad = cdms2.open(fin + 'MERRA2_tsps_monthly1980to2017.nc') #fuv = cdms2.open(fin + 'MERRA2_uv_monthly1980to2017.nc') #fRH = cdms2.open(fin + 'MERRA2_qv10m_monthly1980to2017.nc') #fcE = cdms2.open(fin + 'MERRA2_cE_monthly1980to2017.nc') #fcD = cdms2.open(fin + 'MERRA2_cD_monthly1980to2017.nc') ftau = xr.open_dataset(fin + 'MERRA2_tau_monthly1980to2019.nc') fssh = xr.open_dataset(fin + 'ncep.ssh.198001-201912.nc') fseaice = xr.open_dataset(fin + 'MERRA2_seaice_monthly1980to2019.nc') #fh = xr.open_dataset(fin + 'ncep.mixedlayerdepth.198001-201712.nc')
#!/usr/bin/env python2 # -*- coding: utf-8 -*- """ Created on Mon Mar 20 19:44:28 2017 author: Craig Risien ([email protected]) reference: http://journals.ametsoc.org/doi/abs/10.1175/2008JPO3881.1 """ import xarray as xr import numpy as np from matplotlib import pyplot as plt from mpl_toolkits.basemap import Basemap ds = xr.open_dataset( 'http://chapman.ceoas.oregonstate.edu/thredds/dodsC/CIOSS/HadISST/Monthly/HadISST_sst.nc' ) ds.info() sst = ds.sst.data lat = ds.latitude.data lon = ds.longitude.data time = ds.time.data del ds #clear ds #subset the data for the period 1900-1999 sst = sst[360:1560, :, :] sst[sst == -1000] = np.nan #reshape the sst matrix tmp = np.zeros((180, 360, 1200)) for i in range(0, 1200):
hour = original_hour / 3
if hour % 3 != int(hour % 3):
print(hour % 3)
print("Invalid Hour: Rounding to the closest time")
hour = round(hour, 0)
if hour > 384:
print("Time out of 384hr bounds. Defaulting to 3 ")
hour = 1
hour = int(hour)
lat_north = 59
lat_south = -10
lon_west = 360
lon_east = 220
data = xr.open_dataset(dataset)
data = data.sel(lon=slice(lon_east, lon_west), lat=slice(lat_south, lat_north))
lon = data["lon"]
lat = data["lat"]
mslp = data["msletmsl"][hour] / 100
zr = data["cfrzrsfc"][hour]
ip = data["cicepsfc"][hour]
sn = data["csnowsfc"][hour]
rate = data["prateavesfc"][hour] * 1500
#Heights
Geo_500 = data["hgtprs"][hour].sel(lev=500) / 10
Geo_1000 = data["hgtprs"][hour].sel(lev=1000) / 10
remain = Geo_500 - Geo_1000
- Same grid (spatial)
- same time step (temporal)
- Subset ROI
- assign latitude, longitude, time
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import xarray as xr
from pathlib import Path
data_dir = Path("/Volumes/Lees_Extend/data/ecmwf_sowc/data")
data_dir = Path("data")
ds = xr.open_dataset(data_dir / "raw" / "seasonal" / "fcmean2018.grb",
engine="cfgrib")
# rename the variables to more reasonable names
ds = ds.rename({
"p146.172": "surface_sensible_heat_flux",
"p147.172": "surface_latent_heat_flux"
})
# rename the coords to more reasonable coords
ds = ds.rename({"time": "initialisation_date", "step": "forecast_horizon"})
# total precip rate m/s
d = ds.tprate
fig, ax = plt.subplots()
d.mean(dim=["initialisation_date", "forecast_horizon", "number"]).plot(ax=ax)
def addHistoricalData(station, CTDConfig, work_dir):
if station.name in ['VT69']:
ds = xr.open_dataset(
'/Users/trondkr/Dropbox/NIVA/OKOKYST/Historical_oekokyst_data/VT69_2013_2016.nc'
)
elif station.name in ['VT70']:
ds = xr.open_dataset(
'/Users/trondkr/Dropbox/NIVA/OKOKYST/Historical_oekokyst_data/VT70_2013_2016.nc'
)
else:
raise Exception(
"Should not call addHistoricalData unless station is VT69 or VT70 due to available timeseries"
)
df = ds.to_dataframe().groupby(['time', 'depth']).mean()
first = True
for date_depth_index, row in df.iterrows():
if first:
old_date = date_depth_index[0]
temperature = []
depth = []
salinity = []
oxsat = []
oxygen = []
ftu = []
first = False
if old_date == date_depth_index[0]:
temperature.append(row['temp'])
salinity.append(row['salt'])
oxsat.append(row['O2sat'])
oxygen.append(row['O2vol'])
depth.append(float(date_depth_index[1]))
ftu.append(row['O2vol'] * np.nan)
else:
print("Storing data for time {}".format(date_depth_index[0]))
depth = pd.Series(depth)
temperature = pd.Series(temperature, index=depth)
salinity = pd.Series(salinity, index=depth)
oxygen = pd.Series(oxygen, index=depth)
oxsat = pd.Series(oxsat, index=depth)
ftu = pd.Series(ftu, index=depth)
df_new = pd.DataFrame(columns=[
"Depth", "Temperature", "Salinity", "Oxygen", "Oxsat", "FTU"
])
df_new.set_index('Depth', inplace=True, drop=False)
df_new["Depth"] = depth
df_new["Temperature"] = temperature
df_new["Salinity"] = salinity
df_new["Oxygen"] = oxygen
df_new["Oxsat"] = oxsat
# Add data to station object for later
station.addData(
salinity, temperature, oxygen, oxsat, ftu, salinity.index,
date2num(date_depth_index[0],
CTDConfig.refdate,
calendar="standard"))
temperature = []
depth = []
salinity = []
oxsat = []
oxygen = []
ftu = []
old_date = date_depth_index[0]
temperature.append(row['temp'])
salinity.append(row['temp'])
oxsat.append(row['O2sat'])
oxygen.append(row['O2vol'])
depth.append(float(date_depth_index[1]))
ftu.append(row['O2vol'] * np.nan)
lat = [line.rstrip('\n') for line in open(os.path.join(path, 'grid', 'lat.csv'))]
lon = [line.rstrip('\n') for line in open(os.path.join(path, 'grid', 'lon.csv'))]
grid = []
i = 1
for lat_ in lat:
for lon_ in lon:
grid.append([lat_, lon_, i])
i += 1
files = glob.glob1(os.path.join(path,'Data'), '*.nc')
for file in files:
print(file)
name = file[-7:][:4]
d = xr.open_dataset(os.path.join(path,'Data', file))
d.precip.to_dataframe().to_csv(os.path.join(path, 'CSV', '{}.csv'.format(name)))
columns = ['date', 'lat', 'lon', 'pre']
df = pd.read_csv(os.path.join(path, 'CSV', '{}.csv'.format(name)), skiprows=1, names=columns)
df.loc[df['pre'] == -99, ['pre']] = np.nan
df_l = pd.DataFrame(grid, columns=['lat', 'lon', 'index'])
df_l['lat'] = df['lat'].astype(float)
df_l['lon'] = df['lon'].astype(float)
new_df = pd.merge(df, df_l, how='left', left_on=['lat', 'lon'], right_on=['lat', 'lon'])
new_df.drop(['lat', 'lon'], axis=1, inplace=True)
number_of_chunks = 20
[new_df.to_csv(os.path.join(path, 'CSV', '{name}_{id}.csv'.format(name=name, id=id)), index=False) for id, new_df in
# -------------------- general setup --------------------
t_main_start = time.time()
if args.verbose:
print("Running", sys.argv[0])
print(" -> verbose output = True")
print()
# -------------------- read basin mask --------------------
if args.verbose:
print("... reading basin mask from file '" + args.basin_file + "'")
t_read_start = time.time()
t_read_bmask_start = time.time()
try:
ds_basins = xr.open_dataset(args.basin_file)
except:
print("BASIN_FILE '", args.basin_file, "' can't be found! Exiting.")
sys.exit(1)
basins = ds_basins['basins']
# mask values:
# 0 = grounded ice or open ocean
# 1 = continental shelf but ignored
# 2 = continental shelf as used in PICO
contshelf_mask = ds_basins['pico_contshelf_mask']
# check dimensions
basin_ndim = len(basins.dims)
if basin_ndim != 2 and basin_ndim != 3:
raise ValueError( str("basin field has dimensions " + \
# Date created: 09 Sept 2020
# Description: Script imports gappy mooring timeseries, bins it to daily temperatures, and fits successively longer predicting LSTM models to fill in the gaps
# import packages
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
f'Tensorflow version: {tf.__version__}'
import xarray as xr
import os
import pickle
# open raw netcdf file
temp = xr.open_dataset('/srv/scratch/z3289452/OceanDataScience/Mooring/PH100/IMOS_ANMN-NSW_TZ_20091029_PH100_FV01_TEMP-aggregated-timeseries_END-20190612_C-20190819.nc')
# bin data at various pressure depths
bins = [0,11,19,20,28,106,114,116]
labs = [5.5,15,19.5,24,(106-28)/2,110,115]
# create pd dataframe from scratch
data = {'TIME': temp.TIME.values,
'TEMP': temp.TEMP.values,
'PRES': temp.PRES_REL.values}
temp_df= pd.DataFrame(data, columns = ['TIME','TEMP','PRES'])
# set TIME as the indexing variable
temp_df = temp_df.set_index('TIME')
# create a column with pressure bin labels
temp_df['PRES_BIN'] = pd.cut(temp_df.PRES, bins = bins, labels = labs, include_lowest=True)
# First groupby PRES_BIN column, then resample each group on hourly intervals and mean each resulting bin
init_data, init_data1 = 0, 0
map_sum, map_cnt, map_max = np.zeros([imap_lats, imap_lons]), np.zeros(
[imap_lats, imap_lons]), np.zeros([imap_lats, imap_lons])
map_sum_recov, map_cnt_recov = np.zeros([imap_lats, imap_lons]), np.zeros(
[imap_lats, imap_lons])
map_sum_mld, map_cnt_mld = np.zeros([imap_lats, imap_lons
]), np.zeros([imap_lats, imap_lons])
for inum_storm in range(0, 110):
# for inum_storm in range(16,17):
filename = dir_out + str(iyr_storm) + '/' + str(inum_storm).zfill(
3) + '_interpolated_track.nc'
exists = os.path.isfile(filename)
if not exists:
continue
print(filename)
ds_storm_info = xr.open_dataset(filename)
ds_storm_info = ds_storm_info.sel(j2=0)
ds_storm_info.close()
filename = dir_out + str(iyr_storm) + '/' + str(inum_storm).zfill(
3) + '_combined_data.nc'
ds_all = xr.open_dataset(filename)
ds_all['spd'] = np.sqrt(ds_all.uwnd**2 + ds_all.vwnd**2)
ds_all.close()
filename = dir_out + str(iyr_storm) + '/' + str(inum_storm).zfill(
3) + '_MLD_data_v2.nc'
ds_all2 = xr.open_dataset(filename)
ds_all2.close()
if abs(ds_all.lon[-1] - ds_all.lon[0]) > 180:
ds_all.coords['lon'] = np.mod(ds_all['lon'], 360)
ds_storm_info['lon'] = np.mod(ds_storm_info['lon'], 360)
max_lat = ds_storm_info.lat.max()
# --- Read in data --- #
# Set directories and file names
dailyDir = '/Users/meganfowler/Documents/NCAR/Analysis/Coupling_initial/data/day/'
middleName = '_day_CESM2_amip_r10i1p1f1_gn_'
varNames = ['hfls', 'hfss', 'mrso', 'mrsos', 'tas', 'hurs', 'ps']
# Set time period portion of fileNames
timeName_flx = [
'19700101-19791231', '19800101-19891231', '19900101-19991231',
'20000101-20091231', '20100101-20150101'
]
# Read in test file to get lat/lon
fileName = dailyDir + varNames[0] + middleName + timeName_flx[0] + '.nc'
testDF = xr.open_dataset(fileName, decode_times=True)
# Get lat and lon
lat = testDF.lat.values
lon = testDF.lon.values
# Read in surface properties (T, RH, P)
for iT in range(len(timeName_flx)):
# Sfc Temp
tFile = dailyDir + varNames[4] + middleName + timeName_flx[
iT] + '.nc' # File name
TsfcDF = xr.open_dataset(tFile, decode_times=True)
TsfcDF['time'] = TsfcDF.indexes['time'].to_datetimeindex()
# ===========================
# Load the Data
# ==========================
sns.set_context('paper')
# AVHRR
def preprocess(ds):
data = ds.sel(lat=slice(-90, -40))
return data
# Load all the MAR data
MAR = xr.open_dataset(
'/uio/lagringshotell/geofag/projects/miphclac/shofer/MAR/wBS/mon-CC-MAR_ERA5-1981-2018.nc'
)
MAR_noBS = xr.open_dataset(
'/uio/lagringshotell/geofag/projects/miphclac/shofer/MAR/noBS/mon-CC-MAR_ERA5-1980-2018.nc'
)
MAR_grid = xr.open_dataset(
'/uio/lagringshotell/geofag/projects/miphclac/shofer/MAR/wBS/MARcst-AN35km-176x148.cdf'
)
ds_grid = xr.Dataset(
{
'RIGNOT': (['y', 'x'], MAR_grid.RIGNOT.values),
'SH': (['y', 'x'], MAR_grid.SH.values),
'LAT': (['y', 'x'], MAR_grid.LAT.values),
'LON': (['y', 'x'], MAR_grid.LON.values)
},
coords={
def lagranto_plotting(traj_variable_name,
start_time,
end_time,
trajs_bunch_level='all',
subset=False,
save=False):
'''This function plots the chosen variables for the trajectories
calculated with Lagranto. Supported variables for plotting procedure
are water_vapor, updraft and height.'''
### Predefine some variables ###
traj_data_dir = '/scratch3/thomasl/work/retrospective_part/lagranto/' 'traj_baden_10000_every1000_area_1200.ll'
save_dir = '/scratch3/thomasl/work/retrospective_part' '/casestudy_baden/lagranto/'
start_locations = 'area' # distinction between single and multiple
# starting points of trajectories
number_trajs_plot = 1
# Location of Initiation
lat = 47.25
lon = 7.85
initiation = CoordPair(lat, lon)
# Change extent of plot
subset_extent = [7, 9, 47, 48]
# Variables for getting PBL height of WRF model data
date = '2018-05-30'
time = '16:10'
wrf_filename = '/scratch3/thomasl/work/data/casestudy_baden/' 'wrfout_d02_{}_{}:00'.format(
date, time)
# Variables:
if traj_variable_name == 'water_vapor':
traj_variable_name = 'QVAPOR'
title_name = 'Trajectories of Water Vapor'
colorbar_label_trajs = 'Water Vapor Mixing Ratio [$g$ $kg^{-1}$]'
save_name = 'trajectory_water_vapor_{}_{}'.format(start_time, end_time)
traj_variable_min = 0
traj_variable_max = 15
elif traj_variable_name == 'updraft':
traj_variable_name = 'W_UP_MAX'
title_name = 'Trajectories of Updraft'
colorbar_label_trajs = 'Max Z-Wind Updraft [$m$ $s^-$$^1$]'
save_name = 'trajectory_updraft_{}_{}'.format(start_time, end_time)
traj_variable_min = 0
traj_variable_max = 10
elif traj_variable_name == 'height':
traj_variable_name = 'z'
title_name = 'Height of Trajectories'
colorbar_label_trajs = 'Height of Trajectories [$m$]'
save_name = 'trajectory_height_{}_{}'.format(start_time, end_time)
traj_variable_min = 0
traj_variable_max = 10000
### Plotting procedure ###
trajs = Tra()
trajs.load_ascii(traj_data_dir)
# Get PBL out of WRF model data
ncfile = Dataset(wrf_filename)
wrf_file = xr.open_dataset(wrf_filename)
data = wrf_file.PBLH
location = ll_to_xy(ncfile, lat, lon)
data_point = data.sel(west_east=location[0], south_north=location[1])
pbl = data_point.values
# Separate trajectories in 3 vertical bunches
# Trajectories of pbl (according to pbl height of WRF model data)
trajspbl = []
for t in trajs:
if (t['z'][0] <= pbl):
trajspbl.append(t)
# Trajectories between pbl and 5000 m
trajs5 = []
for t in trajs:
if (t['z'][0] > pbl and t['z'][0] < 5000):
trajs5.append(t)
# Trajectories between 5000 m and 10000 m
trajs10 = []
for t in trajs:
if (t['z'][0] > 5000 and t['z'][0] <= 10000):
trajs10.append(t)
if trajs_bunch_level == 'pbl':
trajs_bunch = trajspbl
elif trajs_bunch_level == '5':
trajs_bunch = trajs5
elif trajs_bunch_level == '10':
trajs_bunch = trajs10
elif trajs_bunch_level == 'all':
trajs_bunch = trajs
# Get terrain height
terrain_height = getvar(ncfile, 'HGT') / 1000 # change to km
# Define cart projection
lats, lons = latlon_coords(terrain_height)
cart_proj = ccrs.LambertConformal(central_longitude=8.722206,
central_latitude=46.73585)
# Create figure
fig = plt.figure(figsize=(15, 10))
ax = plt.axes(projection=cart_proj)
### Set map extent ###
domain_extent = [3.701088, 13.814863, 43.85472, 49.49499]
if subset == True:
ax.set_extent([
subset_extent[0], subset_extent[1], subset_extent[2],
subset_extent[3]
], ccrs.PlateCarree())
else:
ax.set_extent([
domain_extent[0] + 0.7, domain_extent[1] - 0.7,
domain_extent[2] + 0.1, domain_extent[3] - 0.1
], ccrs.PlateCarree())
# Plot trajectories
levels_trajs = np.linspace(traj_variable_min, traj_variable_max, 11)
rounded_levels_trajs = [round(elem, 1) for elem in levels_trajs]
cmap = ListedColormap(sns.cubehelix_palette(
10,
start=.5,
rot=-.75,
))
plt_trajs = plot_trajs(ax,
trajs_bunch[0::number_trajs_plot],
traj_variable_name,
linewidth=2,
levels=rounded_levels_trajs,
cmap=cmap)
# Plot the terrain height with colorbar
levels = np.linspace(0, 4, 21)
terrain = plt.contourf(to_np(lons),
to_np(lats),
to_np(terrain_height),
levels=levels,
transform=ccrs.PlateCarree(),
cmap=get_cmap('Greys'),
alpha=0.75)
cbar = mpu.colorbar(terrain,
ax,
orientation='horizontal',
aspect=40,
shrink=.05,
pad=0.075)
cbar.set_label('Terrain Height [$km$]', fontsize=15)
cbar.set_ticks(levels)
# Make only every second color bar tick label visible
for label in cbar.ax.xaxis.get_ticklabels()[1::2]:
label.set_visible(False)
# Add color bar for trajectory variable
if traj_variable_name == 'QVAPOR':
extend = 'both'
else:
extend = 'max'
cbar_trajs = mpu.colorbar(plt_trajs,
ax,
orientation='vertical',
aspect=40,
shrink=.05,
pad=0.05,
extend=extend)
cbar_trajs.set_label(colorbar_label_trajs, fontsize=15)
cbar_trajs.set_ticks(rounded_levels_trajs)
# Add borders and coastlines
ax.add_feature(cfeature.BORDERS.with_scale('10m'), linewidth=0.8)
ax.add_feature(cfeature.COASTLINE.with_scale('10m'), linewidth=0.8)
# Add cross for initiation location
for t in trajs:
ax.plot(t['lon'][0], t['lat'][0], 'kx', transform=ccrs.PlateCarree())
# Add gridlines
lon = np.arange(0, 20, 1)
lat = np.arange(40, 60, 1)
gl = ax.gridlines(xlocs=lon, ylocs=lat, zorder=3)
# Add tick labels
mpu.yticklabels(lat, ax=ax, fontsize=12.5)
mpu.xticklabels(lon, ax=ax, fontsize=12.5)
# Set title
ax.set_title(title_name, loc='left', fontsize=20)
ax.set_title('Time Range: {} - {} UTC'.format(start_time, end_time),
loc='right',
fontsize=15)
plt.show()
### Save figure ###
if save == True:
if subset == True:
if trajs_bunch_level == 'all':
fig.savefig('{}{}_subset_{}.png'.format(
save_dir, save_name, start_locations),
bbox_inches='tight',
dpi=300)
else:
fig.savefig('{}{}_subset_{}_{}.png'.format(
save_dir, save_name, trajs_bunch_level, start_locations),
bbox_inches='tight',
dpi=300)
else:
if trajs_bunch_level == 'all':
fig.savefig('{}{}_{}.png'.format(save_dir, save_name,
start_locations),
bbox_inches='tight',
dpi=300)
else:
fig.savefig('{}{}_{}_{}.png'.format(save_dir, save_name,
trajs_bunch_level,
start_locations),
bbox_inches='tight',
dpi=300)
#sort el ninio events import numpy as np import xarray as xr import pandas as pd import plots import os os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' RUTA='~/datos/data/' #abro el archivo de geopotencial y junto la coordenada year y numbre ds = xr.open_dataset(RUTA + 'monthly_hgt200_aug_feb.nc') ds = ds - ds.mean(dim='longitude') ninio34 = xr.open_dataset(RUTA + 'ninio34_index.nc') month = ['Aug', 'Sep', 'Oct', 'Nov','Dec', 'Jan', 'Feb'] seas = ['ASO', 'SON', 'OND', 'NDJ', 'DJF'] index_monthly_upper = ninio34.ninio34_mon >= ninio34.ninio34_mon.quantile(0.90, dim='dim_0', interpolation='linear') index_monthly_lower = ninio34.ninio34_mon <= ninio34.ninio34_mon.quantile(0.10, dim='dim_0', interpolation='linear') index_monthly_normal = np.logical_and(ninio34.ninio34_mon < ninio34.ninio34_mon.quantile(0.90, dim='dim_0', interpolation='linear'), ninio34.ninio34_mon > ninio34.ninio34_mon.quantile(0.10, dim='dim_0', interpolation='linear')) for i in np.arange(0,7): var = np.mean(ds.z.values[i, index_monthly_normal.values, :, :], axis=0) tit = 'S4 Z* 200hPa Neutral ENSO years - ' + month[i] filename = './new_figures_decile/z200_neutral_NINIO_' + month[i] +'.png' plots.PlotCompDiff(var, ds.latitude, ds.longitude, tit, filename) for i in np.arange(0,5): var = ds.isel(month=range(i, i + 3)).mean(dim='month') var = np.mean(var.z[index_monthly_normal.values, :, :], axis=0) tit = 'S4 Z* 200hPa Neutral ENSO years - ' + seas[i] filename = './new_figures_decile/z200_neutral_NINIO_' + seas[i] +'.png'
type=str,
help="output filename of the plot to be generated -- PNG format")
# unpack args
args = parser.parse_args()
fubu_fn = args.fubu_fn
points_fn = args.points_fn
out_fn = args.out_fn
# # # TESTING
# fubu_fn = '/workspace/Shared/Tech_Projects/SeaIce_NOAA_Indicators/project_data/nsidc_0051/outputs/NetCDF/nsidc_0051_sic_nasateam_1979-2013_Alaska_hann_smoothed_fubu_dates.nc'
# points_fn = '/workspace/Shared/Tech_Projects/SeaIce_NOAA_Indicators/project_data/nsidc_0051/selection_points/chuckchi_points.shp'
# out_fn = '/workspace/Shared/Tech_Projects/SeaIce_NOAA_Indicators/project_data/nsidc_0051/outputs/png/chuckchi_avg_fig7.png'
# # # # #
ds = xr.open_dataset(fubu_fn, autoclose=True)
a = Affine(*eval(
ds.affine_transform)[:6]) # make an affine transform for lookups
# make barrow points and get their row/col locs
points = gpd.read_file(points_fn).geometry.apply(lambda x: (x.x, x.y)).tolist()
colrows = [~a * pt for pt in points]
colrows = [(int(c), int(r)) for c, r in colrows]
cols = [c for c, r in colrows]
rows = [r for c, r in colrows]
# make a dataframe of the FUBU dates
metrics = ['freezeup_start', 'freezeup_end', 'breakup_start', 'breakup_end']
out = []
for metric in metrics:
hold = [ds[metric][:, r, c].values for c, r in colrows]
import glob
import datetime
import pandas as pd
import sys
today = datetime.date.today()
import sys
sys.path.insert(0, '/home/albert7a/git/xscale')
import xscale
## data location and gridfile
data_dir = '/store/CT1/hmg2840/lbrodeau/eNATL60/eNATL60-BLBT02-S/'
gridfile = '/store/CT1/hmg2840/lbrodeau/eNATL60/eNATL60-I/mesh_mask_eNATL60_3.6_lev1.nc4'
dsgrid = xr.open_dataset(gridfile, chunks={'x': 1000, 'y': 1000})
## box indices
def read_csv(box):
boxes = pd.read_csv('/home/albert7a/git/formation_ANNA/make_boxes/boxes_' +
str(box) + '_1x1_eNATL60.csv',
sep='\t',
index_col=0)
imin = boxes['imin']
imax = boxes['imax']
jmin = boxes['jmin']
jmax = boxes['jmax']
box_name = boxes.index
return imin, imax, jmin, jmax, box_name
import xarray as xr import matplotlib.pyplot as plt import numpy as np # 解决出图中文和负号显示问题 # 指定默认字体 plt.rcParams['font.sans-serif'] = ['SimHei'] # 解决保存图像是负号'-'显示为方块的问题 plt.rcParams['axes.unicode_minus'] = False filepath = r"rhum.2016.nc" nc20 = xr.open_dataset(filepath) # 读取各量 lons = nc20.lon lats = nc20.lat levs = nc20.level times = nc20.time rhums = nc20.rhum #数据切片及相关处理 # 选择点位，经纬度（105,30），时间2016.6.14.08-20时 lon = lons[42] lat = lats[24] t = times[660:689] s = np.arange(660, 689) rhum = rhums.isel(lon=42, lat=24, time=s, level=[0, 1, 2, 3, 4, 5, 6, 7]) # 矩阵转置 rh = rhum.T # 画图 fig = plt.figure(figsize=(10, 8)) ax = fig.subplots()
def get_usgs_data(station_id,
start_date,
end_date,
parameter="00060",
cache_dir=None):
"""Get river discharge data from the USGS REST web service.
See `U.S. Geological Survey Water Services
<https://waterservices.usgs.gov/>`_ (USGS)
Parameters
----------
station_id : str
The station id to get
start_date : str
String for start date in the format: 'YYYY-MM-dd', e.g. '1980-01-01'
end_date : str
String for start date in the format: 'YYYY-MM-dd', e.g. '2018-12-31'
parameter : str
The parameter code to get, e.g. ('00060') discharge, cubic feet per second
cache_dir : str
Directory where files retrieved from the web service are cached.
If set to None then USGS_DATA_HOME env var will be used as cache directory.
Examples
--------
>>> from ewatercycle.observation.usgs import get_usgs_data
>>> data = get_usgs_data('03109500', '2000-01-01', '2000-12-31', cache_dir='.')
>>> data
<xarray.Dataset>
Dimensions: (time: 8032)
Coordinates:
* time (time) datetime64[ns] 2000-01-04T05:00:00 ... 2000-12-23T04:00:00
Data variables:
Streamflow (time) float32 8.296758 10.420501 ... 10.647034 11.694747
Attributes:
title: USGS Data from streamflow data
station: Little Beaver Creek near East Liverpool OH
stationid: 03109500
location: (40.6758974, -80.5406244)
""" # noqa: E501
if cache_dir is None:
cache_dir = os.environ["USGS_DATA_HOME"]
# Check if we have the netcdf data
netcdf = os.path.join(
cache_dir,
"USGS_" + station_id + "_" + parameter + "_" + start_date + "_" +
end_date + ".nc",
)
if os.path.exists(netcdf):
return xr.open_dataset(netcdf)
# Download the data if needed
out = os.path.join(
cache_dir,
"USGS_" + station_id + "_" + parameter + "_" + start_date + "_" +
end_date + ".wml",
)
if not os.path.exists(out):
collector = UsgsRest()
collector.filter(
start=datetime.strptime(start_date, "%Y-%m-%d"),
end=datetime.strptime(end_date, "%Y-%m-%d"),
variables=[parameter],
features=[station_id],
)
data = collector.raw()
with open(out, "w") as file:
file.write(data)
collector.clear()
else:
with open(out, "r") as file:
data = file.read()
# Convert the raw data to an xarray
data = WaterML11ToPaegan(data).feature
# We expect only 1 station
if len(data.elements) == 0:
raise ValueError("Data does not contain any station data")
else:
station = data.elements[0]
# Unit conversion from cubic feet to cubic meter per second
values = np.array(
[
float(point.members[0]["value"]) / 35.315
for point in station.elements
],
dtype=np.float32,
)
times = [point.time for point in station.elements]
attrs = {
"units": "cubic meters per second",
}
# Create the xarray dataset
ds = xr.Dataset({"streamflow": (["time"], values, attrs)},
coords={"time": times})
# Set some nice attributes
ds.attrs["title"] = "USGS Data from streamflow data"
ds.attrs["station"] = station.name
ds.attrs["stationid"] = station.get_uid()
ds.attrs["location"] = (station.location.y, station.location.x)
ds.to_netcdf(netcdf)
return ds
df = e.to_pandas(
parse_dates=True,
skiprows=(1, ) # units information can be dropped.
).dropna()
argo_ids = np.asarray(df['platform_number'])
argo_times = np.asarray(df['time (UTC)'])
argo_press = np.asarray(df['pres (decibar)'])
argo_lons = np.asarray(df['longitude (degrees_east)'])
argo_lats = np.asarray(df['latitude (degrees_north)'])
argo_temps = np.asarray(df['temp (degree_Celsius)'])
argo_salts = np.asarray(df['psal (PSU)'])
#%% GOFS 3.1
GOFS_ts = xr.open_dataset(url_GOFS_ts, decode_times=False)
GOFS_uv = xr.open_dataset(url_GOFS_uv, decode_times=False)
lt_GOFS = np.asarray(GOFS_ts['lat'][:])
ln_GOFS = np.asarray(GOFS_ts['lon'][:])
tt = GOFS_ts['time']
t_GOFS = netCDF4.num2date(tt[:], tt.units)
depth_GOFS = np.asarray(GOFS_ts['depth'][:])
# Conversion from glider longitude and latitude to GOFS convention
lon_limG = np.empty((len(lon_lim), ))
lon_limG[:] = np.nan
for i in range(len(lon_lim)):
if lon_lim[i] < 0:
lon_limG[i] = 360 + lon_lim[i]
# Input directories
in_dir = "/data/ABOVE/MODIS/MYD21A2/"
shp_dir = "/data/home/hamiddashti/mnt/nasa_above/working/modis_analyses/Study_area/"
out_dir = "/data/ABOVE/MODIS/MYD21A2/MYD_QC_Night_Tiles/"
chunks = {"lat": 2692, "lon": 8089}
def myclip(tileID):
import xarray as xr
import rioxarray
print(" Cliping QC Night tile ID:" + str(tileID))
tile_shp = geodf[geodf["OBJECTID"] == tileID]
da.rio.set_crs(4326)
da_clip = da.rio.clip(tile_shp.geometry.apply(mapping), tile_shp.crs)
da_clip.to_netcdf(out_dir + "MYD_QC_Night_Tile_" + str(tileID) + ".nc")
# Read shapefile
geodf = gpd.read_file(shp_dir + "Above_180km_Clip_Geographic_Core.shp")
IDs = np.arange(1, len(geodf) + 1)
# Read the nc file
ds = xr.open_dataset(in_dir + "MYD_QC_Night.nc", chunks=chunks)
# ds = xr.open_dataset('subqc.nc',chunks=chunks)
ds = ds.rename({"lat": "y", "lon": "x"})
da = ds["QC_Night"]
for tileID in IDs:
myclip(tileID)
t2 = time.time()
print(t2 - t1)
