def test_reindex_time(self):
"""Tests whether the reindex_time function works as expected."""
[yr, month, day] = [1850, 1, 15]
expected_times = xr.DataArray(
np.array([
cftime.DatetimeProlepticGregorian(yr, month, day, 0, 0),
cftime.DatetimeProlepticGregorian(yr, month + 1, day, 0, 0)
]))
testdates1 = xr.DataArray(
np.array([
np.datetime64(datetime.datetime(yr, month, day, 0, 0)),
np.datetime64(datetime.datetime(yr, month + 1, day, 0, 0))
]))
testdates2 = xr.DataArray(
np.array([
cftime.DatetimeNoLeap(yr, month, day, 0, 0),
cftime.DatetimeNoLeap(yr, month + 1, day, 0, 0)
]))
testdates3 = xr.DataArray(
np.array([
cftime.Datetime360Day(yr, month, day, 0, 0),
cftime.Datetime360Day(yr, month + 1, day, 0, 0)
]))
dates_to_test = [testdates1, testdates2, testdates3]
# Run test
dates_converted = True
for date_to_test in dates_to_test:
newtimes = reindex_time(date_to_test)
if not np.all((newtimes == expected_times).values):
dates_converted = False
else:
continue
self.assertTrue(dates_converted)
def test_adjust_quantiledeltamapping_year_output_time():
"""Check 'time' year and edges of QDM adjusted output
This tests integration between `dodola.core.adjust_quantiledeltamapping_year`,
`dodola.core.qdm_rollingyearwindow`, and
`dodola.core.train_quantiledeltamapping`.
"""
# Setup input data.
target_variable = "fakevariable"
variable_kind = "+"
n_simdays = 85 * 365
model_bias = 2.0
ts_sim = np.ones(n_simdays, dtype=np.float64)
sim = _timeseriesfactory(ts_sim + model_bias,
start_dt="2015-01-01",
variable_name=target_variable)
target_year = 2088
qdm = _train_simple_qdm(target_variable="fakevariable",
kind=variable_kind,
additive_bias=model_bias)
adjusted_ds = adjust_quantiledeltamapping_year(
simulation=sim,
qdm=qdm,
year=target_year,
variable=target_variable,
)
assert np.unique(adjusted_ds["time"].dt.year).item() == 2088
assert min(adjusted_ds["time"].values) == cftime.DatetimeNoLeap(
2088, 1, 1, 0, 0, 0, 0)
assert max(adjusted_ds["time"].values) == cftime.DatetimeNoLeap(
2088, 12, 31, 0, 0, 0, 0)
def year_float2datetime(year_float, resolution='day'):
if np.min(year_float) < 0:
raise ValueError(
'Cannot handel negative years. Please truncate first.')
return None
''' Convert an array of floats in unit of year to a datetime time; accuracy: one day
'''
year = np.array([int(y) for y in year_float], dtype=int)
month = np.zeros(np.size(year), dtype=int)
day = np.zeros(np.size(year), dtype=int)
for i, y in enumerate(year):
fst_day = datetime(year=y, month=1, day=1)
lst_day = datetime(year=y + 1, month=1, day=1)
year_length = lst_day - fst_day
year_part = (year_float[i] - y) * year_length + timedelta(
minutes=1) # to fix the numerical error
date = year_part + fst_day
month[i] = date.month
day[i] = date.day
if resolution == 'day':
time = [
cftime.DatetimeNoLeap(y, m, d, 0, 0, 0, 0, 0, 0)
for y, m, d in zip(year, month, day)
]
elif resolution == 'month':
time = [
cftime.DatetimeNoLeap(y, m, 1, 0, 0, 0, 0, 0, 0)
for y, m in zip(year, month)
]
return time
def test_shift_cftime_singular():
"""Tests that a singular ``cftime`` is shifted the appropriate amount."""
cftime_initial = cftime.DatetimeNoLeap(1990, 1, 1)
cftime_expected = cftime.DatetimeNoLeap(1990, 3, 1)
# Shift forward two months at month start.
cftime_from_func = shift_cftime_singular(cftime_initial, 2, "MS")
assert cftime_expected == cftime_from_func
def _read_single_nc(filename, yearrange, bbox):
"""Import data from single nc file, return as xarray
Parameters:
filename (str or pathlib.Path): full path of input netcdf file
yearrange: (tuple): year range to be extracted from file
bbox (tuple of float): geographical bounding box in the form:
(lon_min, lat_min, lon_max, lat_max)
Returns:
dis_xarray (xarray)
"""
dis_xarray = xr.open_dataset(filename)
try:
if not bbox:
return dis_xarray.sel(time=slice(dt.datetime(yearrange[0], 1, 1),
dt.datetime(yearrange[-1], 12, 31)))
lon_min, lat_min, lon_max, lat_max = bbox
return dis_xarray.sel(lon=slice(lon_min, lon_max), lat=slice(lat_max, lat_min),
time=slice(dt.datetime(yearrange[0], 1, 1),
dt.datetime(yearrange[-1], 12, 31)))
except TypeError:
# fix date format if not datetime
if not bbox:
dis_xarray = dis_xarray.sel(time=slice(cftime.DatetimeNoLeap(yearrange[0], 1, 1),
cftime.DatetimeNoLeap(yearrange[-1], 12, 31)))
else:
lon_min, lat_min, lon_max, lat_max = bbox
dis_xarray = dis_xarray.sel(lon=slice(lon_min, lon_max), lat=slice(lat_max, lat_min),
time=slice(cftime.DatetimeNoLeap(yearrange[0], 1, 1),
cftime.DatetimeNoLeap(yearrange[-1], 12, 31)))
datetimeindex = dis_xarray.indexes['time'].to_datetimeindex()
dis_xarray['time'] = datetimeindex
return dis_xarray
def fix_metadata(self, cubes):
"""Fix parent time units.
FGOALS-f3-L Amon data may have a bad time bounds spanning 20 days.
Parameters
----------
cubes : iris.cube.CubeList
Input cubes.
Returns
-------
iris.cube.CubeList
"""
for cube in cubes:
if cube.attributes['table_id'] == 'Amon':
time = cube.coord('time')
if np.any(time.bounds[:-1, 1] != time.bounds[1:, 0]):
times = time.units.num2date(time.points)
starts = [
cftime.DatetimeNoLeap(c.year, c.month, 1)
for c in times
]
ends = [
cftime.DatetimeNoLeap(c.year, c.month +
1, 1) if c.month < 12 else
cftime.DatetimeNoLeap(c.year + 1, 1, 1) for c in times
]
time.bounds = time.units.date2num(
np.stack([starts, ends], -1))
return cubes
def test_cftime_datetime_mean_long_time_period():
import cftime
times = np.array(
[
[
cftime.DatetimeNoLeap(400, 12, 31, 0, 0, 0, 0),
cftime.DatetimeNoLeap(520, 12, 31, 0, 0, 0, 0),
],
[
cftime.DatetimeNoLeap(520, 12, 31, 0, 0, 0, 0),
cftime.DatetimeNoLeap(640, 12, 31, 0, 0, 0, 0),
],
[
cftime.DatetimeNoLeap(640, 12, 31, 0, 0, 0, 0),
cftime.DatetimeNoLeap(760, 12, 31, 0, 0, 0, 0),
],
]
)
da = DataArray(times, dims=["time", "d2"])
result = da.mean("d2")
expected = DataArray(
[
cftime.DatetimeNoLeap(460, 12, 31, 0, 0, 0, 0),
cftime.DatetimeNoLeap(580, 12, 31, 0, 0, 0, 0),
cftime.DatetimeNoLeap(700, 12, 31, 0, 0, 0, 0),
],
dims=["time"],
)
assert_equal(result, expected)
def time_bnds_mon(years):
return xr.DataArray(np.array([[
cftime.date2num(cftime.DatetimeNoLeap(year, mon, 1),
units='days since 0001-01-01',
calendar='noleap') - 1,
cftime.date2num(cftime.DatetimeNoLeap(year, mon,
eom_day_noleap[mon - 1]),
units='days since 0001-01-01',
calendar='noleap')
] for year, mon in product(years, range(1, 13))]),
dims=('time', 'ntb'))
def remove_drift(da, ver_time, y1, y2):
'''function takes in a raw DP DataArray and returns anomalies with lead-time dependent drift+climo removed'''
d1 = cftime.DatetimeNoLeap(y1, 1, 1, 0, 0, 0)
d2 = cftime.DatetimeNoLeap(y2, 12, 31, 23, 59, 59)
masked_period = da.where((ver_time.mean('d2') > d1)
& (ver_time.mean('d2') < d2))
climodrift = masked_period.mean('M').mean('Y')
da_anom = da - climodrift
return da_anom, climodrift
def test_init_default_dates(self):
gdl = GFDLDataLoader(data_start_date=cftime.DatetimeNoLeap(1, 1, 1),
data_end_date=cftime.DatetimeNoLeap(1, 12, 31))
run_ = Run(data_loader=gdl)
self.assertEqual(run_.default_start_date,
cftime.DatetimeNoLeap(1, 1, 1))
self.assertEqual(run_.default_end_date,
cftime.DatetimeNoLeap(1, 12, 31))
ddl = DictDataLoader({'monthly': '/a/'})
run_ = Run(data_loader=ddl)
self.assertEqual(run_.default_start_date, None)
self.assertEqual(run_.default_end_date, None)
def perform_cmip6_query(conf, query_string):
df_sub = conf.df.query(query_string)
if (df_sub.zstore.values.size == 0):
return df_sub
mapper = conf.fs.get_mapper(df_sub.zstore.values[-1])
ds = xr.open_zarr(mapper, consolidated=True)
print("Time encoding: {} - {}".format(ds.indexes['time'],
ds.indexes['time'].dtype))
if not ds.indexes["time"].dtype in ["datetime64[ns]", "object"]:
time_object = ds.indexes['time'].to_datetimeindex(
) #pd.DatetimeIndex([ds["time"].values[0]])
print(time_object, time_object.year)
# Convert if necesssary
if time_object[0].year == 1:
times = ds.indexes['time'].to_datetimeindex(
) # pd.DatetimeIndex([ds["time"].values])
times_plus_2000 = []
for t in times:
times_plus_2000.append(
cftime.DatetimeNoLeap(t.year + 2000, t.month, t.day,
t.hour))
ds["time"].values = times_plus_2000
ds = xr.decode_cf(ds)
return ds
def plot_dict_with_date_keys(dict_in, title, legend=None):
"""
Assume that keys of dict_in are 'YYYYMMDD' and values are numeric
"""
time_units = "days since 0001-01-01 0:00:00"
time = []
array_val = []
for date in dict_in.keys():
if "log" not in date:
(year, month, day) = date.split("-")
time.append(cftime.DatetimeNoLeap(int(year), int(month), int(day)))
array_val.append(dict_in[date])
if type(array_val[0]) == list:
dim2 = len(array_val[0])
da = xr.DataArray(array_val, dims=["time", "dim2"])
else:
dim2 = None
da = xr.DataArray(array_val, dims="time")
da["time"] = time
fig = plt.figure(figsize=(9.0, 5.25), clear=True)
if dim2:
for dim2ind in range(dim2):
da.isel(dim2=dim2ind).plot()
else:
da.plot()
if legend:
plt.legend(legend)
plt.title(title)
plt.show()
def fix_broken_time_index(ds: xr.Dataset):
"""Fix for a single broken index in a specific file"""
if "time" not in ds.dims:
return
time_dim = ds.time.values
times_are_encoded = "units" in ds.time.attrs
if times_are_encoded:
wrong_id = np.argwhere(np.isclose(time_dim, 0))
else:
wrong_id = np.argwhere(time_dim == cftime.DatetimeNoLeap(
year=1850, month=1, day=1, hour=0))
if wrong_id.size == 0:
return
wrong_id = wrong_id[0, 0]
if wrong_id == 0 or wrong_id == len(ds.time) - 1:
return
daily_gap = 1.0 if times_are_encoded else timedelta(days=1)
is_daily = time_dim[wrong_id + 1] - time_dim[wrong_id - 1] == daily_gap * 2
if is_daily:
fixed_time = time_dim
fixed_time[wrong_id] = time_dim[wrong_id - 1] + daily_gap
attrs = ds.time.attrs
ds["time"] = fixed_time
ds.time.attrs = attrs
def perform_cmip6_query(self, config, query_string: str) -> xr.Dataset:
df_sub = config.df.query(query_string)
if df_sub.zstore.values.size == 0:
return df_sub
mapper = config.fs.get_mapper(df_sub.zstore.values[-1])
logging.debug("[CMIP6_IO] df_sub: {}".format(df_sub))
ds = xr.open_zarr(mapper, consolidated=True, mask_and_scale=True)
# print("Time encoding: {} - {}".format(ds.indexes['time'], ds.indexes['time'].dtype))
if not ds.indexes["time"].dtype in ["datetime64[ns]", "object"]:
time_object = ds.indexes['time'].to_datetimeindex(
) # pd.DatetimeIndex([ds["time"].values[0]])
# Convert if necessary
if time_object[0].year == 1:
times = ds.indexes['time'].to_datetimeindex(
) # pd.DatetimeIndex([ds["time"].values])
times_plus_2000 = []
for t in times:
times_plus_2000.append(
cftime.DatetimeNoLeap(t.year + 2000, t.month, t.day,
t.hour))
ds["time"].values = times_plus_2000
ds = xr.decode_cf(ds)
return ds
def add_months(date_in, nmonth):
""" adds the given number of months to a given date """
year = date_in.year + (date_in.month + nmonth - 1) // 12
mth = (date_in.month + nmonth - 1) % 12 + 1
day = min(date_in.day,
calendar.monthrange(1, mth)[1]) # -> this assumes LEAP YEAR
return cft.DatetimeNoLeap(year, mth, day)
def make_time(year_range):
from itertools import product
return [
cftime.DatetimeNoLeap(year, month, 1)
for year, month in product(range(year_range[0], year_range[1] +
1), range(1, 13))
]
def load_pdo(year_range=None, apply_ann_filter=False):
'''read pdo from JSON file:
https://www.ncdc.noaa.gov/teleconnections/pdo/data.json
'''
import json
with open('data/pdo-data.json', 'r') as f:
pdo_data = json.load(f)
year = xr.DataArray([float(d[0:4]) for d in pdo_data['data'].keys()],
dims='time')
mon = xr.DataArray([float(d[4:6]) for d in pdo_data['data'].keys()],
dims='time')
time = xr.DataArray([
cftime.DatetimeNoLeap(y, m, 1) for y, m in zip(year.values, mon.values)
],
dims='time')
data = xr.DataArray([float(d) for d in pdo_data['data'].values()],
dims='time',
coords={'time': time})
ds = xr.Dataset({'PDO': data, 'year': year, 'mon': mon})
if year_range is not None:
nx = np.where((year_range[0] <= year) & (year <= year_range[1]))[0]
ds = ds.isel(time=nx)
if apply_ann_filter:
save_attrs = {v: ds[v].attrs for v in ds.variables}
N = 12
ds = ds.rolling(time=N, center=True).mean()
return ds
def time_mid_mon(years):
mid_mon = eom_day_noleap / 2 + 1.
mid_mon_day = np.floor(mid_mon)
mid_mon_hr = (mid_mon - mid_mon_day) * 24
return [
cftime.DatetimeNoLeap(year, mon, mid_mon_day[mon - 1],
mid_mon_hr[mon - 1])
for year, mon in product(years, range(1, 13))
]
def load_base_data(info):
if (info.verbose): print("Loading: "+info.atmdir + "/" + info.atmfile)
ds = xr.open_mfdataset(info.atmdir + "/" + info.atmfile.format(info.base_vars[0]))
for v in info.base_vars[1:]:
this_data = xr.open_mfdataset(info.atmdir + "/" + info.atmfile.format(v))
data = interpolate_data(this_data[v], ds)
ds[v] = data
# for example, this could do:
# ds = xr.open_mfdataset(["hus_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "ta_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "ua_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "va_6hrLev_CanESM2_historical_r1i1p1_198001010000-198012311800.nc",
# "orog_fx_CanESM2_historical_r0i0p0.nc",
# "sftlf_fx_CanESM2_historical_r0i0p0.nc"])
# perhaps as :
# ds = xr.open_mfdataset("*CanESM2*.nc")
if (info.verbose): print("Loading base file metadata (this could take a while)")
# Time objects have to be defined with the calendar used in the input GCM
start_time = ds.time.values[0]
if start_time.calendar == "360_day":
start_date = cftime.Datetime360Day(*info.start_date.timetuple())
end_date = cftime.Datetime360Day(*info.end_date.timetuple())
elif start_time.calendar == "noleap":
start_date = cftime.DatetimeNoLeap(*info.start_date.timetuple())
end_date = cftime.DatetimeNoLeap(*info.end_date.timetuple())
else:
start_date = cftime.Datetime(*info.start_date.timetuple())
end_date = cftime.Datetime(*info.end_date.timetuple())
# subset the dataset in space and time
return ds.sel(time= slice(start_date, end_date),
lat = slice(info.lat[0], info.lat[1]),
lon = slice(info.lon[0], info.lon[1]))
def test_cftime_transform_noleap_warn(self):
try:
import cftime
except:
raise SkipTest('Test requires cftime library')
gregorian_dates = [cftime.DatetimeNoLeap(2000, 2, 28),
cftime.DatetimeNoLeap(2000, 3, 1),
cftime.DatetimeNoLeap(2000, 3, 2)]
curve = Curve((gregorian_dates, [1, 2, 3]))
plot = bokeh_renderer.get_plot(curve)
xs = plot.handles['cds'].data['x']
self.assertEqual(xs.astype('int64'),
np.array([951696000000, 951868800000, 951955200000]))
substr = (
"Converting cftime.datetime from a non-standard calendar "
"(noleap) to a standard calendar for plotting. This may "
"lead to subtle errors in formatting dates, for accurate "
"tick formatting switch to the matplotlib backend.")
self.log_handler.assertEndsWith('WARNING', substr)
def datetime(
year, month, day, hour=0, minute=0, second=0, microsecond=0,
tzinfo=None, calendar='proleptic_gregorian'):
"""
Retrieves a datetime-like object with the requested calendar. Calendar types
other than proleptic_gregorian require the netcdftime module to be
installed.
Parameters
----------
year : int,
month : int,
day : int,
hour : int, optional
minute : int, optional
second : int, optional
microsecond : int, optional
tzinfo : datetime.tzinfo, optional
A timezone informaton class, such as from pytz. Can only be used with
'proleptic_gregorian' calendar, as netcdftime does not support
timezones.
calendar : string, optional
Should be one of 'proleptic_gregorian', 'no_leap', '365_day',
'all_leap', '366_day', '360_day', 'julian', or 'gregorian'. Default
is 'proleptic_gregorian', which returns a normal Python datetime.
Other options require the netcdftime module to be installed.
Returns
-------
datetime : datetime-like
The requested datetime. May be a Python datetime, or one of the
datetime-like types in netcdftime.
"""
kwargs = {
'year': year, 'month': month, 'day': day, 'hour': hour,
'minute': minute, 'second': second, 'microsecond': microsecond
}
if calendar.lower() == 'proleptic_gregorian':
return real_datetime(tzinfo=tzinfo, **kwargs)
elif tzinfo is not None:
raise ValueError('netcdftime does not support timezone-aware datetimes')
elif ct is None:
raise DependencyError(
"Calendars other than 'proleptic_gregorian' require the netcdftime "
"package, which is not installed.")
elif calendar.lower() in ('all_leap', '366_day'):
return ct.DatetimeAllLeap(**kwargs)
elif calendar.lower() in ('no_leap', 'noleap', '365_day'):
return ct.DatetimeNoLeap(**kwargs)
elif calendar.lower() == '360_day':
return ct.Datetime360Day(**kwargs)
elif calendar.lower() == 'julian':
return ct.DatetimeJulian(**kwargs)
elif calendar.lower() == 'gregorian':
return ct.DatetimeGregorian(**kwargs)
def make_year_constraint_all_calendars(start, end):
"""Utility function to create a dict of time constraints on year-basis
This create a dict of the same time constraint, but for different calendars.
Since comparisons between different calendar types are not (always) possible,
a calendar for a time coordinate should be compared to the specific constraint
with the same calendar.
The calendar type (as a string) can be obtained through the coordinate's `units`
attribute: `cube.coord('time').units.calendar`; the resulting string is the key
for the dict, which then as a value yields the correct constraint
Arguments
---------
start, end: integer
Start and end year. Month and day are 1, 1 for the starting year, and
31, 12 or 30, 12 (for a 360-day calendar) for the end year
"""
dates = {
'default': (cftime.datetime(start, 1, 1), cftime.datetime(end, 12,
31)),
'360_day':
(cftime.Datetime360Day(start, 1,
1), cftime.Datetime360Day(end, 12, 30)),
'365_day': (cftime.DatetimeNoLeap(start, 1, 1),
cftime.DatetimeNoLeap(end, 12, 31)),
'proleptic_gregorian': (cftime.DatetimeProlepticGregorian(start, 1, 1),
cftime.DatetimeProlepticGregorian(end, 12,
31)),
'gregorian': (cftime.DatetimeGregorian(start, 1, 1),
cftime.DatetimeGregorian(end, 12, 31)),
'julian': (cftime.DatetimeJulian(start, 1, 1),
cftime.DatetimeJulian(end, 12, 31)),
}
constraints = {
key: make_date_constraint(*value)
for key, value in dates.items()
}
return constraints
def get_time_constraint(time_list):
"""Get the time constraint used for reading an iris cube."""
if time_list:
if not type(time_list) in (list, tuple):
time_list = [time_list]
date_pattern = '([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})'
start_date = time_list[0]
start_year, start_month, start_day = start_date.split('-')
assert re.search(date_pattern, start_date)
if len(time_list) == 1:
time_constraint = iris.Constraint(
time=lambda t: cftime.DatetimeNoLeap(int(
start_year), int(start_month), int(start_day)) <= t.point)
else:
end_date = time_list[1]
assert re.search(date_pattern, end_date)
if (start_date == end_date):
time_constraint = iris.Constraint(
time=iris.time.PartialDateTime(year=int(start_year),
month=int(start_month),
day=int(start_day)))
else:
end_year, end_month, end_day = end_date.split('-')
start_constraint = iris.Constraint(
time=lambda t: cftime.DatetimeNoLeap(
int(start_year), int(start_month), int(start_day)
) <= t.point)
end_constraint = iris.Constraint(
time=lambda t: t.point <= cftime.DatetimeNoLeap(
int(end_year), int(end_month), int(end_day)))
time_constraint = start_constraint & end_constraint
#time_constraint = iris.Constraint(time=lambda t: cftime.DatetimeNoLeap(int(start_year), int(start_month), int(start_day)) <= t.point <= cftime.DatetimeNoLeap(int(end_year), int(end_month), int(end_day)))
#time_constraint = iris.Constraint(time=lambda t: iris.time.PartialDateTime(year=int(start_year), month=int(start_month), day=int(start_day)) <= t.point <= iris.time.PartialDateTime(year=int(end_year), month=int(end_month), day=int(end_day)))
#time_constraint = iris.Constraint(time=lambda t: iris.time.PartialDateTime(year=int(start_year), month=int(start_month), day=int(start_day)) <= t.point and iris.time.PartialDateTime(year=int(end_year), month=int(end_month), day=int(end_day)) >= t.point)
else:
time_constraint = iris.Constraint()
return time_constraint
def convert_vmro3_to_toz(self, scenario: str, ds: xr.Dataset,
baseurl: str):
R = 287.3 # Jkg-1K-1 (Specific gas constant for air)
T0 = 273.15 # Kelvin(Standard temperaure)
P0 = 1.01325e5 # Pa (Standard pressure at surface)
g0 = 9.80665 # ms-2 (Global average gravity at surface)
Na = 6.0220e23 # Avogadro´s number
# Integrating the total column of a trace gas from input4MPI forcing data. Here
# P is the pressure in hPa, VMR is the colume mixing ration in ppm and TOZ is the trace gas
# column amount in Dobson Units (DU):'bnds', 'lat', 'lon', 'plev', 'time'
mole2ppmv = 1e6
VMR = ds["vmro3"].values * mole2ppmv
plev = ds["plev"].values
times = ds["time"].values
times_plus = []
# Mix of Timestamp and DateTimeNoLeap - convert all to DateTimeNoLeap
for t in times:
if isinstance(t, datetime.datetime):
times_plus.append(
cftime.DatetimeNoLeap(t.year, t.month, t.day, t.hour))
else:
times_plus.append(t)
VMR = np.where(VMR > 1000, np.nan, VMR)
plev = np.where(plev > 1000, np.nan, plev)
TOZ = 10 * ((R * T0) / (g0 * P0)) * np.nansum(0.5 * (
(VMR[:, 0:-2:1, :, :] + VMR[:, 1:-1:1, :, :]) *
(plev[None, 0:-2:1, None, None] - plev[None, 1:-1:1, None, None])),
axis=1)
# Create a dataset
toz_ds = xr.DataArray(
name="TOZ",
data=TOZ,
coords={
'time': (['time'], times_plus),
'lat': (['lat'], ds["lat"].values),
'lon': (['lon'], ds["lon"].values)
},
dims=["time", "lat", "lon"],
).to_dataset()
toz_ds.to_netcdf(baseurl + "/TOZ_{}.nc".format(scenario))
logging.info(
"[CMIP6_ozone] Results written to file covering period {} to {}".
format(times_plus[0], times_plus[-1]))
logging.info("[CMIP6_ozone] TOZ min {} to max {} and mean {}".format(
np.nanmin(TOZ), np.nanmax(TOZ), np.nanmean(TOZ)))
def compute_multimodel_mean(V):
"""
for now require spatial grids are fixed
assume we have spatio-temporal data
"""
v0 = V[list(V.keys())[0]]
ds = [V[v].ds for v in V]
tm = None
if v0.temporal():
tb = []
for v in V:
V[v].ds['time'] = [
cftime.DatetimeNoLeap(t.dt.year, t.dt.month, t.dt.day)
for t in V[v].ds['time']
]
tbnd = V[v].timeBounds()
tb.append(tbnd)
t0 = max([t[0] for t in tb])
tf = min([t[1] for t in tb])
t0 = cftime.DatetimeNoLeap(t0.dt.year, t0.dt.month, 1)
tf = cftime.DatetimeNoLeap(tf.dt.year, tf.dt.month, 28)
ds = [d.sel(time=slice(t0, tf)) for d in ds]
tm = ds[0]['time_measure']
if v0.spatial() and not same_space_grid(V):
res = 1.0
lat = np.linspace(-90, 90, int(round(180 / res)) + 1)
lon = np.linspace(0, 360, int(round(360 / res)) + 1)
lat = 0.5 * (lat[:-1] + lat[1:])
lon = 0.5 * (lon[:-1] + lon[1:])
ds = [
d.interp(coords={
'lat': lat,
'lon': lon
}, method='nearest') for d in ds
]
ds = [d[v0.varname] for d in ds]
ds = xr.align(*ds, join='override')
ds = xr.concat(ds, dim='model').mean(dim='model')
mn = Variable(da=ds, varname=v0.varname, time_measure=tm)
mn.ds[mn.varname].attrs = v0.ds[v0.varname].attrs
return mn
def main():
""" main function"""
global glob
# get the list of files
filePaths = get_file_paths(args.f, comm.Get_rank(), args.x, args.v)
# determine beginning and ending dates for files to be generated
user_date0_out = user_date1_out = None
if args.d0:
user_date0_out = cft.DatetimeNoLeap(year=int(args.d0[0:4]),
month=int(args.d0[5:7]),
day=1)
if args.d1:
user_date1_out = next_month_1st(
cft.DatetimeNoLeap(year=int(args.d1[0:4]),
month=int(args.d1[5:7]),
day=1))
# now get the global information
glob.obtain_global_info(filePaths, comm, user_date0_out, user_date1_out)
# broadcast/receive the global information:
glob = comm.bcast(glob, root=0)
avg_intervals = preprocess_out_files()
# print global information
if comm.Get_rank() == 0:
print("Input begin date: ", glob.date0_in)
print("Input end date: ", glob.date1_in)
print("Output begin date: ", glob.date0_out)
print("Output end date: ", glob.date1_out)
print("Number of monthly avgs to generate", glob.nmonths)
print("Max number of months per core:", glob.m_per_proc)
# generate the avg files:
process_out_files(avg_intervals)
if comm.Get_rank() == 0:
print("done.")
def _get_centered_timecoord(cube):
"""Fix time coordinate.
Time points start at the beginning of month at 00:00:00.
"""
time = cube.coord('time')
times = time.units.num2date(time.points)
# get bounds
starts = [
cftime.DatetimeNoLeap(c.year, c.month, 1)
for c in times
]
ends = [
cftime.DatetimeNoLeap(c.year, c.month + 1, 1) if c.month < 12
else cftime.DatetimeNoLeap(c.year + 1, 1, 1)
for c in times
]
time.bounds = time.units.date2num(np.stack([starts, ends], -1))
# get points
time.points = [np.mean((t1, t2)) for t1, t2 in time.bounds]
def state_list_with_inconsistent_calendars(base_state, numpy):
state_list = []
state_times = [
cftime.DatetimeNoLeap(2000, 1, 1),
cftime.Datetime360Day(2000, 1, 2)
]
for i in range(2):
new_state = copy.deepcopy(base_state)
for name in set(new_state.keys()).difference(["time"]):
new_state[name].view[:] = numpy.random.randn(
*new_state[name].extent)
state_list.append(new_state)
new_state["time"] = state_times[i]
return state_list
def test_assert_all_valid_date_type(date_type, index):
import cftime
if date_type is cftime.DatetimeNoLeap:
mixed_date_types = [
date_type(1, 1, 1),
cftime.DatetimeAllLeap(1, 2, 1)
]
else:
mixed_date_types = [date_type(1, 1, 1), cftime.DatetimeNoLeap(1, 2, 1)]
with pytest.raises(TypeError):
assert_all_valid_date_type(mixed_date_types)
with pytest.raises(TypeError):
assert_all_valid_date_type([1, date_type(1, 1, 1)])
assert_all_valid_date_type([date_type(1, 1, 1), date_type(1, 2, 1)])
def perform_cmip6_query(conf, query_string):
df_sub = conf.df.query(query_string)
if (df_sub.zstore.values.size == 0):
return df_sub
mapper = conf.fs.get_mapper(df_sub.zstore.values[-1])
ds = xr.open_zarr(mapper, consolidated=True)
time_object = ds["time"].values[0]
# Convert if necesssary
if time_object.year == 1:
times = ds["time"].values
times_plus_2000 = []
for t in times:
times_plus_2000.append(
cftime.DatetimeNoLeap(t.year + 2000, t.month, t.day, t.hour))
ds["time"].values = times_plus_2000
ds = xr.decode_cf(ds)
return ds
