def test_decode_standard_calendar_multidim_time_inside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D')
times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D')
noleap_time1 = cftime.date2num(times1.to_pydatetime(),
units, calendar=calendar)
noleap_time2 = cftime.date2num(times2.to_pydatetime(),
units, calendar=calendar)
mdim_time = np.empty((len(noleap_time1), 2), )
mdim_time[:, 0] = noleap_time1
mdim_time[:, 1] = noleap_time2
expected1 = times1.values
expected2 = times2.values
actual = coding.times.decode_cf_datetime(
mdim_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == np.dtype('M8[ns]')
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def test_decode_standard_calendar_multidim_time_inside_timestamp_range(
calendar):
import cftime
units = "days since 0001-01-01"
times1 = pd.date_range("2001-04-01", end="2001-04-05", freq="D")
times2 = pd.date_range("2001-05-01", end="2001-05-05", freq="D")
time1 = cftime.date2num(times1.to_pydatetime(), units, calendar=calendar)
time2 = cftime.date2num(times2.to_pydatetime(), units, calendar=calendar)
mdim_time = np.empty((len(time1), 2))
mdim_time[:, 0] = time1
mdim_time[:, 1] = time2
expected1 = times1.values
expected2 = times2.values
actual = coding.times.decode_cf_datetime(mdim_time,
units,
calendar=calendar)
assert actual.dtype == np.dtype("M8[ns]")
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, "s")).all()
assert (abs_diff2 <= np.timedelta64(1, "s")).all()
def test_decode_multidim_time_outside_timestamp_range(calendar):
from datetime import datetime
import cftime
units = "days since 0001-01-01"
times1 = [datetime(1, 4, day) for day in range(1, 6)]
times2 = [datetime(1, 5, day) for day in range(1, 6)]
time1 = cftime.date2num(times1, units, calendar=calendar)
time2 = cftime.date2num(times2, units, calendar=calendar)
mdim_time = np.empty((len(time1), 2))
mdim_time[:, 0] = time1
mdim_time[:, 1] = time2
expected1 = cftime.num2date(time1, units, calendar, only_use_cftime_datetimes=True)
expected2 = cftime.num2date(time2, units, calendar, only_use_cftime_datetimes=True)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "Unable to decode time axis")
actual = coding.times.decode_cf_datetime(mdim_time, units, calendar=calendar)
assert actual.dtype == np.dtype("O")
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, "s")).all()
assert (abs_diff2 <= np.timedelta64(1, "s")).all()
def test_decode_standard_calendar_multidim_time_inside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D')
times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D')
noleap_time1 = cftime.date2num(times1.to_pydatetime(),
units, calendar=calendar)
noleap_time2 = cftime.date2num(times2.to_pydatetime(),
units, calendar=calendar)
mdim_time = np.empty((len(noleap_time1), 2), )
mdim_time[:, 0] = noleap_time1
mdim_time[:, 1] = noleap_time2
expected1 = times1.values
expected2 = times2.values
actual = coding.times.decode_cf_datetime(
mdim_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == np.dtype('M8[ns]')
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def time_bnds_mon(years):
return xr.DataArray(np.array([[
cftime.date2num(cftime.DatetimeNoLeap(year, mon, 1),
units='days since 0001-01-01',
calendar='noleap') - 1,
cftime.date2num(cftime.DatetimeNoLeap(year, mon,
eom_day_noleap[mon - 1]),
units='days since 0001-01-01',
calendar='noleap')
] for year, mon in product(years, range(1, 13))]),
dims=('time', 'ntb'))
def restore_dataset(self, ds, attrs={}, encoding={}):
"""Return the original time variable to decoded or undecoded state.
"""
ds = xr.merge(
(
ds,
self._ds_time_computed.drop(
[v for v in self._ds.variables if v not in self.static_variables]
),
)
)
# get the time data from dataset
time_data = ds[self.time_coord_name].data
# if time was not originally decoded, return the dataset with time
# un-decoded
if not self.time_orig_decoded and self.isdecoded(time_data):
time_data = cftime.date2num(
time_data, units=self.time_attrs['units'], calendar=self.time_attrs['calendar']
)
elif self.time_orig_decoded and not self.isdecoded(time_data):
time_data = cftime.num2date(
time_data, units=self.time_attrs['units'], calendar=self.time_attrs['calendar']
)
ds[self.time_coord_name].attrs = self.time_attrs
ds[self.time_coord_name].data = time_data.astype(self.time.dtype)
if self.time_bound is not None:
time_bound_data = ds[self.tb_name].data
if not self.time_orig_decoded and self.isdecoded(time_bound_data):
time_bound_data = cftime.date2num(
time_bound_data,
units=self.time_attrs['units'],
calendar=self.time_attrs['calendar'],
)
elif self.time_orig_decoded and not self.isdecoded(time_bound_data):
time_bound_data = cftime.num2date(
time_bound_data,
units=self.time_attrs['units'],
calendar=self.time_attrs['calendar'],
)
ds[self.tb_name].data = time_bound_data
ds[self.tb_name].attrs = self.time_bound_attrs
return self.update_metadata(ds, new_attrs=attrs, new_encoding=encoding)
def test_decode_multidim_time_outside_timestamp_range(calendar,
enable_cftimeindex):
from datetime import datetime
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = [datetime(1, 4, day) for day in range(1, 6)]
times2 = [datetime(1, 5, day) for day in range(1, 6)]
noleap_time1 = cftime.date2num(times1, units, calendar=calendar)
noleap_time2 = cftime.date2num(times2, units, calendar=calendar)
mdim_time = np.empty((len(noleap_time1), 2), )
mdim_time[:, 0] = noleap_time1
mdim_time[:, 1] = noleap_time2
if enable_cftimeindex:
expected1 = cftime.num2date(noleap_time1,
units,
calendar,
only_use_cftime_datetimes=True)
expected2 = cftime.num2date(noleap_time2,
units,
calendar,
only_use_cftime_datetimes=True)
else:
expected1 = cftime.num2date(noleap_time1, units, calendar)
expected2 = cftime.num2date(noleap_time2, units, calendar)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
mdim_time,
units,
calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == np.dtype('O')
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def time_set_mid(ds, time_name, deep=False):
"""
Return copy of ds with values of ds[time_name] replaced with midpoints of
ds[time_name].attrs['bounds'], if bounds attribute exists.
Except for time_name, the returned Dataset is a copy of ds2.
The copy is deep or not depending on the argument deep.
"""
ds_out = ds.copy(deep)
if "bounds" not in ds[time_name].attrs:
return ds_out
tb_name = ds[time_name].attrs["bounds"]
tb = ds[tb_name]
bounds_dim = next(dim for dim in tb.dims if dim != time_name)
# Use da = da.copy(data=...), in order to preserve attributes and encoding.
# If tb is an array of datetime objects then encode time before averaging.
# Do this because computing the mean on datetime objects with xarray fails
# if the time span is 293 or more years.
# https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/issues/7
if tb.dtype == np.dtype("O"):
units = "days since 0001-01-01"
calendar = "noleap"
tb_vals = cftime.date2num(ds[tb_name].values, units=units, calendar=calendar)
tb_mid_decode = cftime.num2date(
tb_vals.mean(axis=1), units=units, calendar=calendar
)
ds_out[time_name] = ds[time_name].copy(data=tb_mid_decode)
else:
ds_out[time_name] = ds[time_name].copy(data=tb.mean(bounds_dim))
return ds_out
def make_netcdf(config, in_file):
"""
Main program function
"""
out_file = f"{os.path.splitext(in_file)[0]}.nc"
quince = pd.read_csv(in_file, parse_dates=["Date/Time"], low_memory=False)
with Dataset(out_file, "w", format="NETCDF4") as nc:
# Set global attributes from config
for attr in config["global_attributes"]:
nc.setncattr(attr, config["global_attributes"][attr])
nc.setncattr("creation_date",
datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'))
nc.createDimension("time")
timevar = nc.createVariable("time", "f8", ("time", ))
timevar.units = "days since 1970-01-01 00:00:00.0"
timevar[:] = date2num(quince["Date/Time"], timevar.units)
for column in config["columns"]:
column_info = config["columns"][column]
var = nc.createVariable(column_info["variable"],
column_info["type"], ("time", ))
var.units = column_info["units"]
var[:] = quince[column]
def test_decode_non_standard_calendar_single_element(calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
try:
dt = cftime.netcdftime.datetime(2001, 2, 29)
except AttributeError:
# Must be using the standalone cftime library
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
actual = coding.times.decode_cf_datetime(num_time,
units,
calendar=calendar)
if cftime.__name__ == 'cftime':
expected = np.asarray(
cftime.num2date(num_time,
units,
calendar,
only_use_cftime_datetimes=True))
else:
expected = np.asarray(cftime.num2date(num_time, units, calendar))
assert actual.dtype == np.dtype('O')
assert expected == actual
def cfencode_time(dsvar,
calendar="proleptic_gregorian",
units="seconds since 1970-01-01 00:00:00"):
dsvar[:] = date2num(dsvar[:], units, calendar)
dsvar.attrs["units"] = units
dsvar.attrs["calendar"] = calendar
dsvar.attrs["axis"] = "T"
def test_decode_non_standard_calendar_inside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H')
noleap_time = cftime.date2num(times.to_pydatetime(),
units,
calendar=calendar)
if enable_cftimeindex:
expected = cftime.num2date(noleap_time, units, calendar=calendar)
expected_dtype = np.dtype('O')
else:
expected = times.values
expected_dtype = np.dtype('M8[ns]')
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
noleap_time,
units,
calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == expected_dtype
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def test_decode_dates_outside_timestamp_range(
calendar, enable_cftimeindex):
from datetime import datetime
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = [datetime(1, 4, 1, h) for h in range(1, 5)]
noleap_time = cftime.date2num(times, units, calendar=calendar)
if enable_cftimeindex:
expected = cftime.num2date(noleap_time, units, calendar=calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(noleap_time, units, calendar=calendar)
expected_date_type = type(expected[0])
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
noleap_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert all(isinstance(value, expected_date_type) for value in actual)
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def get_masked_datetime_array(t, tvar, mask_nan=True):
# If we are passed in a scalar... return a scalar
if isinstance(t, np.ma.core.MaskedConstant):
return t
elif np.isscalar(t):
return num2date(t, tvar.units, getattr(tvar, 'calendar', 'standard'))
if mask_nan is True:
t = np.ma.masked_invalid(t)
t_cal = getattr(tvar, 'calendar', 'standard')
# Get the min value we can have and mask anything else
# This is limited by **python** datetime objects and not
# nc4 objects. The min nc4 datetime object is
# min_date = nc4.netcdftime.datetime(-4713, 1, 1, 12, 0, 0, 40)
# There is no max date for nc4.
min_nums = date2num([datetime.min, datetime.max], tvar.units, t_cal)
t = np.ma.masked_outside(t, *min_nums)
dts = num2pydate(t, tvar.units, t_cal)
if isinstance(dts, (datetime, cfdt)):
dts = np.array([dts.isoformat()], dtype='datetime64')
return dts
def get_time_bounds(self, time_coord):
"""
Get contiguous time bounds for sea ice maps
Creates new time bounds [
[01-01-current year, 01-01-next year],
]
"""
dt_object = cftime.num2pydate(time_coord.points[0],
time_coord.units.name)
start = datetime.datetime(dt_object.year, 1, 1)
end = datetime.datetime(start.year + 1, 1, 1)
start_seconds = cftime.date2num(start, time_coord.units.name)
end_seconds = cftime.date2num(end, time_coord.units.name)
new_bounds = np.array([[start_seconds, end_seconds]])
return new_bounds
def _verif_dates_xr(hindcast, alignment, reference, date2num_units):
"""Create ``valid_time`` ``xr.DataArray`` with dims lead and init in units passed to
cftime.date2num."""
inits, verif_dates = return_inits_and_verif_dates(
hindcast.get_initialized().rename({"init": "time"}),
hindcast.get_observations(),
alignment,
reference=reference,
hist=hindcast.get_uninitialized() if isinstance(
hindcast.get_uninitialized(), xr.Dataset) else None,
)
verif_dates_xr = xr.concat(
[
xr.DataArray(
cftime.date2num(verif_dates[k], date2num_units),
dims="init",
coords={"init": v.rename({
"time": "init"
}).to_index()},
name="valid_time",
attrs=dict(units=date2num_units),
) for k, v in inits.items()
],
dim="lead",
).assign_coords(lead=hindcast.get_initialized().lead)
return verif_dates_xr
def test_decode_non_standard_calendar_inside_timestamp_range(
calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = pd.date_range('2001-04-01-00', end='2001-04-30-23',
freq='H')
non_standard_time = cftime.date2num(
times.to_pydatetime(), units, calendar=calendar)
if cftime.__name__ == 'cftime':
expected = cftime.num2date(
non_standard_time, units, calendar=calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(non_standard_time, units,
calendar=calendar)
expected_dtype = np.dtype('O')
actual = coding.times.decode_cf_datetime(
non_standard_time, units, calendar=calendar)
assert actual.dtype == expected_dtype
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def st2rt(array, units_in, units_out, dummy1=None):
"""The returned array is always independent.
:Parameters:
array: numpy array-like of ISO 8601 date-time strings
units_in: `Units` or `None`
units_out: `Units`
dummy1:
Ignored.
:Returns:
`numpy.ndarray`
An array of floats with the same shape as *array*.
"""
array = st2dt(array, units_in)
# array = units_out._utime.date2num(array)
array = cftime.date2num(array,
units=units_out.units,
calendar=units_out._utime.calendar)
if not np.ndim(array):
array = np.asanyarray(array)
return array
def test_decode_dates_outside_timestamp_range(calendar):
from datetime import datetime
import cftime
units = "days since 0001-01-01"
times = [datetime(1, 4, 1, h) for h in range(1, 5)]
time = cftime.date2num(times, units, calendar=calendar)
expected = cftime.num2date(time,
units,
calendar=calendar,
only_use_cftime_datetimes=True)
expected_date_type = type(expected[0])
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "Unable to decode time axis")
actual = coding.times.decode_cf_datetime(time,
units,
calendar=calendar)
assert all(isinstance(value, expected_date_type) for value in actual)
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, "s")).all()
def set_time(self, time_coord_name=None, year_offset=None):
"""store the original time and time_bound data from the dataset;
ensure that time_bound, if present, is not decoded.
"""
self.infer_time_coord_name(time_coord_name)
self.year_offset = year_offset
self.time = self._ds[self.time_coord_name].copy()
self.time_orig_decoded = self.isdecoded(self.time)
self.infer_time_bound_var()
if self.tb_name is None:
self.time_bound = None
else:
self.time_bound = self._ds[self.tb_name].copy()
if self.isdecoded(self._ds[self.tb_name]):
try:
tb_data = cftime.date2num(
self._ds[self.tb_name],
units=self.time_attrs['units'],
calendar=self.time_attrs['calendar'],
)
self.time_bound.data = tb_data
except Exception as exc:
print('Please open dataset with `decode_times=False`')
raise exc
self.setup()
return self
def time_set_mid(ds, time_name):
"""
set ds[time_name] to midpoint of ds[time_name].attrs['bounds'], if bounds attribute exists
type of ds[time_name] is not changed
ds is returned
"""
if 'bounds' not in ds[time_name].attrs:
return ds
# determine units and calendar of unencoded time values
if ds[time_name].dtype == np.dtype('O'):
units = 'days since 0000-01-01'
calendar = 'noleap'
else:
units = ds[time_name].attrs['units']
calendar = ds[time_name].attrs['calendar']
# construct unencoded midpoint values, assumes bounds dim is 2nd
tb_name = ds[time_name].attrs['bounds']
if ds[tb_name].dtype == np.dtype('O'):
tb_vals = cftime.date2num(ds[tb_name].values,
units=units,
calendar=calendar)
else:
tb_vals = ds[tb_name].values
tb_mid = tb_vals.mean(axis=1)
# set ds[time_name] to tb_mid
if ds[time_name].dtype == np.dtype('O'):
ds[time_name] = cftime.num2date(tb_mid, units=units, calendar=calendar)
else:
ds[time_name] = tb_mid
return ds
def test_decode_non_standard_calendar_inside_timestamp_range(calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H')
non_standard_time = cftime.date2num(times.to_pydatetime(),
units,
calendar=calendar)
if cftime.__name__ == 'cftime':
expected = cftime.num2date(non_standard_time,
units,
calendar=calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(non_standard_time, units, calendar=calendar)
expected_dtype = np.dtype('O')
actual = coding.times.decode_cf_datetime(non_standard_time,
units,
calendar=calendar)
assert actual.dtype == expected_dtype
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def dataList2NetCDF(
self, ncFileName, shortVarNameList, varFieldList, timeStamp, posCnt=None
):
rootgrp = nc.Dataset(ncFileName, "a")
date_time = rootgrp.variables["time"]
if posCnt == None:
posCnt = len(date_time)
for shortVarName in shortVarNameList:
date_time[posCnt] = cftime.date2num(
timeStamp, date_time.units, date_time.calendar
)
varField = varFieldList[shortVarName]
# flip variable if necessary (to follow cf_convention)
if self.netcdf_y_orientation_follow_cf_convention:
varField = np.flipud(varField)
rootgrp.variables[shortVarName][posCnt, :, :] = varField
rootgrp.sync()
rootgrp.close()
def test_decode_non_standard_calendar_inside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = pd.date_range('2001-04-01-00', end='2001-04-30-23',
freq='H')
noleap_time = cftime.date2num(times.to_pydatetime(), units,
calendar=calendar)
if enable_cftimeindex:
expected = cftime.num2date(noleap_time, units, calendar=calendar)
expected_dtype = np.dtype('O')
else:
expected = times.values
expected_dtype = np.dtype('M8[ns]')
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
noleap_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == expected_dtype
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def dataList2NetCDF(self,
ncFileName,
shortVarNameList,
varFieldList,
timeStamp,
posCnt=None):
rootgrp = nc.Dataset(ncFileName, "a")
date_time = rootgrp.variables["time"]
if posCnt == None:
posCnt = len(date_time)
for shortVarName in shortVarNameList:
date_time[posCnt] = cftime.date2num(timeStamp, date_time.units,
date_time.calendar)
varField = varFieldList[shortVarName]
# flip variable if necessary (to follow cf_convention)
if self.netcdf_y_orientation_follow_cf_convention:
varField = np.flipud(varField)
rootgrp.variables[shortVarName][posCnt, :, :] = varField
rootgrp.sync()
rootgrp.close()
def open_ann_fosi(anomaly=True):
#-- open the dataset
xr_open_ds = { #'chunks' : {'time':1}, # chunking breaks "rolling" method
'decode_coords': False,
'decode_times': False
}
case = 'g.e11_LENS.GECOIAF.T62_g16.009'
file_in = f'/glade/work/yeager/{case}/budget_O2_npac_{case}.0249-0316.nc'
ds = xr.open_dataset(file_in, **xr_open_ds)
#-- convert units
ds = conform_budget_dataset(ds)
grid = ds.drop([v for v in ds.variables if 'time' in ds[v].dims])
#-- interpret time: make time into "year"
offset = cftime.date2num(cftime.DatetimeGregorian(1699, 1, 1),
ds.time.attrs['units'], ds.time.attrs['calendar'])
ds['date'] = cftime.num2date(ds.time + offset, ds.time.attrs['units'],
ds.time.attrs['calendar'])
ds.time.values = [d.year * 1. for d in ds.date.values]
#-- make into an anomaly
if anomaly:
for v in ds.variables:
if 'time' in ds[v].dims and v != 'time':
attrs = ds[v].attrs
ds[v] = ds[v] - ds[v].sel(time=year_range_clim).mean('time')
ds[v].attrs = attrs
return ds
def test_decode_non_standard_calendar_single_element_fallback(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
try:
dt = cftime.netcdftime.datetime(2001, 2, 29)
except AttributeError:
# Must be using standalone netcdftime library
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
if enable_cftimeindex:
actual = coding.times.decode_cf_datetime(
num_time,
units,
calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
else:
with pytest.warns(SerializationWarning,
match='Unable to decode time axis'):
actual = coding.times.decode_cf_datetime(
num_time,
units,
calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
expected = np.asarray(cftime.num2date(num_time, units, calendar))
assert actual.dtype == np.dtype('O')
assert expected == actual
def test_decode_non_standard_calendar_single_element_fallback(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
try:
dt = cftime.netcdftime.datetime(2001, 2, 29)
except AttributeError:
# Must be using standalone netcdftime library
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
if enable_cftimeindex:
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
else:
with pytest.warns(SerializationWarning,
match='Unable to decode time axis'):
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
expected = np.asarray(cftime.num2date(num_time, units, calendar))
assert actual.dtype == np.dtype('O')
assert expected == actual
def datetime_to_float(x):
""" converts datetime values to float for regression """
time_ = x.time.values
if type(x.time.values[0])==np.datetime64:
x.time.values = np.array(x.time.values - np.datetime64('0001-01-01'), dtype=np.float)
elif type(x.time.values[0])==cftime._cftime.Datetime360Day:
x.time.values = cftime.date2num(x.time.values, units='months since 0001-01-01', calendar='360_day')
return x, time_
def datetime_from_grid(grid, epoch=False, **kwargs):
""" Return a datetime for the volume start in a Grid. """
if epoch:
dtrad = num2date(grid.time['data'][0], grid.time['units'])
epnum = date2num(dtrad, EPOCH_UNITS)
return num2date(epnum, EPOCH_UNITS, **kwargs)
else:
return num2date(grid.time['data'][0], grid.time['units'], **kwargs)
def datetimes_from_radar(radar, epoch=False, **kwargs):
""" Return an array of datetimes for the rays in a Radar. """
if epoch:
dtrad = num2date(radar.time['data'][:], radar.time['units'])
epnum = date2num(dtrad, EPOCH_UNITS)
return num2date(epnum, EPOCH_UNITS, **kwargs)
else:
return num2date(radar.time['data'][:], radar.time['units'], **kwargs)
def save_to_netcdf(self, r_text, file_path):
results = re.findall(
r'^(\d+-\d+-\d+ \d+:\d+:\d+.\d+)\s*(\d+)\s*([+\-\d.]+)\s*([+\-\d.]+)\s*([+\-\d.]+)\s*([+\-\d.]+)',
r_text, re.M)
results = list(zip(*results))
# time_array = np.array([(datetime.datetime.strptime(dtstr+'000', "%Y-%m-%d %H:%M:%S.%f")
# - datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1)
# for dtstr in results[0]])
dts = [
datetime.datetime.strptime(dtstr + '000', "%Y-%m-%d %H:%M:%S.%f")
for dtstr in results[0]
]
time_array = np.array(
cftime.date2num(dts, units='seconds since 1970-01-01 00:00:00.0'))
print('From {} to {}.'.format(
datetime.datetime.utcfromtimestamp(time_array[0]),
datetime.datetime.utcfromtimestamp(time_array[-1])))
asy_d_array = np.array(results[2])
asy_d_array.astype(np.float32)
asy_h_array = np.array(results[3])
asy_h_array.astype(np.float32)
sym_d_array = np.array(results[4])
sym_d_array.astype(np.float32)
sym_h_array = np.array(results[5])
sym_h_array.astype(np.float32)
num_rows = len(results[0])
fnc = netCDF4.Dataset(file_path, 'w')
fnc.createDimension('UNIX_TIME', num_rows)
fnc.title = "WDC ASY/SYM indices"
time = fnc.createVariable('UNIX_TIME', np.float64, ('UNIX_TIME', ))
time.units = 'seconds since 1970-01-01 00:00:00.0'
asy_d = fnc.createVariable('ASY_D', np.float32, ('UNIX_TIME', ))
asy_h = fnc.createVariable('ASY_H', np.float32, ('UNIX_TIME', ))
sym_d = fnc.createVariable('SYM_D', np.float32, ('UNIX_TIME', ))
sym_h = fnc.createVariable('SYM_H', np.float32, ('UNIX_TIME', ))
time[::] = time_array[::]
asy_d[::] = asy_d_array[::]
asy_h[::] = asy_h_array[::]
sym_d[::] = sym_d_array[::]
sym_h[::] = sym_h_array[::]
# for i, res in enumerate(results):
# dt = datetime.datetime.strptime(res[0]+'000', "%Y-%m-%d %H:%M:%S.%f")
# time[i] = (dt - datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1)
# asy_d[i] = float(res[2])
# asy_h[i] = float(res[3])
# sym_d[i] = float(res[4])
# sym_h[i] = float(res[5])
fnc.close()
mylog.StreamLogger.info(
"The requested data has been downloaded and saved in the file {}.".
format(file_path))
self.done = True
def _make_index(time):
year = int(time.dt.year[0])
doys = cftime.date2num(time, f"days since {year:04d}-01-01", calendar=calendar)
return DataArray(
year + doys / _days_in_year(year, calendar),
dims=(dim,),
coords=time.coords,
name=dim,
)
def save_to_netcdf(self, ystr, file_path):
with open(file_path, 'r') as f:
text = f.read()
results = re.findall(
r'^(\d{4} \d{2} \d{2})\s*([-+]?\d*\.\d+|\d+)' +
r'\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)'
+
r'\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)',
text, re.M)
results = list(zip(*results))
dts = [
datetime.datetime.strptime(dtstr, "%Y %m %d") +
datetime.timedelta(hours=float(hs))
for dtstr, hs in zip(results[0], results[2])
]
time_array = np.array(
cftime.date2num(dts,
units='seconds since 1970-01-01 00:00:00.0'))
hp_array = np.array(results[5]).astype(np.float32)
hp_array = np.where(hp_array == -1, np.nan, hp_array)
ap_array = np.array(results[6]).astype(np.float32)
ap_array = np.where(ap_array == -1, np.nan, ap_array)
flag_array = np.array(results[7]).astype(np.int32)
num_rows = len(results[0])
fp = file_path.parent.resolve() / (f"GFZ_Hpo_Hp{self.data_res}_" +
ystr + '.nc')
fp.parent.resolve().mkdir(parents=True, exist_ok=True)
fnc = nc.Dataset(fp, 'w')
fnc.createDimension('UNIX_TIME', num_rows)
fnc.title = f"GFZ Hpo/Hp{self.data_res} index"
time = fnc.createVariable('UNIX_TIME', np.float64, ('UNIX_TIME', ))
time.units = 'seconds since 1970-01-01 00:00:00.0'
hp = fnc.createVariable('Hp', np.float32, ('UNIX_TIME', ))
ap = fnc.createVariable('ap', np.float32, ('UNIX_TIME', ))
flag = fnc.createVariable('FLAG', np.float32, ('UNIX_TIME', ))
time[::] = time_array[::]
hp[::] = hp_array[::]
ap[::] = ap_array[::]
flag[::] = flag_array
print('From {} to {}.'.format(
datetime.datetime.utcfromtimestamp(time_array[0]),
datetime.datetime.utcfromtimestamp(time_array[-1])))
mylog.StreamLogger.info(
"The requested Hpo/Hp{} data has been downloaded and saved in the file {}."
.format(self.data_res, fp))
fnc.close()
self.done = True
def setup_module(module):
if verbose: print ("setup_module module:%s" % module.__name__)
if verbose: print ("Python version: {}".format(sys.version))
# Put any setup code in here, like making temporary files
# Make 5 years of a noleap calendar on the first of each month
global times
for y in range(1,6):
for m in range(1,13):
times.append(np.round(cftime.date2num(cftime.datetime(y,m,1),units='days since 01-01-01',calendar='noleap'),8))
times = np.array(times)
def datetimes_from_dataset(dataset, epoch=False, **kwargs):
""" Return an array of datetimes for the times in a netCDF Dataset. """
if epoch:
dtdata = num2date(dataset.variables['time'][:],
dataset.variables['time'].units)
epnum = date2num(dtdata, EPOCH_UNITS)
return num2date(epnum, EPOCH_UNITS, **kwargs)
else:
return num2date(dataset.variables['time'][:],
dataset.variables['time'].units, **kwargs)
def test_decode_multidim_time_outside_timestamp_range(
calendar, enable_cftimeindex):
from datetime import datetime
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = [datetime(1, 4, day) for day in range(1, 6)]
times2 = [datetime(1, 5, day) for day in range(1, 6)]
noleap_time1 = cftime.date2num(times1, units, calendar=calendar)
noleap_time2 = cftime.date2num(times2, units, calendar=calendar)
mdim_time = np.empty((len(noleap_time1), 2), )
mdim_time[:, 0] = noleap_time1
mdim_time[:, 1] = noleap_time2
if enable_cftimeindex:
expected1 = cftime.num2date(noleap_time1, units, calendar,
only_use_cftime_datetimes=True)
expected2 = cftime.num2date(noleap_time2, units, calendar,
only_use_cftime_datetimes=True)
else:
expected1 = cftime.num2date(noleap_time1, units, calendar)
expected2 = cftime.num2date(noleap_time2, units, calendar)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
mdim_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == np.dtype('O')
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range(
calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D')
times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D')
time1 = cftime.date2num(times1.to_pydatetime(),
units, calendar=calendar)
time2 = cftime.date2num(times2.to_pydatetime(),
units, calendar=calendar)
mdim_time = np.empty((len(time1), 2), )
mdim_time[:, 0] = time1
mdim_time[:, 1] = time2
if cftime.__name__ == 'cftime':
expected1 = cftime.num2date(time1, units, calendar,
only_use_cftime_datetimes=True)
expected2 = cftime.num2date(time2, units, calendar,
only_use_cftime_datetimes=True)
else:
expected1 = cftime.num2date(time1, units, calendar)
expected2 = cftime.num2date(time2, units, calendar)
expected_dtype = np.dtype('O')
actual = coding.times.decode_cf_datetime(
mdim_time, units, calendar=calendar)
assert actual.dtype == expected_dtype
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def prepare_nc(
trg_file,
times,
x,
y,
metadata={},
logging=logging,
units="Days since 1900-01-01 00:00:00",
calendar="gregorian",
):
"""
This function prepares a NetCDF file with given metadata, for a certain year, daily basis data
The function assumes a gregorian calendar and a time unit 'Days since 1900-01-01 00:00:00'
"""
logger.info('Setting up "' + trg_file + '"')
times_list = cftime.date2num(times, units=units, calendar=calendar)
nc_trg = nc.Dataset(trg_file, "w")
logger.info("Setting up dimensions and attributes")
nc_trg.createDimension("time", 0) # NrOfDays*8
nc_trg.createDimension("lat", len(y))
nc_trg.createDimension("lon", len(x))
times_nc = nc_trg.createVariable("time", "f8", ("time",))
times_nc.units = units
times_nc.calendar = calendar
times_nc.standard_name = "time"
times_nc.long_name = "time"
times_nc[:] = times_list
y_var = nc_trg.createVariable("lat", "f4", ("lat",))
y_var.standard_name = "latitude"
y_var.long_name = "latitude"
y_var.units = "degrees_north"
x_var = nc_trg.createVariable("lon", "f4", ("lon",))
x_var.standard_name = "longitude"
x_var.long_name = "longitude"
x_var.units = "degrees_east"
y_var[:] = y
x_var[:] = x
projection = nc_trg.createVariable("projection", "c")
projection.long_name = "wgs84"
projection.EPSG_code = "EPSG:4326"
projection.proj4_params = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
projection.grid_mapping_name = "latitude_longitude"
# now add all attributes from user-defined metadata
for attr in metadata:
nc_trg.setncattr(attr, metadata[attr])
nc_trg.sync()
return nc_trg
def test_decode_non_standard_calendar_single_element(
calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
try:
dt = cftime.netcdftime.datetime(2001, 2, 29)
except AttributeError:
# Must be using the standalone cftime library
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar)
if cftime.__name__ == 'cftime':
expected = np.asarray(cftime.num2date(
num_time, units, calendar, only_use_cftime_datetimes=True))
else:
expected = np.asarray(cftime.num2date(num_time, units, calendar))
assert actual.dtype == np.dtype('O')
assert expected == actual
def prepare_nc(
trgFile,
timeList,
x,
y,
metadata,
logger,
EPSG="EPSG:4326",
units=None,
calendar="gregorian",
Format="NETCDF4",
complevel=9,
zlib=True,
least_significant_digit=None,
FillValue=1e31,
):
"""
This function prepares a NetCDF file with given metadata, for a certain year, daily basis data
The function assumes a gregorian calendar and a time unit 'Days since 1900-01-01 00:00:00'
"""
logger.info("Setting up netcdf output: " + trgFile)
if units == None: # Use start of the run
epoch = timeList[0]
units = "seconds since %04d-%02d-%02d %02d:%02d:%02d.0 00:00" % (
epoch.year,
epoch.month,
epoch.day,
epoch.hour,
epoch.minute,
epoch.second,
)
startDayNr = cftime.date2num(
timeList[0].replace(tzinfo=None), units=units, calendar=calendar
)
endDayNr = cftime.date2num(
timeList[-1].replace(tzinfo=None), units=units, calendar=calendar
)
timeAR = np.linspace(startDayNr, endDayNr, num=len(timeList))
if os.path.exists(trgFile):
os.remove(trgFile)
nc_trg = netCDF4.Dataset(
trgFile, "w", format=Format, zlib=zlib, complevel=complevel
)
logger.info(
"Setting up dimensions and attributes. Steps: "
+ str(len(timeList))
+ " lat: "
+ str(len(y))
+ " lon: "
+ str(len(x))
)
if len(timeAR) == 1:
nc_trg.createDimension("time", 1)
else:
nc_trg.createDimension("time", 0) # NrOfDays*8
DateHour = nc_trg.createVariable(
"time", "f8", ("time",), fill_value=FillValue, zlib=zlib, complevel=complevel
)
DateHour.units = units
DateHour.calendar = calendar
DateHour.standard_name = "time"
DateHour.long_name = "time"
DateHour.axis = "T"
DateHour[:] = timeAR
# make a proj4 string
srs = osgeo.osr.SpatialReference()
res = srs.ImportFromEPSG(int(EPSG[5:]))
if res != 0:
logger.error(
"EPGS not converted correctly: "
+ EPSG
+ ". Is the GDAL_DATA environment variable set correctly?"
)
sys.exit(1)
projStr = srs.ExportToProj4()
proj_src = "+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs"
if srs.IsProjected() == 0: # ONly lat lon needed
nc_trg.createDimension("lat", len(y))
nc_trg.createDimension("lon", len(x))
y_var = nc_trg.createVariable(
"lat", "f4", ("lat",), fill_value=FillValue, zlib=zlib, complevel=complevel
)
y_var.standard_name = "latitude"
y_var.long_name = "latitude"
y_var.units = "degrees_north"
y_var.axis = "Y"
x_var = nc_trg.createVariable(
"lon", "f4", ("lon",), fill_value=FillValue, zlib=zlib, complevel=complevel
)
x_var.standard_name = "longitude"
x_var.long_name = "longitude"
x_var.units = "degrees_east"
x_var.axis = "X"
y_var[:] = y
x_var[:] = x
crs = nc_trg.createVariable("crs", "c")
crs.long_name = "wgs84"
crs.proj4_params = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
crs.grid_mapping_name = "latitude_longitude"
else: # Assume regular grid in m
nc_trg.createDimension("y", len(y))
nc_trg.createDimension("x", len(x))
y_var = nc_trg.createVariable(
"y", "f4", ("y",), fill_value=FillValue, zlib=zlib, complevel=complevel
)
y_var.standard_name = "projection_y_coordinate"
y_var.long_name = "y-coordinate in Cartesian system"
y_var.units = "m"
y_var.axis = "Y"
x_var = nc_trg.createVariable(
"x", "f4", ("x",), fill_value=FillValue, zlib=zlib, complevel=complevel
)
x_var.standard_name = "projection_x_coordinate"
x_var.long_name = "x-coordinate in Cartesian system"
x_var.units = "m"
x_var.axis = "X"
y_var[:] = y
x_var[:] = x
crs = nc_trg.createVariable("crs", "c")
crs.long_name = EPSG
crs.grid_mapping_name = "universal_transverse_mercator"
crs.utm_zone_number = srs.GetUTMZone()
crs.semi_major_axis = srs.GetSemiMajor()
crs.inverse_flattening = srs.GetInvFlattening()
crs._CoordinateTransformType = "Projection"
crs._CoordinateAxisTypes = "y x"
crs.proj4_params = projStr
# Also write lat lon fields
XI, YI = np.meshgrid(x, y)
lon_vals, lat_vals = convertCoord(projStr, proj_src, XI, YI)
# Need to create lat-lon fields
lat = nc_trg.createVariable("lat", "f4", ("y", "x"))
lat.standard_name = "latitude"
lat.long_name = "latitude coordinate"
lat.units = "degrees_north"
lat.coordinates = "lat lon"
lat.grid_mapping = "wgs84"
# lat._CoordinateAxisType = "Lat"
lat[:, :] = lat_vals
lon = nc_trg.createVariable("lon", "f4", ("y", "x"))
lon.standard_name = "longitude"
lon.long_name = "longitude coordinate"
lon.units = "degrees_east"
lon.coordinates = "lat lon"
lon.grid_mapping = "wgs84"
# lon._CoordinateAxisType = "Lon"
lon[:, :] = lon_vals
crs.EPSG_code = EPSG
# now add all attributes from user-defined metadata
for attr in metadata:
nc_trg.setncattr(attr, metadata[attr])
nc_trg.sync()
nc_trg.close()
def encode_datetime(d):
return np.nan if d is None else cftime.date2num(d, units, calendar)
