def test_decode_non_standard_calendar_inside_timestamp_range(
calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = pd.date_range('2001-04-01-00', end='2001-04-30-23',
freq='H')
non_standard_time = cftime.date2num(
times.to_pydatetime(), units, calendar=calendar)
if cftime.__name__ == 'cftime':
expected = cftime.num2date(
non_standard_time, units, calendar=calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(non_standard_time, units,
calendar=calendar)
expected_dtype = np.dtype('O')
actual = coding.times.decode_cf_datetime(
non_standard_time, units, calendar=calendar)
assert actual.dtype == expected_dtype
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def _decode_datetime_with_cftime(num_dates, units, calendar,
enable_cftimeindex):
cftime = _import_cftime()
if enable_cftimeindex:
_require_standalone_cftime()
dates = np.asarray(cftime.num2date(num_dates, units, calendar,
only_use_cftime_datetimes=True))
else:
dates = np.asarray(cftime.num2date(num_dates, units, calendar))
if (dates[np.nanargmin(num_dates)].year < 1678 or
dates[np.nanargmax(num_dates)].year >= 2262):
if not enable_cftimeindex or calendar in _STANDARD_CALENDARS:
warnings.warn(
'Unable to decode time axis into full '
'numpy.datetime64 objects, continuing using dummy '
'cftime.datetime objects instead, reason: dates out '
'of range', SerializationWarning, stacklevel=3)
else:
if enable_cftimeindex:
if calendar in _STANDARD_CALENDARS:
dates = cftime_to_nptime(dates)
else:
try:
dates = cftime_to_nptime(dates)
except ValueError as e:
warnings.warn(
'Unable to decode time axis into full '
'numpy.datetime64 objects, continuing using '
'dummy cftime.datetime objects instead, reason:'
'{0}'.format(e), SerializationWarning, stacklevel=3)
return dates
def test_decode_dates_outside_timestamp_range(
calendar, enable_cftimeindex):
from datetime import datetime
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = [datetime(1, 4, 1, h) for h in range(1, 5)]
noleap_time = cftime.date2num(times, units, calendar=calendar)
if enable_cftimeindex:
expected = cftime.num2date(noleap_time, units, calendar=calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(noleap_time, units, calendar=calendar)
expected_date_type = type(expected[0])
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
noleap_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert all(isinstance(value, expected_date_type) for value in actual)
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def _decode_datetime_with_cftime(num_dates, units, calendar):
cftime = _import_cftime()
if cftime.__name__ == 'cftime':
return np.asarray(cftime.num2date(num_dates, units, calendar,
only_use_cftime_datetimes=True))
else:
# Must be using num2date from an old version of netCDF4 which
# does not have the only_use_cftime_datetimes option.
return np.asarray(cftime.num2date(num_dates, units, calendar))
def test_decode_non_standard_calendar_inside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times = pd.date_range('2001-04-01-00', end='2001-04-30-23',
freq='H')
noleap_time = cftime.date2num(times.to_pydatetime(), units,
calendar=calendar)
if enable_cftimeindex:
expected = cftime.num2date(noleap_time, units, calendar=calendar)
expected_dtype = np.dtype('O')
else:
expected = times.values
expected_dtype = np.dtype('M8[ns]')
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
noleap_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == expected_dtype
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
def test_decode_non_standard_calendar_single_element_fallback(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
try:
dt = cftime.netcdftime.datetime(2001, 2, 29)
except AttributeError:
# Must be using standalone netcdftime library
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
if enable_cftimeindex:
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
else:
with pytest.warns(SerializationWarning,
match='Unable to decode time axis'):
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
expected = np.asarray(cftime.num2date(num_time, units, calendar))
assert actual.dtype == np.dtype('O')
assert expected == actual
def test_decode_non_standard_calendar_fallback(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
# ensure leap year doesn't matter
for year in [2010, 2011, 2012, 2013, 2014]:
units = 'days since {0}-01-01'.format(year)
num_times = np.arange(100)
expected = cftime.num2date(num_times, units, calendar)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
actual = coding.times.decode_cf_datetime(
num_times, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
if enable_cftimeindex:
assert len(w) == 0
else:
assert len(w) == 1
assert 'Unable to decode time axis' in str(w[0].message)
assert actual.dtype == np.dtype('O')
assert_array_equal(actual, expected)
def test_decode_single_element_outside_timestamp_range(
calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
for days in [1, 1470376]:
for num_time in [days, [days], [[days]]]:
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar)
if cftime.__name__ == 'cftime':
expected = cftime.num2date(days, units, calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(days, units, calendar)
assert isinstance(actual.item(), type(expected))
def test_decode_multidim_time_outside_timestamp_range(
calendar, enable_cftimeindex):
from datetime import datetime
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = [datetime(1, 4, day) for day in range(1, 6)]
times2 = [datetime(1, 5, day) for day in range(1, 6)]
noleap_time1 = cftime.date2num(times1, units, calendar=calendar)
noleap_time2 = cftime.date2num(times2, units, calendar=calendar)
mdim_time = np.empty((len(noleap_time1), 2), )
mdim_time[:, 0] = noleap_time1
mdim_time[:, 1] = noleap_time2
if enable_cftimeindex:
expected1 = cftime.num2date(noleap_time1, units, calendar,
only_use_cftime_datetimes=True)
expected2 = cftime.num2date(noleap_time2, units, calendar,
only_use_cftime_datetimes=True)
else:
expected1 = cftime.num2date(noleap_time1, units, calendar)
expected2 = cftime.num2date(noleap_time2, units, calendar)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
mdim_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == np.dtype('O')
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def _generate_linear_range(start, end, periods):
"""Generate an equally-spaced sequence of cftime.datetime objects between
and including two dates (whose length equals the number of periods)."""
import cftime
total_seconds = (end - start).total_seconds()
values = np.linspace(0., total_seconds, periods, endpoint=True)
units = 'seconds since {}'.format(format_cftime_datetime(start))
calendar = start.calendar
return cftime.num2date(values, units=units, calendar=calendar,
only_use_cftime_datetimes=True)
def test_cf_datetime(num_dates, units, calendar):
cftime = _import_cftime()
if cftime.__name__ == "cftime":
expected = cftime.num2date(num_dates,
units,
calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(num_dates, units, calendar)
min_y = np.ravel(np.atleast_1d(expected))[np.nanargmin(num_dates)].year
max_y = np.ravel(np.atleast_1d(expected))[np.nanargmax(num_dates)].year
if min_y >= 1678 and max_y < 2262:
expected = cftime_to_nptime(expected)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "Unable to decode time axis")
actual = coding.times.decode_cf_datetime(num_dates, units, calendar)
abs_diff = np.asarray(abs(actual - expected)).ravel()
abs_diff = pd.to_timedelta(abs_diff.tolist()).to_numpy()
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, "s")).all()
encoded, _, _ = coding.times.encode_cf_datetime(actual, units, calendar)
if "1-1-1" not in units:
# pandas parses this date very strangely, so the original
# units/encoding cannot be preserved in this case:
# (Pdb) pd.to_datetime('1-1-1 00:00:0.0')
# Timestamp('2001-01-01 00:00:00')
assert_array_equal(num_dates, np.around(encoded, 1))
if hasattr(num_dates,
"ndim") and num_dates.ndim == 1 and "1000" not in units:
# verify that wrapping with a pandas.Index works
# note that it *does not* currently work to even put
# non-datetime64 compatible dates into a pandas.Index
encoded, _, _ = coding.times.encode_cf_datetime(
pd.Index(actual), units, calendar)
assert_array_equal(num_dates, np.around(encoded, 1))
def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range(
calendar):
import cftime
units = "days since 0001-01-01"
times1 = pd.date_range("2001-04-01", end="2001-04-05", freq="D")
times2 = pd.date_range("2001-05-01", end="2001-05-05", freq="D")
time1 = cftime.date2num(times1.to_pydatetime(), units, calendar=calendar)
time2 = cftime.date2num(times2.to_pydatetime(), units, calendar=calendar)
mdim_time = np.empty((len(time1), 2))
mdim_time[:, 0] = time1
mdim_time[:, 1] = time2
if cftime.__name__ == "cftime":
expected1 = cftime.num2date(time1,
units,
calendar,
only_use_cftime_datetimes=True)
expected2 = cftime.num2date(time2,
units,
calendar,
only_use_cftime_datetimes=True)
else:
expected1 = cftime.num2date(time1, units, calendar)
expected2 = cftime.num2date(time2, units, calendar)
expected_dtype = np.dtype("O")
actual = coding.times.decode_cf_datetime(mdim_time,
units,
calendar=calendar)
assert actual.dtype == expected_dtype
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, "s")).all()
assert (abs_diff2 <= np.timedelta64(1, "s")).all()
def test_cf_datetime(num_dates, units, calendar):
cftime = _import_cftime()
if cftime.__name__ == 'cftime':
expected = cftime.num2date(num_dates, units, calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(num_dates, units, calendar)
min_y = np.ravel(np.atleast_1d(expected))[np.nanargmin(num_dates)].year
max_y = np.ravel(np.atleast_1d(expected))[np.nanargmax(num_dates)].year
if min_y >= 1678 and max_y < 2262:
expected = cftime_to_nptime(expected)
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(num_dates, units,
calendar)
abs_diff = np.atleast_1d(abs(actual - expected)).astype(np.timedelta64)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, 's')).all()
encoded, _, _ = coding.times.encode_cf_datetime(actual, units,
calendar)
if '1-1-1' not in units:
# pandas parses this date very strangely, so the original
# units/encoding cannot be preserved in this case:
# (Pdb) pd.to_datetime('1-1-1 00:00:0.0')
# Timestamp('2001-01-01 00:00:00')
assert_array_equal(num_dates, np.around(encoded, 1))
if (hasattr(num_dates, 'ndim') and num_dates.ndim == 1 and
'1000' not in units):
# verify that wrapping with a pandas.Index works
# note that it *does not* currently work to even put
# non-datetime64 compatible dates into a pandas.Index
encoded, _, _ = coding.times.encode_cf_datetime(
pd.Index(actual), units, calendar)
assert_array_equal(num_dates, np.around(encoded, 1))
def test_decode_360_day_calendar():
cftime = _import_cftime()
calendar = '360_day'
# ensure leap year doesn't matter
for year in [2010, 2011, 2012, 2013, 2014]:
units = 'days since {0}-01-01'.format(year)
num_times = np.arange(100)
if cftime.__name__ == 'cftime':
expected = cftime.num2date(num_times, units, calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(num_times, units, calendar)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
actual = coding.times.decode_cf_datetime(
num_times, units, calendar=calendar)
assert len(w) == 0
assert actual.dtype == np.dtype('O')
assert_array_equal(actual, expected)
def test_decode_non_standard_calendar_single_element(
calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
try:
dt = cftime.netcdftime.datetime(2001, 2, 29)
except AttributeError:
# Must be using the standalone cftime library
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar)
if cftime.__name__ == 'cftime':
expected = np.asarray(cftime.num2date(
num_time, units, calendar, only_use_cftime_datetimes=True))
else:
expected = np.asarray(cftime.num2date(num_time, units, calendar))
assert actual.dtype == np.dtype('O')
assert expected == actual
def test_use_cftime_true(calendar, units_year):
from cftime import num2date
numerical_dates = [0, 1]
units = f"days since {units_year}-01-01"
expected = num2date(
numerical_dates, units, calendar, only_use_cftime_datetimes=True
)
with pytest.warns(None) as record:
result = decode_cf_datetime(numerical_dates, units, calendar, use_cftime=True)
np.testing.assert_array_equal(result, expected)
assert not record
def validate_array_values(name, array):
if name == "time":
for i, s in enumerate(states):
value = cftime.num2date(
array[i],
units="seconds since 2010-01-01 00:00:00",
calendar=calendar,
)
assert value == s["time"]
else:
for i, s in enumerate(states):
numpy.testing.assert_array_equal(array[i, 0, :],
s[name].view[:])
def test_decode_non_standard_calendar_single_element(
calendar):
cftime = _import_cftime()
units = 'days since 0001-01-01'
try:
dt = cftime.netcdftime.datetime(2001, 2, 29)
except AttributeError:
# Must be using the standalone cftime library
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar)
if cftime.__name__ == 'cftime':
expected = np.asarray(cftime.num2date(
num_time, units, calendar, only_use_cftime_datetimes=True))
else:
expected = np.asarray(cftime.num2date(num_time, units, calendar))
assert actual.dtype == np.dtype('O')
assert expected == actual
def test_decode_360_day_calendar():
cftime = _import_cftime()
calendar = '360_day'
# ensure leap year doesn't matter
for year in [2010, 2011, 2012, 2013, 2014]:
units = 'days since {0}-01-01'.format(year)
num_times = np.arange(100)
if cftime.__name__ == 'cftime':
expected = cftime.num2date(num_times, units, calendar,
only_use_cftime_datetimes=True)
else:
expected = cftime.num2date(num_times, units, calendar)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
actual = coding.times.decode_cf_datetime(
num_times, units, calendar=calendar)
assert len(w) == 0
assert actual.dtype == np.dtype('O')
assert_array_equal(actual, expected)
def test_use_cftime_default_non_standard_calendar(calendar, units_year):
from cftime import num2date
numerical_dates = [0, 1]
units = "days since {}-01-01".format(units_year)
expected = num2date(
numerical_dates, units, calendar, only_use_cftime_datetimes=True
)
with pytest.warns(None) as record:
result = decode_cf_datetime(numerical_dates, units, calendar)
np.testing.assert_array_equal(result, expected)
assert not record
def test_format_parse_datetime():
dates = [
cftime.num2date(t, units="days since 01-01-01", calendar="noleap")
for t in times
]
assert format_datetime(dates[0]) == "0001-01-01 00:00:00"
assert format_datetime(dates[-1]) == "0005-12-01 00:00:00"
for d in dates:
assert parse_datetime(format_datetime(d), "noleap") == d
dates = [
cftime.num2date(t,
units="days since 01-01-01",
calendar="proleptic_gregorian") for t in times
]
assert format_datetime(dates[0]) == "0001-01-01 00:00:00"
assert format_datetime(dates[-1]) == "0005-11-30 00:00:00"
for d in dates:
assert parse_datetime(format_datetime(d), "proleptic_gregorian") == d
def _generate_linear_range(start, end, periods):
"""Generate an equally-spaced sequence of cftime.datetime objects between
and including two dates (whose length equals the number of periods)."""
if cftime is None:
raise ModuleNotFoundError("No module named 'cftime'")
total_seconds = (end - start).total_seconds()
values = np.linspace(0.0, total_seconds, periods, endpoint=True)
units = f"seconds since {format_cftime_datetime(start)}"
calendar = start.calendar
return cftime.num2date(
values, units=units, calendar=calendar, only_use_cftime_datetimes=True
)
def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D')
times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D')
noleap_time1 = cftime.date2num(times1.to_pydatetime(),
units, calendar=calendar)
noleap_time2 = cftime.date2num(times2.to_pydatetime(),
units, calendar=calendar)
mdim_time = np.empty((len(noleap_time1), 2), )
mdim_time[:, 0] = noleap_time1
mdim_time[:, 1] = noleap_time2
if enable_cftimeindex:
expected1 = cftime.num2date(noleap_time1, units, calendar)
expected2 = cftime.num2date(noleap_time2, units, calendar)
expected_dtype = np.dtype('O')
else:
expected1 = times1.values
expected2 = times2.values
expected_dtype = np.dtype('M8[ns]')
actual = coding.times.decode_cf_datetime(
mdim_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == expected_dtype
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D')
times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D')
noleap_time1 = cftime.date2num(times1.to_pydatetime(),
units, calendar=calendar)
noleap_time2 = cftime.date2num(times2.to_pydatetime(),
units, calendar=calendar)
mdim_time = np.empty((len(noleap_time1), 2), )
mdim_time[:, 0] = noleap_time1
mdim_time[:, 1] = noleap_time2
if enable_cftimeindex:
expected1 = cftime.num2date(noleap_time1, units, calendar)
expected2 = cftime.num2date(noleap_time2, units, calendar)
expected_dtype = np.dtype('O')
else:
expected1 = times1.values
expected2 = times2.values
expected_dtype = np.dtype('M8[ns]')
actual = coding.times.decode_cf_datetime(
mdim_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
assert actual.dtype == expected_dtype
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, 's')).all()
assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def _generate_linear_range(start, end, periods):
"""Generate an equally-spaced sequence of cftime.datetime objects between
and including two dates (whose length equals the number of periods)."""
import cftime
total_seconds = (end - start).total_seconds()
values = np.linspace(0.0, total_seconds, periods, endpoint=True)
units = "seconds since {}".format(format_cftime_datetime(start))
calendar = start.calendar
return cftime.num2date(values,
units=units,
calendar=calendar,
only_use_cftime_datetimes=True)
def _defineGeneralHeader(self, header_items=None):
"""
Defines known header items and overwrites any with header_items
key/value pairs.
"""
if header_items == None:
header_items = {}
warning_message = "Nappy Warning: Could not get the first date in the file. You will need to manually edit the output file."
# Check if DATE field previously known in NASA Ames file
time_now = [
int(i) for i in time.strftime("%Y %m %d",
time.localtime(time.time())).split()
]
if not "RDATE" in self.na_dict:
self.na_dict["RDATE"] = time_now
if xarray_utils.is_time(self.ax0):
# Get first date in list
try:
units = self.ax0.encoding["units"]
first_day = self.na_dict["X"][0]
# Cope with "X" being a list or list of lists (for different FFIs)
while hasattr(first_day, "__len__"):
first_day = first_day[0]
self.na_dict["DATE"] = \
[getattr(cftime.num2date(first_day, units), attr) for attr in ('year', 'month', 'day')]
except Exception:
msg = warning_message
log.info(msg)
self.output_message.append(msg)
self.na_dict["DATE"] = [999] * 3
else:
if not "DATE" in self.na_dict:
msg = warning_message
log.info(msg)
self.output_message.append(msg)
self.na_dict["DATE"] = [999] * 3
else:
pass # i.e. use existing DATE
self.na_dict["IVOL"] = 1
self.na_dict["NVOL"] = 1
for key in header_items.keys():
self.na_dict[key] = header_items[key]
def compute_mon_climatology(dsm):
'''Compute a monthly climatology'''
tb_name, tb_dim = time_bound_var(dsm)
grid_vars = [v for v in dsm.variables if 'time' not in dsm[v].dims]
variables = [
v for v in dsm.variables
if 'time' in dsm[v].dims and v not in ['time', tb_name]
]
# save attrs
attrs = {v: dsm[v].attrs for v in dsm.variables}
encoding = {
v: {
key: val
for key, val in dsm[v].encoding.items()
if key in ['dtype', '_FillValue', 'missing_value']
}
for v in dsm.variables
}
#-- compute time variable
date = cftime.num2date(dsm[tb_name].mean(tb_dim),
units=dsm.time.attrs['units'],
calendar=dsm.time.attrs['calendar'])
dsm.time.values = date
if len(date) % 12 != 0:
raise ValueError('Time axis not evenly divisible by 12!')
#-- compute climatology
ds = dsm.drop(grid_vars).groupby('time.month').mean('time').rename(
{'month': 'time'})
#-- put grid_vars back
ds = xr.merge(
(ds, dsm.drop([v for v in dsm.variables if v not in grid_vars])))
attrs['time'] = {'long_name': 'Month', 'units': 'month'}
del encoding['time']
# put the attributes back
for v in ds.variables:
ds[v].attrs = attrs[v]
# put the encoding back
for v in ds.variables:
if v in encoding:
ds[v].encoding = encoding[v]
return ds
def test_decode_multidim_time_outside_timestamp_range(calendar):
from datetime import datetime
import cftime
units = "days since 0001-01-01"
times1 = [datetime(1, 4, day) for day in range(1, 6)]
times2 = [datetime(1, 5, day) for day in range(1, 6)]
time1 = cftime.date2num(times1, units, calendar=calendar)
time2 = cftime.date2num(times2, units, calendar=calendar)
mdim_time = np.empty((len(time1), 2))
mdim_time[:, 0] = time1
mdim_time[:, 1] = time2
expected1 = cftime.num2date(time1,
units,
calendar,
only_use_cftime_datetimes=True)
expected2 = cftime.num2date(time2,
units,
calendar,
only_use_cftime_datetimes=True)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "Unable to decode time axis")
actual = coding.times.decode_cf_datetime(mdim_time,
units,
calendar=calendar)
assert actual.dtype == np.dtype("O")
abs_diff1 = abs(actual[:, 0] - expected1)
abs_diff2 = abs(actual[:, 1] - expected2)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff1 <= np.timedelta64(1, "s")).all()
assert (abs_diff2 <= np.timedelta64(1, "s")).all()
def _decode_datetime_with_cftime(num_dates, units, calendar,
enable_cftimeindex):
cftime = _import_cftime()
if enable_cftimeindex:
_require_standalone_cftime()
dates = np.asarray(
cftime.num2date(num_dates,
units,
calendar,
only_use_cftime_datetimes=True))
else:
dates = np.asarray(cftime.num2date(num_dates, units, calendar))
if (dates[np.nanargmin(num_dates)].year < 1678
or dates[np.nanargmax(num_dates)].year >= 2262):
if not enable_cftimeindex or calendar in _STANDARD_CALENDARS:
warnings.warn(
'Unable to decode time axis into full '
'numpy.datetime64 objects, continuing using dummy '
'cftime.datetime objects instead, reason: dates out '
'of range',
SerializationWarning,
stacklevel=3)
else:
if enable_cftimeindex:
if calendar in _STANDARD_CALENDARS:
dates = cftime_to_nptime(dates)
else:
try:
dates = cftime_to_nptime(dates)
except ValueError as e:
warnings.warn('Unable to decode time axis into full '
'numpy.datetime64 objects, continuing using '
'dummy cftime.datetime objects instead, reason:'
'{0}'.format(e),
SerializationWarning,
stacklevel=3)
return dates
def _decode_datetime_with_cftime(num_dates, units, calendar):
cftime = _import_cftime()
if cftime.__name__ == 'cftime':
dates = np.asarray(cftime.num2date(num_dates, units, calendar,
only_use_cftime_datetimes=True))
else:
# Must be using num2date from an old version of netCDF4 which
# does not have the only_use_cftime_datetimes option.
dates = np.asarray(cftime.num2date(num_dates, units, calendar))
if (dates[np.nanargmin(num_dates)].year < 1678 or
dates[np.nanargmax(num_dates)].year >= 2262):
if calendar in _STANDARD_CALENDARS:
warnings.warn(
'Unable to decode time axis into full '
'numpy.datetime64 objects, continuing using dummy '
'cftime.datetime objects instead, reason: dates out '
'of range', SerializationWarning, stacklevel=3)
else:
if calendar in _STANDARD_CALENDARS:
dates = cftime_to_nptime(dates)
return dates
def test_use_cftime_default_standard_calendar_out_of_range(
calendar, units_year):
from cftime import num2date
numerical_dates = [0, 1]
units = f"days since {units_year}-01-01"
expected = num2date(numerical_dates,
units,
calendar,
only_use_cftime_datetimes=True)
with pytest.warns(SerializationWarning):
result = decode_cf_datetime(numerical_dates, units, calendar)
np.testing.assert_array_equal(result, expected)
def get_ncdates(nc, tvar='time'):
""" Return dates from netcdf time coordinate """
t = nc.variables[tvar]
dts = cftime.num2date(t[:], t.units, calendar=t.calendar)
# Matplotlib does not support the real_datetime format returned by cftime. Furthermore,
# it is not possible to create datetime.datetime directly (i.e. using cftime.num2pydate)
# for certain calendars (e.g. Gregorian). The nc-time-axis package extends matplotlib
# to work directly with cftime objects. However, to avoid this additional dependency,
# we manually create the python datetime objects from cftime objects.
pydts = np.array([
datetime.datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute,
dt.second) for dt in dts
])
return pydts
def _decode_datetime_with_cftime(num_dates, units, calendar):
cftime = _import_cftime()
if cftime.__name__ == 'cftime':
dates = np.asarray(cftime.num2date(num_dates, units, calendar,
only_use_cftime_datetimes=True))
else:
# Must be using num2date from an old version of netCDF4 which
# does not have the only_use_cftime_datetimes option.
dates = np.asarray(cftime.num2date(num_dates, units, calendar))
if (dates[np.nanargmin(num_dates)].year < 1678 or
dates[np.nanargmax(num_dates)].year >= 2262):
if calendar in _STANDARD_CALENDARS:
warnings.warn(
'Unable to decode time axis into full '
'numpy.datetime64 objects, continuing using dummy '
'cftime.datetime objects instead, reason: dates out '
'of range', SerializationWarning, stacklevel=3)
else:
if calendar in _STANDARD_CALENDARS:
dates = cftime_to_nptime(dates)
return dates
def test_use_cftime_default_non_standard_calendar(calendar,
units_year) -> None:
from cftime import num2date
numerical_dates = [0, 1]
units = f"days since {units_year}-01-01"
expected = num2date(numerical_dates,
units,
calendar,
only_use_cftime_datetimes=True)
with assert_no_warnings():
result = decode_cf_datetime(numerical_dates, units, calendar)
np.testing.assert_array_equal(result, expected)
def test_calendar_dask_cftime() -> None:
from cftime import num2date
# 3D lazy dask
data = xr.DataArray(
num2date(
np.random.randint(1, 1000000, size=(4, 5, 6)),
"hours since 1970-01-01T00:00",
calendar="noleap",
),
dims=("x", "y", "z"),
).chunk()
with raise_if_dask_computes(max_computes=2):
assert data.dt.calendar == "noleap"
def test_decode_non_standard_calendar_single_element(calendar):
import cftime
units = "days since 0001-01-01"
dt = cftime.datetime(2001, 2, 29)
num_time = cftime.date2num(dt, units, calendar)
actual = coding.times.decode_cf_datetime(num_time, units, calendar=calendar)
expected = np.asarray(
cftime.num2date(num_time, units, calendar, only_use_cftime_datetimes=True)
)
assert actual.dtype == np.dtype("O")
assert expected == actual
def test_decode_non_standard_calendar_inside_timestamp_range(calendar):
cftime = _import_cftime()
units = "days since 0001-01-01"
times = pd.date_range("2001-04-01-00", end="2001-04-30-23", freq="H")
non_standard_time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar)
if cftime.__name__ == "cftime":
expected = cftime.num2date(
non_standard_time, units, calendar=calendar, only_use_cftime_datetimes=True
)
else:
expected = cftime.num2date(non_standard_time, units, calendar=calendar)
expected_dtype = np.dtype("O")
actual = coding.times.decode_cf_datetime(
non_standard_time, units, calendar=calendar
)
assert actual.dtype == expected_dtype
abs_diff = abs(actual - expected)
# once we no longer support versions of netCDF4 older than 1.1.5,
# we could do this check with near microsecond accuracy:
# https://github.com/Unidata/netcdf4-python/issues/355
assert (abs_diff <= np.timedelta64(1, "s")).all()
def _convert_datetime(
datetime: Union[pydt.datetime, cftime.datetime],
new_doy: Optional[Union[float, int]] = None,
calendar: str = "default",
) -> Union[cftime.datetime, pydt.datetime, float]:
"""Convert a datetime object to another calendar.
Nanosecond information are lost as cftime.datetime doesn't support them.
Parameters
----------
datetime: Union[datetime.datetime, cftime.datetime]
A datetime object to convert.
new_doy: Optional[Union[float, int]]
Allows for redefining the day of year (thus ignoring month and day information from the source datetime).
-1 is understood as a nan.
calendar: str
The target calendar
Returns
-------
Union[cftime.datetime, datetime.datetime, np.nan]
A datetime object of the target calendar with the same year, month, day and time
as the source (month and day according to `new_doy` if given).
If the month and day doesn't exist in the target calendar, returns np.nan. (Ex. 02-29 in "noleap")
"""
if new_doy in [np.nan, -1]:
return np.nan
if new_doy is not None:
new_date = cftime.num2date(
new_doy - 1,
f"days since {datetime.year}-01-01",
calendar=calendar if calendar != "default" else "standard",
)
else:
new_date = datetime
try:
return datetime_classes[calendar](
datetime.year,
new_date.month,
new_date.day,
datetime.hour,
datetime.minute,
datetime.second,
datetime.microsecond,
)
except ValueError:
return np.nan
def test_decode_single_element_outside_timestamp_range(calendar):
import cftime
units = "days since 0001-01-01"
for days in [1, 1470376]:
for num_time in [days, [days], [[days]]]:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "Unable to decode time axis")
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar
)
expected = cftime.num2date(
days, units, calendar, only_use_cftime_datetimes=True
)
assert isinstance(actual.item(), type(expected))
def restore_dataset(self, ds):
"""Return the original time variable.
"""
if not self._time_computed:
raise ValueError("time was not computed; cannot restore dataset")
time_values = ds[self.orig_time_coord_name].values
if self.orig_time_coord_decoded:
time_values = xr.CFTimeIndex(
cftime.num2date(
time_values,
units=self.time_attrs["units"],
calendar=self.time_attrs["calendar"],
))
ds[self.time_coord_name].values = time_values
ds = ds.drop(self.orig_time_coord_name)
return ds
def default_freq(**indexer) -> str:
"""Return the default frequency."""
freq = "AS-JAN"
if indexer:
group, value = indexer.popitem()
if group == "season":
month = 12 # The "season" scheme is based on AS-DEC
elif group == "month":
month = np.take(value, 0)
elif group == "doy_bounds":
month = cftime.num2date(value[0] - 1,
"days since 2004-01-01").month
elif group == "date_bounds":
month = int(value[0][:2])
freq = "AS-" + _MONTH_ABBREVIATIONS[month]
return freq
def test_decode_single_element_outside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
for days in [1, 1470376]:
for num_time in [days, [days], [[days]]]:
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
expected = cftime.num2date(days, units, calendar)
assert isinstance(actual.item(), type(expected))
def load(self):
fnc = netCDF4.Dataset(self.file_path)
variables = {}
for var_name, var_name_nc in cdf_variable_name_dict.items():
variables[var_name] = np.array(fnc[var_name_nc]).reshape((fnc[var_name_nc].shape[0], 1))
time_units = fnc['UNIX_TIME'].units
variables['DATETIME'] = cftime.num2date(variables['UNIX_TIME'].flatten(),
units=time_units,
only_use_cftime_datetimes=False,
only_use_python_datetimes=True)
variables['DATETIME'] = np.reshape(variables['DATETIME'], (fnc['UNIX_TIME'].shape[0], 1))
self.variables = variables
self.done = True
fnc.close()
def time_year_plus_frac(ds, time_name):
"""return time variable, as year plus fraction of year"""
# this is straightforward if time has units='days since 0000-01-01' and calendar='noleap'
# so convert specification of time to that representation
# get time values as an np.ndarray of cftime objects
if np.dtype(ds[time_name]) == np.dtype('O'):
tvals_cftime = ds[time_name].values
else:
tvals_cftime = cftime.num2date(
ds[time_name].values, ds[time_name].attrs['units'], ds[time_name].attrs['calendar'])
# convert cftime objects to representation mentioned above
tvals_days = cftime.date2num(tvals_cftime, 'days since 0000-01-01', calendar='noleap')
return tvals_days / 365.0
def time_index_to_datetime(timestamps, time_units: str):
if isinstance(timestamps, np.ndarray):
timestamps = timestamps.tolist()
if not isinstance(timestamps, list):
timestamps = [timestamps]
result = [
cftime.num2date(timestamp, time_units).replace(tzinfo=pytz.UTC)
for timestamp in timestamps
]
if isinstance(result[0], list):
return list(itertools.chain(*result))
return result
def test_decode_single_element_outside_timestamp_range(
calendar, enable_cftimeindex):
if enable_cftimeindex:
pytest.importorskip('cftime')
cftime = _import_cftime()
units = 'days since 0001-01-01'
for days in [1, 1470376]:
for num_time in [days, [days], [[days]]]:
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(
num_time, units, calendar=calendar,
enable_cftimeindex=enable_cftimeindex)
expected = cftime.num2date(days, units, calendar)
assert isinstance(actual.item(), type(expected))
def test_cf_datetime(num_dates, units, calendar):
cftime = _import_cftime()
expected = _ensure_naive_tz(
cftime.num2date(num_dates, units, calendar))
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
'Unable to decode time axis')
actual = coding.times.decode_cf_datetime(num_dates, units,
calendar)
if (isinstance(actual, np.ndarray) and
np.issubdtype(actual.dtype, np.datetime64)):
# self.assertEqual(actual.dtype.kind, 'M')
# For some reason, numpy 1.8 does not compare ns precision
# datetime64 arrays as equal to arrays of datetime objects,
# but it works for us precision. Thus, convert to us
# precision for the actual array equal comparison...
actual_cmp = actual.astype('M8[us]')
else:
actual_cmp = actual
assert_array_equal(expected, actual_cmp)
encoded, _, _ = coding.times.encode_cf_datetime(actual, units,
calendar)
if '1-1-1' not in units:
# pandas parses this date very strangely, so the original
# units/encoding cannot be preserved in this case:
# (Pdb) pd.to_datetime('1-1-1 00:00:0.0')
# Timestamp('2001-01-01 00:00:00')
assert_array_equal(num_dates, np.around(encoded, 1))
if (hasattr(num_dates, 'ndim') and num_dates.ndim == 1 and
'1000' not in units):
# verify that wrapping with a pandas.Index works
# note that it *does not* currently work to even put
# non-datetime64 compatible dates into a pandas.Index
encoded, _, _ = coding.times.encode_cf_datetime(
pd.Index(actual), units, calendar)
assert_array_equal(num_dates, np.around(encoded, 1))
epsg = options.epsg
extract_type = options.extract_type
level = options.level
shp_filename = options.out_file
ts_fieldname = "timestamp"
dst_fieldname = options.dst_fieldname
step = options.step
nc = NC(filename, "r")
xdim, ydim, zdim, tdim = ppt.get_dims(nc)
if tdim:
time = nc.variables[tdim]
time_units = time.units
time_calendar = time.calendar
timestamps = cftime.num2date(time[:], time_units, time_calendar)
has_time = True
else:
tdim = None
nc.close()
src_ds = gdal.Open("NETCDF:{}:{}".format(filename, dst_fieldname))
# Get Memory Driver
mem_driver = ogr.GetDriverByName("Memory")
mem_ds = mem_driver.CreateDataSource("memory_layer")
# Get SHP Driver
shp_driver = ogr.GetDriverByName("ESRI Shapefile")
shp_filename = validateShapePath(shp_filename)
if os.path.exists(shp_filename):
# Bounding box for our catchment
BB = dict(lon=[143, 150], lat=[-37, -33])
(latidx,) = logical_and(lat >= BB["lat"][0], lat < BB["lat"][1]).nonzero()
(lonidx,) = logical_and(lon >= BB["lon"][0], lon < BB["lon"][1]).nonzero()
print(lonidx)
print(latidx)
print(lat[latidx])
print(lon[lonidx])
# get rid of the non used lat/lon now
lat = lat[latidx]
lon = lon[lonidx]
# Now get the time for the x-axis
time = ncdataset.variables["time"]
timeObj = cftime.num2date(time[:], units=time.units, calendar=time.calendar)
# Now determine area P for each timestep and display in a graph
# first the mean per area lat, next average those also
# Multiply with timestep in seconds to get mm
# unfortunateley Tair also has heigh dimension and Precip not
if mapstackname == "P":
p_select = (
ncdata[:, latidx.min() : latidx.max(), lonidx.min() : lonidx.max()]
* 86400
)
if mapstackname == "TEMP":
p_select = (
ncdata[:, 0, latidx.min() : latidx.max(), lonidx.min() : lonidx.max()]
- 273.15
def write_netcdf_timeseries(
srcFolder,
srcPrefix,
trgFile,
trgVar,
trgUnits,
trgName,
timeList,
metadata,
logger,
clone,
maxbuf=600,
Format="NETCDF4",
zlib=True,
least_significant_digit=None,
startidx=0,
EPSG="EPSG:4326",
FillVal=1e31,
):
"""
Write pcraster mapstack to netcdf file. Taken from GLOFRIS_Utils.py
- srcFolder - Folder with pcraster mapstack
- srcPrefix - name of the mapstack
- trgFile - target netcdf file
- tgrVar - variable in nc file
- trgUnits - units for the netcdf file
- timeLists - list of times
- metadata - dict with metedata for var
Optional argumenrs
- maxbuf = 600: number of timesteps to buffer before writing
"""
complevel = 9
# if necessary, make trgPrefix maximum of 8 characters
if len(srcPrefix) > 8:
srcPrefix = srcPrefix[0:8]
# Open target netCDF file
nc_trg = nc4.Dataset(trgFile, "a", format=Format)
# read time axis and convert to time objects
logger.debug("Creating time object..")
time = nc_trg.variables["time"]
nc_trg.set_fill_off()
timeObj = cftime.num2date(time[:], units=time.units, calendar=time.calendar)
try:
nc_var = nc_trg.variables[trgVar]
except:
# prepare the variable
if EPSG.lower() == "epsg:4326":
nc_var = nc_trg.createVariable(
trgVar,
"f4",
("time", "lat", "lon"),
fill_value=FillVal,
zlib=zlib,
complevel=complevel,
least_significant_digit=least_significant_digit,
)
nc_var.coordinates = "lat lon"
else:
nc_var = nc_trg.createVariable(
trgVar,
"f4",
("time", "y", "x"),
fill_value=FillVal,
zlib=zlib,
complevel=complevel,
least_significant_digit=least_significant_digit,
)
nc_var.coordinates = "lat lon"
nc_var.grid_mapping = "crs"
nc_var.units = trgUnits
nc_var.standard_name = trgName
# print metadata
for attr in metadata:
# print metadata[attr]
nc_var.setncattr(attr, metadata[attr])
nc_Fill = nc_var._FillValue
# Create a buffer of a number of timesteps to speed-up writing
bufsize = minimum(len(timeList), maxbuf)
if len(shape(nc_trg.variables["lat"])) == 2:
latlen = shape(nc_trg.variables["lat"])[0]
lonlen = shape(nc_trg.variables["lon"])[1]
else:
latlen = len(nc_trg.variables["lat"])
lonlen = len(nc_trg.variables["lon"])
timestepbuffer = np.zeros((bufsize, latlen, lonlen))
# now loop over all time steps, check the date and write valid dates to a list, write time series to PCRaster maps
for nn, curTime in enumerate(timeList):
logger.debug("Adding time: " + str(curTime))
idx = int(where(timeObj == curTime)[0])
count = nn + startidx
below_thousand = count % 1000
above_thousand = count / 1000
# read the file of interest
pcraster_file = str(srcPrefix + "%0" + str(8 - len(srcPrefix)) + ".f.%03.f") % (
above_thousand,
below_thousand,
)
pcraster_path = os.path.join(srcFolder, pcraster_file)
# write grid to PCRaster file
logger.debug("processing map: " + pcraster_file)
# x, y, data, FillVal = readMap(pcraster_path, 'PCRaster',logger)
x, y, data, FFillVal = _readMap(pcraster_path, "PCRaster", logger)
logger.debug("Setting fillval...")
data[data == FFillVal] = float(nc_Fill)
data[isinf(data)] = float(nc_Fill)
data[isnan(data)] = float(nc_Fill)
data[clone <= -999] = float(nc_Fill)
data[clone == FFillVal] = float(nc_Fill)
buffreset = (idx + 1) % maxbuf
bufpos = (idx) % maxbuf
logger.debug("Adding data to array...")
timestepbuffer[bufpos, :, :] = data
logger.debug(
"index: "
+ str(idx - bufpos)
+ " index: "
+ str(idx)
+ "bufpos: "
+ str(bufpos)
+ "idx: "
+ str(idx)
)
if buffreset == 0 or idx == bufsize - 1 or nn + 1 == len(timeList):
logger.info(
"Writing buffer to file at: "
+ str(curTime)
+ " "
+ str(int(bufpos) + 1)
+ " timesteps"
)
nc_var[idx - bufpos : idx + 1, :, :] = timestepbuffer[0 : bufpos + 1, :, :]
nc_trg.sync()
# nc_trg.sync()
nc_trg.close()
def main():
### Read input arguments #####
parser = OptionParser()
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option(
"-q",
"--quiet",
dest="verbose",
default=True,
action="store_false",
help="do not print status messages to stdout",
)
parser.add_option(
"-i",
"--ini",
dest="inifile",
default="hand_contour_inun.ini",
nargs=1,
help="ini configuration file",
)
parser.add_option(
"-f",
"--flood_map",
nargs=1,
dest="flood_map",
help="Flood map file (NetCDF point time series file",
)
parser.add_option(
"-v",
"--flood_variable",
nargs=1,
dest="flood_variable",
default="water_level",
help="variable name of flood water level",
)
parser.add_option(
"-b",
"--bankfull_map",
dest="bankfull_map",
default="",
help="Map containing bank full level (is subtracted from flood map, in NetCDF)",
)
parser.add_option(
"-c",
"--catchment",
dest="catchment_strahler",
default=7,
type="int",
help="Strahler order threshold >= are selected as catchment boundaries",
)
parser.add_option(
"-t",
"--time",
dest="time",
default="",
help="time in YYYYMMDDHHMMSS, overrides time in NetCDF input if set",
)
# parser.add_option('-s', '--hand_strahler',
# dest='hand_strahler', default=7, type='int',
# help='Strahler order threshold >= selected as riverine')
parser.add_option(
"-m",
"--max_strahler",
dest="max_strahler",
default=1000,
type="int",
help="Maximum Strahler order to loop over",
)
parser.add_option(
"-d", "--destination", dest="dest_path", default="inun", help="Destination path"
)
parser.add_option(
"-H",
"--hand_file_prefix",
dest="hand_file_prefix",
default="",
help="optional HAND file prefix of already generated HAND files",
)
parser.add_option(
"-n",
"--neg_HAND",
dest="neg_HAND",
default=0,
type="int",
help="if set to 1, allow for negative HAND values in HAND maps",
)
(options, args) = parser.parse_args()
if not os.path.exists(options.inifile):
print("path to ini file cannot be found")
sys.exit(1)
options.dest_path = os.path.abspath(options.dest_path)
if not (os.path.isdir(options.dest_path)):
os.makedirs(options.dest_path)
# set up the logger
flood_name = os.path.split(options.flood_map)[1].split(".")[0]
# case_name = 'inun_{:s}_hand_{:02d}_catch_{:02d}'.format(flood_name, options.hand_strahler, options.catchment_strahler)
case_name = "inun_{:s}_catch_{:02d}".format(flood_name, options.catchment_strahler)
logfilename = os.path.join(options.dest_path, "hand_contour_inun.log")
logger, ch = inun_lib.setlogger(logfilename, "HAND_INUN", options.verbose)
logger.info("$Id: $")
logger.info("Flood map: {:s}".format(options.flood_map))
logger.info("Bank full map: {:s}".format(options.bankfull_map))
logger.info("Destination path: {:s}".format(options.dest_path))
# read out ini file
### READ CONFIG FILE
# open config-file
config = inun_lib.open_conf(options.inifile)
# read settings
options.dem_file = inun_lib.configget(config, "HighResMaps", "dem_file", True)
options.ldd_file = inun_lib.configget(config, "HighResMaps", "ldd_file", True)
options.stream_file = inun_lib.configget(config, "HighResMaps", "stream_file", True)
options.riv_length_fact_file = inun_lib.configget(
config, "wflowResMaps", "riv_length_fact_file", True
)
options.ldd_wflow = inun_lib.configget(config, "wflowResMaps", "ldd_wflow", True)
options.riv_width_file = inun_lib.configget(
config, "wflowResMaps", "riv_width_file", True
)
options.file_format = inun_lib.configget(
config, "file_settings", "file_format", 0, datatype="int"
)
options.out_format = inun_lib.configget(
config, "file_settings", "out_format", 0, datatype="int"
)
options.latlon = inun_lib.configget(
config, "file_settings", "latlon", 0, datatype="int"
)
options.x_tile = inun_lib.configget(
config, "tiling", "x_tile", 10000, datatype="int"
)
options.y_tile = inun_lib.configget(
config, "tiling", "y_tile", 10000, datatype="int"
)
options.x_overlap = inun_lib.configget(
config, "tiling", "x_overlap", 1000, datatype="int"
)
options.y_overlap = inun_lib.configget(
config, "tiling", "y_overlap", 1000, datatype="int"
)
options.iterations = inun_lib.configget(
config, "inundation", "iterations", 20, datatype="int"
)
options.initial_level = inun_lib.configget(
config, "inundation", "initial_level", 32.0, datatype="float"
)
options.flood_volume_type = inun_lib.configget(
config, "inundation", "flood_volume_type", 0, datatype="int"
)
# options.area_multiplier = inun_lib.configget(config, 'inundation',
# 'area_multiplier', 1., datatype='float')
logger.info("DEM file: {:s}".format(options.dem_file))
logger.info("LDD file: {:s}".format(options.ldd_file))
logger.info("streamfile: {:s}".format(options.stream_file))
logger.info("Columns per tile: {:d}".format(options.x_tile))
logger.info("Rows per tile: {:d}".format(options.y_tile))
logger.info("Columns overlap: {:d}".format(options.x_overlap))
logger.info("Rows overlap: {:d}".format(options.y_overlap))
metadata_global = {}
# add metadata from the section [metadata]
meta_keys = config.options("metadata_global")
for key in meta_keys:
metadata_global[key] = config.get("metadata_global", key)
# add a number of metadata variables that are mandatory
metadata_global["config_file"] = os.path.abspath(options.inifile)
metadata_var = {}
metadata_var["units"] = "m"
metadata_var["standard_name"] = "water_surface_height_above_reference_datum"
metadata_var["long_name"] = "flooding"
metadata_var[
"comment"
] = "water_surface_reference_datum_altitude is given in file {:s}".format(
options.dem_file
)
if not os.path.exists(options.dem_file):
logger.error("path to dem file {:s} cannot be found".format(options.dem_file))
sys.exit(1)
if not os.path.exists(options.ldd_file):
logger.error("path to ldd file {:s} cannot be found".format(options.ldd_file))
sys.exit(1)
# Read extent from a GDAL compatible file
try:
extent = inun_lib.get_gdal_extent(options.dem_file)
except:
msg = "Input file {:s} not a gdal compatible file".format(options.dem_file)
inun_lib.close_with_error(logger, ch, msg)
sys.exit(1)
try:
x, y = inun_lib.get_gdal_axes(options.dem_file, logging=logger)
srs = inun_lib.get_gdal_projection(options.dem_file, logging=logger)
except:
msg = "Input file {:s} not a gdal compatible file".format(options.dem_file)
inun_lib.close_with_error(logger, ch, msg)
sys.exit(1)
# read history from flood file
if options.file_format == 0:
a = nc.Dataset(options.flood_map, "r")
metadata_global[
"history"
] = "Created by: $Id: $, boundary conditions from {:s},\nhistory: {:s}".format(
os.path.abspath(options.flood_map), a.history
)
a.close()
else:
metadata_global[
"history"
] = "Created by: $Id: $, boundary conditions from {:s},\nhistory: {:s}".format(
os.path.abspath(options.flood_map), "PCRaster file, no history"
)
# first write subcatch maps and hand maps
############### TODO ######
# setup a HAND file for each strahler order
max_s = inun_lib.define_max_strahler(options.stream_file, logging=logger)
stream_max = np.minimum(max_s, options.max_strahler)
for hand_strahler in range(options.catchment_strahler, stream_max + 1, 1):
dem_name = os.path.split(options.dem_file)[1].split(".")[0]
if os.path.isfile(
"{:s}_{:02d}.tif".format(options.hand_file_prefix, hand_strahler)
):
hand_file = "{:s}_{:02d}.tif".format(
options.hand_file_prefix, hand_strahler
)
else:
logger.info(
"No HAND files with HAND prefix were found, checking {:s}_hand_strahler_{:02d}.tif".format(
dem_name, hand_strahler
)
)
hand_file = os.path.join(
options.dest_path,
"{:s}_hand_strahler_{:02d}.tif".format(dem_name, hand_strahler),
)
if not (os.path.isfile(hand_file)):
# hand file does not exist yet! Generate it, otherwise skip!
logger.info(
"HAND file {:s} not found, start setting up...please wait...".format(
hand_file
)
)
hand_file_tmp = os.path.join(
options.dest_path,
"{:s}_hand_strahler_{:02d}.tif.tmp".format(dem_name, hand_strahler),
)
ds_hand, band_hand = inun_lib.prepare_gdal(
hand_file_tmp, x, y, logging=logger, srs=srs
)
# band_hand = ds_hand.GetRasterBand(1)
# Open terrain data for reading
ds_dem, rasterband_dem = inun_lib.get_gdal_rasterband(options.dem_file)
ds_ldd, rasterband_ldd = inun_lib.get_gdal_rasterband(options.ldd_file)
ds_stream, rasterband_stream = inun_lib.get_gdal_rasterband(
options.stream_file
)
n = 0
for x_loop in range(0, len(x), options.x_tile):
x_start = np.maximum(x_loop, 0)
x_end = np.minimum(x_loop + options.x_tile, len(x))
# determine actual overlap for cutting
for y_loop in range(0, len(y), options.y_tile):
x_overlap_min = x_start - np.maximum(x_start - options.x_overlap, 0)
x_overlap_max = (
np.minimum(x_end + options.x_overlap, len(x)) - x_end
)
n += 1
# print('tile {:001d}:'.format(n))
y_start = np.maximum(y_loop, 0)
y_end = np.minimum(y_loop + options.y_tile, len(y))
y_overlap_min = y_start - np.maximum(y_start - options.y_overlap, 0)
y_overlap_max = (
np.minimum(y_end + options.y_overlap, len(y)) - y_end
)
# cut out DEM
logger.debug(
"Computing HAND for xmin: {:d} xmax: {:d} ymin {:d} ymax {:d}".format(
x_start, x_end, y_start, y_end
)
)
terrain = rasterband_dem.ReadAsArray(
float(x_start - x_overlap_min),
float(y_start - y_overlap_min),
int((x_end + x_overlap_max) - (x_start - x_overlap_min)),
int((y_end + y_overlap_max) - (y_start - y_overlap_min)),
)
drainage = rasterband_ldd.ReadAsArray(
float(x_start - x_overlap_min),
float(y_start - y_overlap_min),
int((x_end + x_overlap_max) - (x_start - x_overlap_min)),
int((y_end + y_overlap_max) - (y_start - y_overlap_min)),
)
stream = rasterband_stream.ReadAsArray(
float(x_start - x_overlap_min),
float(y_start - y_overlap_min),
int((x_end + x_overlap_max) - (x_start - x_overlap_min)),
int((y_end + y_overlap_max) - (y_start - y_overlap_min)),
)
# write to temporary file
terrain_temp_file = os.path.join(
options.dest_path, "terrain_temp.map"
)
drainage_temp_file = os.path.join(
options.dest_path, "drainage_temp.map"
)
stream_temp_file = os.path.join(
options.dest_path, "stream_temp.map"
)
if rasterband_dem.GetNoDataValue() is not None:
inun_lib.gdal_writemap(
terrain_temp_file,
"PCRaster",
np.arange(0, terrain.shape[1]),
np.arange(0, terrain.shape[0]),
terrain,
rasterband_dem.GetNoDataValue(),
gdal_type=gdal.GDT_Float32,
logging=logger,
)
else:
# in case no nodata value is found
logger.warning(
"No nodata value found in {:s}. assuming -9999".format(
options.dem_file
)
)
inun_lib.gdal_writemap(
terrain_temp_file,
"PCRaster",
np.arange(0, terrain.shape[1]),
np.arange(0, terrain.shape[0]),
terrain,
-9999.0,
gdal_type=gdal.GDT_Float32,
logging=logger,
)
inun_lib.gdal_writemap(
drainage_temp_file,
"PCRaster",
np.arange(0, terrain.shape[1]),
np.arange(0, terrain.shape[0]),
drainage,
rasterband_ldd.GetNoDataValue(),
gdal_type=gdal.GDT_Int32,
logging=logger,
)
inun_lib.gdal_writemap(
stream_temp_file,
"PCRaster",
np.arange(0, terrain.shape[1]),
np.arange(0, terrain.shape[0]),
stream,
rasterband_ldd.GetNoDataValue(),
gdal_type=gdal.GDT_Int32,
logging=logger,
)
# read as pcr objects
pcr.setclone(terrain_temp_file)
terrain_pcr = pcr.readmap(terrain_temp_file)
drainage_pcr = pcr.lddrepair(
pcr.ldd(pcr.readmap(drainage_temp_file))
) # convert to ldd type map
stream_pcr = pcr.scalar(
pcr.readmap(stream_temp_file)
) # convert to ldd type map
# check if the highest stream order of the tile is below the hand_strahler
# if the highest stream order of the tile is smaller than hand_strahler, than DEM values are taken instead of HAND values.
max_stream_tile = inun_lib.define_max_strahler(
stream_temp_file, logging=logger
)
if max_stream_tile < hand_strahler:
hand_pcr = terrain_pcr
logger.info(
"For this tile, DEM values are used instead of HAND because there is no stream order larger than {:02d}".format(
hand_strahler
)
)
else:
# compute streams
stream_ge, subcatch = inun_lib.subcatch_stream(
drainage_pcr, hand_strahler, stream=stream_pcr
) # generate streams
# compute basins
stream_ge_dummy, subcatch = inun_lib.subcatch_stream(
drainage_pcr, options.catchment_strahler, stream=stream_pcr
) # generate streams
basin = pcr.boolean(subcatch)
hand_pcr, dist_pcr = inun_lib.derive_HAND(
terrain_pcr,
drainage_pcr,
3000,
rivers=pcr.boolean(stream_ge),
basin=basin,
neg_HAND=options.neg_HAND,
)
# convert to numpy
hand = pcr.pcr2numpy(hand_pcr, -9999.0)
# cut relevant part
if y_overlap_max == 0:
y_overlap_max = -hand.shape[0]
if x_overlap_max == 0:
x_overlap_max = -hand.shape[1]
hand_cut = hand[
0 + y_overlap_min : -y_overlap_max,
0 + x_overlap_min : -x_overlap_max,
]
band_hand.WriteArray(hand_cut, float(x_start), float(y_start))
os.unlink(terrain_temp_file)
os.unlink(drainage_temp_file)
os.unlink(stream_temp_file)
band_hand.FlushCache()
ds_dem = None
ds_ldd = None
ds_stream = None
band_hand.SetNoDataValue(-9999.0)
ds_hand = None
logger.info("Finalizing {:s}".format(hand_file))
# rename temporary file to final hand file
os.rename(hand_file_tmp, hand_file)
else:
logger.info("HAND file {:s} already exists...skipping...".format(hand_file))
#####################################################################################
# HAND file has now been prepared, moving to flood mapping part #
#####################################################################################
# set the clone
pcr.setclone(options.ldd_wflow)
# read wflow ldd as pcraster object
ldd_pcr = pcr.readmap(options.ldd_wflow)
xax, yax, riv_width, fill_value = inun_lib.gdal_readmap(
options.riv_width_file, "GTiff", logging=logger
)
# determine cell length in meters using ldd_pcr as clone (if latlon=True, values are converted to m2
x_res, y_res, reallength_wflow = pcrut.detRealCellLength(
pcr.scalar(ldd_pcr), not (bool(options.latlon))
)
cell_surface_wflow = pcr.pcr2numpy(x_res * y_res, 0)
if options.flood_volume_type == 0:
# load the staticmaps needed to estimate volumes across all
# xax, yax, riv_length, fill_value = inun_lib.gdal_readmap(options.riv_length_file, 'GTiff', logging=logger)
# riv_length = np.ma.masked_where(riv_length==fill_value, riv_length)
xax, yax, riv_width, fill_value = inun_lib.gdal_readmap(
options.riv_width_file, "GTiff", logging=logger
)
riv_width[riv_width == fill_value] = 0
# read river length factor file (multiplier)
xax, yax, riv_length_fact, fill_value = inun_lib.gdal_readmap(
options.riv_length_fact_file, "GTiff", logging=logger
)
riv_length_fact = np.ma.masked_where(
riv_length_fact == fill_value, riv_length_fact
)
drain_length = wflow_lib.detdrainlength(ldd_pcr, x_res, y_res)
# compute river length in each cell
riv_length = pcr.pcr2numpy(drain_length, 0) * riv_length_fact
# riv_length_pcr = pcr.numpy2pcr(pcr.Scalar, riv_length, 0)
flood_folder = os.path.join(options.dest_path, case_name)
flood_vol_map = os.path.join(
flood_folder,
"{:s}_vol.tif".format(os.path.split(options.flood_map)[1].split(".")[0]),
)
if not (os.path.isdir(flood_folder)):
os.makedirs(flood_folder)
if options.out_format == 0:
inun_file_tmp = os.path.join(flood_folder, "{:s}.tif.tmp".format(case_name))
inun_file = os.path.join(flood_folder, "{:s}.tif".format(case_name))
else:
inun_file_tmp = os.path.join(flood_folder, "{:s}.nc.tmp".format(case_name))
inun_file = os.path.join(flood_folder, "{:s}.nc".format(case_name))
hand_temp_file = os.path.join(flood_folder, "hand_temp.map")
drainage_temp_file = os.path.join(flood_folder, "drainage_temp.map")
stream_temp_file = os.path.join(flood_folder, "stream_temp.map")
flood_vol_temp_file = os.path.join(flood_folder, "flood_warp_temp.tif")
# load the data with river levels and compute the volumes
if options.file_format == 0:
# assume we need the maximum value in a NetCDF time series grid
logger.info("Reading flood from {:s} NetCDF file".format(options.flood_map))
a = nc.Dataset(options.flood_map, "r")
if options.latlon == 0:
xax = a.variables["x"][:]
yax = a.variables["y"][:]
else:
try:
xax = a.variables["lon"][:]
yax = a.variables["lat"][:]
except:
xax = a.variables["x"][:]
yax = a.variables["y"][:]
if options.time == "":
time_list = cftime.num2date(
a.variables["time"][:],
units=a.variables["time"].units,
calendar=a.variables["time"].calendar,
)
time = [time_list[len(time_list) // 2]]
else:
time = [dt.datetime.strptime(options.time, "%Y%m%d%H%M%S")]
flood_series = a.variables[options.flood_variable][:]
flood_data = flood_series.max(axis=0)
if np.ma.is_masked(flood_data):
flood = flood_data.data
flood[flood_data.mask] = 0
if yax[-1] > yax[0]:
yax = np.flipud(yax)
flood = np.flipud(flood)
a.close()
elif options.file_format == 1:
logger.info("Reading flood from {:s} PCRaster file".format(options.flood_map))
xax, yax, flood, flood_fill_value = inun_lib.gdal_readmap(
options.flood_map, "PCRaster", logging=logger
)
flood = np.ma.masked_equal(flood, flood_fill_value)
if options.time == "":
options.time = "20000101000000"
time = [dt.datetime.strptime(options.time, "%Y%m%d%H%M%S")]
flood[flood == flood_fill_value] = 0.0
# load the bankfull depths
if options.bankfull_map == "":
bankfull = np.zeros(flood.shape)
else:
if options.file_format == 0:
logger.info(
"Reading bankfull from {:s} NetCDF file".format(options.bankfull_map)
)
a = nc.Dataset(options.bankfull_map, "r")
xax = a.variables["x"][:]
yax = a.variables["y"][:]
# xax = a.variables['lon'][:]
# yax = a.variables['lat'][:]
bankfull_series = a.variables[options.flood_variable][:]
bankfull_data = bankfull_series.max(axis=0)
if np.ma.is_masked(bankfull_data):
bankfull = bankfull_data.data
bankfull[bankfull_data.mask] = 0
if yax[-1] > yax[0]:
yax = np.flipud(yax)
bankfull = np.flipud(bankfull)
a.close()
elif options.file_format == 1:
logger.info(
"Reading bankfull from {:s} PCRaster file".format(options.bankfull_map)
)
xax, yax, bankfull, bankfull_fill_value = inun_lib.gdal_readmap(
options.bankfull_map, "PCRaster", logging=logger
)
bankfull = np.ma.masked_equal(bankfull, bankfull_fill_value)
# flood = bankfull*2
# res_x = 2000
# res_y = 2000
# subtract the bankfull water level to get flood levels (above bankfull)
flood_vol = np.maximum(flood - bankfull, 0)
if options.flood_volume_type == 0:
flood_vol_m = (
riv_length * riv_width * flood_vol / cell_surface_wflow
) # volume expressed in meters water disc
flood_vol_m_pcr = pcr.numpy2pcr(pcr.Scalar, flood_vol_m, 0)
else:
flood_vol_m = flood_vol / cell_surface_wflow
flood_vol_m_data = flood_vol_m.data
flood_vol_m_data[flood_vol_m.mask] = -999.0
logger.info("Saving water layer map to {:s}".format(flood_vol_map))
# write to a tiff file
inun_lib.gdal_writemap(
flood_vol_map,
"GTiff",
xax,
yax,
np.maximum(flood_vol_m_data, 0),
-999.0,
logging=logger,
)
# this is placed later in the hand loop
# ds_hand, rasterband_hand = inun_lib.get_gdal_rasterband(hand_file)
ds_ldd, rasterband_ldd = inun_lib.get_gdal_rasterband(options.ldd_file)
ds_stream, rasterband_stream = inun_lib.get_gdal_rasterband(options.stream_file)
logger.info("Preparing flood map in {:s} ...please wait...".format(inun_file))
if options.out_format == 0:
ds_inun, band_inun = inun_lib.prepare_gdal(
inun_file_tmp, x, y, logging=logger, srs=srs
)
# band_inun = ds_inun.GetRasterBand(1)
else:
ds_inun, band_inun = inun_lib.prepare_nc(
inun_file_tmp,
time,
x,
np.flipud(y),
metadata=metadata_global,
metadata_var=metadata_var,
logging=logger,
)
# loop over all the tiles
n = 0
for x_loop in range(0, len(x), options.x_tile):
x_start = np.maximum(x_loop, 0)
x_end = np.minimum(x_loop + options.x_tile, len(x))
# determine actual overlap for cutting
for y_loop in range(0, len(y), options.y_tile):
x_overlap_min = x_start - np.maximum(x_start - options.x_overlap, 0)
x_overlap_max = np.minimum(x_end + options.x_overlap, len(x)) - x_end
n += 1
# print('tile {:001d}:'.format(n))
y_start = np.maximum(y_loop, 0)
y_end = np.minimum(y_loop + options.y_tile, len(y))
y_overlap_min = y_start - np.maximum(y_start - options.y_overlap, 0)
y_overlap_max = np.minimum(y_end + options.y_overlap, len(y)) - y_end
x_tile_ax = x[x_start - x_overlap_min : x_end + x_overlap_max]
y_tile_ax = y[y_start - y_overlap_min : y_end + y_overlap_max]
# cut out DEM
logger.debug(
"handling xmin: {:d} xmax: {:d} ymin {:d} ymax {:d}".format(
x_start, x_end, y_start, y_end
)
)
drainage = rasterband_ldd.ReadAsArray(
float(x_start - x_overlap_min),
float(y_start - y_overlap_min),
int((x_end + x_overlap_max) - (x_start - x_overlap_min)),
int((y_end + y_overlap_max) - (y_start - y_overlap_min)),
)
stream = rasterband_stream.ReadAsArray(
float(x_start - x_overlap_min),
float(y_start - y_overlap_min),
int((x_end + x_overlap_max) - (x_start - x_overlap_min)),
int((y_end + y_overlap_max) - (y_start - y_overlap_min)),
)
# stream_max = np.minimum(stream.max(), options.max_strahler)
inun_lib.gdal_writemap(
drainage_temp_file,
"PCRaster",
x_tile_ax,
y_tile_ax,
drainage,
rasterband_ldd.GetNoDataValue(),
gdal_type=gdal.GDT_Int32,
logging=logger,
)
inun_lib.gdal_writemap(
stream_temp_file,
"PCRaster",
x_tile_ax,
y_tile_ax,
stream,
rasterband_stream.GetNoDataValue(),
gdal_type=gdal.GDT_Int32,
logging=logger,
)
# read as pcr objects
pcr.setclone(stream_temp_file)
drainage_pcr = pcr.lddrepair(
pcr.ldd(pcr.readmap(drainage_temp_file))
) # convert to ldd type map
stream_pcr = pcr.scalar(
pcr.readmap(stream_temp_file)
) # convert to ldd type map
# warp of flood volume to inundation resolution
inun_lib.gdal_warp(
flood_vol_map,
stream_temp_file,
flood_vol_temp_file,
gdal_interp=gdalconst.GRA_NearestNeighbour,
) # ,
x_tile_ax, y_tile_ax, flood_meter, fill_value = inun_lib.gdal_readmap(
flood_vol_temp_file, "GTiff", logging=logger
)
# make sure that the option unittrue is on !! (if unitcell was is used in another function)
x_res_tile, y_res_tile, reallength = pcrut.detRealCellLength(
pcr.scalar(stream_pcr), not (bool(options.latlon))
)
cell_surface_tile = pcr.pcr2numpy(x_res_tile * y_res_tile, 0)
# convert meter depth to volume [m3]
flood_vol = pcr.numpy2pcr(
pcr.Scalar, flood_meter * cell_surface_tile, fill_value
)
# first prepare a basin map, belonging to the lowest order we are looking at
inundation_pcr = pcr.scalar(stream_pcr) * 0
for hand_strahler in range(options.catchment_strahler, stream_max + 1, 1):
# hand_temp_file = os.path.join(flood_folder, 'hand_temp.map')
if os.path.isfile(
os.path.join(
options.dest_path,
"{:s}_hand_strahler_{:02d}.tif".format(dem_name, hand_strahler),
)
):
hand_file = os.path.join(
options.dest_path,
"{:s}_hand_strahler_{:02d}.tif".format(dem_name, hand_strahler),
)
else:
hand_file = "{:s}_{:02d}.tif".format(
options.hand_file_prefix, hand_strahler
)
ds_hand, rasterband_hand = inun_lib.get_gdal_rasterband(hand_file)
hand = rasterband_hand.ReadAsArray(
float(x_start - x_overlap_min),
float(y_start - y_overlap_min),
int((x_end + x_overlap_max) - (x_start - x_overlap_min)),
int((y_end + y_overlap_max) - (y_start - y_overlap_min)),
)
print(
(
"len x-ax: {:d} len y-ax {:d} x-shape {:d} y-shape {:d}".format(
len(x_tile_ax), len(y_tile_ax), hand.shape[1], hand.shape[0]
)
)
)
inun_lib.gdal_writemap(
hand_temp_file,
"PCRaster",
x_tile_ax,
y_tile_ax,
hand,
rasterband_hand.GetNoDataValue(),
gdal_type=gdal.GDT_Float32,
logging=logger,
)
hand_pcr = pcr.readmap(hand_temp_file)
stream_ge_hand, subcatch_hand = inun_lib.subcatch_stream(
drainage_pcr, options.catchment_strahler, stream=stream_pcr
)
# stream_ge_hand, subcatch_hand = inun_lib.subcatch_stream(drainage_pcr, hand_strahler, stream=stream_pcr)
stream_ge, subcatch = inun_lib.subcatch_stream(
drainage_pcr,
options.catchment_strahler,
stream=stream_pcr,
basin=pcr.boolean(pcr.cover(subcatch_hand, 0)),
assign_existing=True,
min_strahler=hand_strahler,
max_strahler=hand_strahler,
) # generate subcatchments, only within basin for HAND
flood_vol_strahler = pcr.ifthenelse(
pcr.boolean(pcr.cover(subcatch, 0)), flood_vol, 0
) # mask the flood volume map with the created subcatch map for strahler order = hand_strahler
inundation_pcr_step = inun_lib.volume_spread(
drainage_pcr,
hand_pcr,
pcr.subcatchment(
drainage_pcr, subcatch
), # to make sure backwater effects can occur from higher order rivers to lower order rivers
flood_vol_strahler,
volume_thres=0.0,
iterations=options.iterations,
cell_surface=pcr.numpy2pcr(pcr.Scalar, cell_surface_tile, -9999),
logging=logger,
order=hand_strahler,
neg_HAND=options.neg_HAND,
) # 1166400000.
# use maximum value of inundation_pcr_step and new inundation for higher strahler order
inundation_pcr = pcr.max(inundation_pcr, inundation_pcr_step)
inundation = pcr.pcr2numpy(inundation_pcr, -9999.0)
# cut relevant part
if y_overlap_max == 0:
y_overlap_max = -inundation.shape[0]
if x_overlap_max == 0:
x_overlap_max = -inundation.shape[1]
inundation_cut = inundation[
0 + y_overlap_min : -y_overlap_max, 0 + x_overlap_min : -x_overlap_max
]
# inundation_cut
if options.out_format == 0:
band_inun.WriteArray(inundation_cut, float(x_start), float(y_start))
band_inun.FlushCache()
else:
# with netCDF, data is up-side-down.
inun_lib.write_tile_nc(band_inun, inundation_cut, x_start, y_start)
# clean up
os.unlink(flood_vol_temp_file)
os.unlink(drainage_temp_file)
os.unlink(hand_temp_file)
os.unlink(
stream_temp_file
) # also remove temp stream file from output folder
# if n == 35:
# band_inun.SetNoDataValue(-9999.)
# ds_inun = None
# sys.exit(0)
# os.unlink(flood_vol_map)
logger.info("Finalizing {:s}".format(inun_file))
# add the metadata to the file and band
# band_inun.SetNoDataValue(-9999.)
# ds_inun.SetMetadata(metadata_global)
# band_inun.SetMetadata(metadata_var)
if options.out_format == 0:
ds_inun = None
ds_hand = None
else:
ds_inun.close()
ds_ldd = None
# rename temporary file to final hand file
if os.path.isfile(inun_file):
# remove an old result if available
os.unlink(inun_file)
os.rename(inun_file_tmp, inun_file)
logger.info("Done! Thank you for using hand_contour_inun.py")
logger, ch = inun_lib.closeLogger(logger, ch)
del logger, ch
sys.exit(0)
def __init__(self, netcdffile, logging, vars=[]):
"""
First try to setup a class read netcdf files
(converted with pcr2netcdf.py)
netcdffile: file to read the forcing data from
logging: python logging object
vars: list of variables to get from file
"""
self.fname = netcdffile
if os.path.exists(netcdffile):
self.dataset = netCDF4.Dataset(netcdffile, mode="r")
else:
msg = os.path.abspath(netcdffile) + " not found!"
logging.error(msg)
raise ValueError(msg)
logging.info("Reading state input from netCDF file: " + netcdffile)
self.alldat = {}
a = pcr.pcr2numpy(pcr.cover(0.0), 0.0).flatten()
# Determine steps to load in mem based on estimated memory usage
floatspermb = 1048576 / 4
maxmb = 40
self.maxsteps = maxmb * len(a) / floatspermb + 1
self.fstep = 0
self.lstep = self.fstep + self.maxsteps
self.datetime = self.dataset.variables["time"][:]
if hasattr(self.dataset.variables["time"], "units"):
self.timeunits = self.dataset.variables["time"].units
else:
self.timeunits = "Seconds since 1970-01-01 00:00:00"
if hasattr(self.dataset.variables["time"], "calendar"):
self.calendar = self.dataset.variables["time"].calendar
else:
self.calendar = "gregorian"
self.datetimelist = cftime.num2date(
self.datetime, self.timeunits, calendar=self.calendar
)
try:
self.x = self.dataset.variables["x"][:]
except:
self.x = self.dataset.variables["lon"][:]
# Now check Y values to see if we must flip the data
try:
self.y = self.dataset.variables["y"][:]
except:
self.y = self.dataset.variables["lat"][:]
# test if 1D or 2D array
if len(self.y.shape) == 1:
if self.y[0] > self.y[-1]:
self.flip = False
else:
self.flip = True
else: # not sure if this works
self.y = self.y[:][0]
if self.y[0] > self.y[-1]:
self.flip = False
else:
self.flip = True
x = pcr.pcr2numpy(pcr.xcoordinate(pcr.boolean(pcr.cover(1.0))), np.nan)[0, :]
y = pcr.pcr2numpy(pcr.ycoordinate(pcr.boolean(pcr.cover(1.0))), np.nan)[:, 0]
# Get average cell size
acc = (
np.diff(x).mean() * 0.25
) # non-exact match needed becuase of possible rounding problems
if self.flip:
(self.latidx,) = np.logical_and(
self.y[::-1] + acc >= y.min(), self.y[::-1] <= y.max() + acc
).nonzero()
(self.lonidx,) = np.logical_and(
self.x + acc >= x.min(), self.x <= x.max() + acc
).nonzero()
else:
(self.latidx,) = np.logical_and(
self.y + acc >= y.min(), self.y <= y.max() + acc
).nonzero()
(self.lonidx,) = np.logical_and(
self.x + acc >= x.min(), self.x <= x.max() + acc
).nonzero()
if len(self.lonidx) != len(x):
logging.error("error in determining X coordinates in netcdf...")
logging.error("model expects: " + str(x.min()) + " to " + str(x.max()))
logging.error(
"got coordinates netcdf: "
+ str(self.x.min())
+ " to "
+ str(self.x.max())
)
logging.error(
"got len from netcdf x: "
+ str(len(x))
+ " expected "
+ str(len(self.lonidx))
)
raise ValueError("X coordinates in netcdf do not match model")
if len(self.latidx) != len(y):
logging.error("error in determining Y coordinates in netcdf...")
logging.error("model expects: " + str(y.min()) + " to " + str(y.max()))
logging.error(
"got from netcdf: " + str(self.y.min()) + " to " + str(self.y.max())
)
logging.error(
"got len from netcdf y: "
+ str(len(y))
+ " expected "
+ str(len(self.latidx))
)
raise ValueError("Y coordinates in netcdf do not match model")
for var in vars:
try:
self.alldat[var] = self.dataset.variables[var][
self.fstep : self.maxsteps
]
except:
self.alldat.pop(var, None)
logging.warning(
"Variable " + var + " not found in netcdf file: " + netcdffile
)
def times(calendar):
import cftime
return cftime.num2date(
np.arange(_NT), units='hours since 2000-01-01', calendar=calendar,
only_use_cftime_datetimes=True)
