def test_extrapolate():
# Arrange
## Dataset 1
nt = 2
data = [np.zeros([nt, 10, 3])]
time = list(rrule(freq=DAILY, count=nt, dtstart=datetime(2000, 1, 1)))
items = [ItemInfo("Foo")]
ds1 = Dataset(data, time, items)
assert ds1.data[0].shape == (nt, 10, 3)
## Dataset 2 partly overlapping with Dataset 1
nt = 3
data = [np.ones([nt, 10, 3])]
time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 2)))
items = [ItemInfo("Foo")]
ds2 = Dataset(data, time, items)
# Act
## interp
dsi = ds1.interp_time(dt=ds2.time, fill_value=1.0)
# Assert
assert dsi.time[0] == ds2.time[0]
assert dsi.time[-1] == ds2.time[-1]
assert len(dsi.time) == len(ds2.time)
assert dsi.data[0][0] == pytest.approx(0.0)
assert dsi.data[0][1] == pytest.approx(1.0) # filled
assert dsi.data[0][2] == pytest.approx(1.0) # filled
def test_write_timestep_7days(tmpdir):
filename = os.path.join(tmpdir.dirname, "random.dfs0")
data = []
nt = 10
d1 = np.zeros(nt)
data.append(d1)
d2 = np.ones(nt)
data.append(d2)
items = [ItemInfo("Zeros"), ItemInfo("Ones")]
dfs = Dfs0()
dfs.write(
filename=filename,
data=data,
items=items,
title="Zeros and ones",
timeseries_unit=TimeStep.DAY,
dt=7,
)
assert os.path.exists(filename)
newdfs = Dfs0(filename)
res = newdfs.read()
dt = res.time[1] - res.time[0]
assert dt == timedelta(days=7)
assert res["Zeros"][-1] == 0.0
assert res["Ones"][-1] == 1.0
def test_write_from_data_frame_different_types(tmpdir):
df = pd.read_csv(
"tests/testdata/co2-mm-mlo.csv",
parse_dates=True,
index_col="Date",
na_values=-99.99,
)
df = df[["Average", "Trend"]]
filename = os.path.join(tmpdir.dirname, "dataframe.dfs0")
items = [
ItemInfo("Average", EUMType.Concentration, EUMUnit.gram_per_meter_pow_3),
ItemInfo("Trend", EUMType.Undefined),
]
Dfs0.from_dataframe(df, filename, items=items)
ds = mikeio.read(filename)
assert len(ds.items) == 2
assert ds.items[0].type == EUMType.Concentration
assert ds.items[0].unit == EUMUnit.gram_per_meter_pow_3
assert ds.items[1].type == EUMType.Undefined
assert ds.items[1].unit == EUMUnit.undefined
def test_temporal_subset_fancy():
nt = (24 * 31) + 1
d1 = np.zeros([nt, 100, 30]) + 1.5
d2 = np.zeros([nt, 100, 30]) + 2.0
data = [d1, d2]
time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 1)))
items = [ItemInfo("Foo"), ItemInfo("Bar")]
ds = Dataset(data, time, items)
assert ds.time[0].hour == 0
assert ds.time[-1].hour == 0
selds = ds["2000-01-01 00:00":"2000-01-02 00:00"]
assert len(selds) == 2
assert selds["Foo"].shape == (25, 100, 30)
selds = ds[:"2000-01-02 00:00"]
assert selds["Foo"].shape == (25, 100, 30)
selds = ds["2000-01-31 00:00":]
assert selds["Foo"].shape == (25, 100, 30)
selds = ds["2000-01-30":]
assert selds["Foo"].shape == (49, 100, 30)
def test_write_equidistant_calendar(tmpdir):
dfs0file = os.path.join(tmpdir.dirname, "random.dfs0")
d1 = np.random.random([1000])
d2 = np.random.random([1000])
data = []
data.append(d1)
data.append(d2)
start_time = datetime.datetime(2017, 1, 1)
timeseries_unit = 1402
title = "Hello Test"
items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)]
data_value_type = [0, 1] # TODO add data_value_type to ItemInfo
dt = 5
dfs = Dfs0()
dfs.write(
filename=dfs0file,
data=data,
start_time=start_time,
timeseries_unit=timeseries_unit,
dt=dt,
items=items,
title=title,
data_value_type=data_value_type,
)
def test_write_non_equidistant_calendar(tmpdir):
dfs0file = os.path.join(tmpdir.dirname, "neq.dfs0")
d1 = np.zeros(1000)
d2 = np.ones([1000])
data = []
data.append(d1)
data.append(d2)
start_time = datetime.datetime(2017, 1, 1)
time_vector = []
for i in range(1000):
time_vector.append(start_time + datetime.timedelta(hours=i * 0.1))
title = "Hello Test"
items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)]
data_value_type = [0, 1]
dfs = Dfs0()
dfs.write(
filename=dfs0file,
data=data,
datetimes=time_vector,
items=items,
title=title,
data_value_type=data_value_type,
)
assert os.path.exists(dfs0file)
def test_create_timestep_7days():
dfs0File = r"zeros_ones.dfs0"
data = []
nt = 10
d1 = np.zeros(nt)
data.append(d1)
d2 = np.ones(nt)
data.append(d2)
items = [ItemInfo("Zeros"), ItemInfo("Ones")]
dfs = Dfs0()
dfs.create(
filename=dfs0File,
data=data,
items=items,
title="Zeros and ones",
timeseries_unit=TimeStep.DAY,
dt=7,
)
assert True
res = dfs.read(dfs0File)
dt = res.time[1] - res.time[0]
assert dt == timedelta(days=7)
os.remove(dfs0File)
def test_create_non_equidistant_calendar():
dfs0file = r"neq.dfs0"
d1 = np.random.random([1000])
d2 = np.random.random([1000])
data = []
data.append(d1)
data.append(d2)
start_time = datetime.datetime(2017, 1, 1)
time_vector = []
for i in range(1000):
time_vector.append(start_time + datetime.timedelta(hours=i * 0.1))
title = "Hello Test"
items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)]
data_value_type = [0, 1]
dfs = Dfs0()
dfs.create(
filename=dfs0file,
data=data,
datetimes=time_vector,
items=items,
title=title,
data_value_type=data_value_type,
)
assert True
os.remove(dfs0file)
def test_create_equidistant_calendar():
dfs0file = r"random.dfs0"
d1 = np.random.random([1000])
d2 = np.random.random([1000])
data = []
data.append(d1)
data.append(d2)
start_time = datetime.datetime(2017, 1, 1)
timeseries_unit = 1402
title = "Hello Test"
items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)]
data_value_type = [0, 1] # TODO add data_value_type to ItemInfo
dt = 5
dfs = Dfs0()
dfs.create(
filename=dfs0file,
data=data,
start_time=start_time,
timeseries_unit=timeseries_unit,
dt=dt,
items=items,
title=title,
data_value_type=data_value_type,
)
os.remove(dfs0file)
assert True
def test_int_is_valid_type_info():
item = ItemInfo("Foo", 100123)
assert item.type == EUMType.Viscosity
item = ItemInfo("U", 100002)
assert item.type == EUMType.Wind_Velocity
def test_interp_time_to_other_dataset():
# Arrange
## Dataset 1
nt = 4
data = [np.zeros([nt, 10, 3])]
time = list(rrule(freq=DAILY, count=nt, dtstart=datetime(2000, 1, 1)))
items = [ItemInfo("Foo")]
ds1 = Dataset(data, time, items)
assert ds1.data[0].shape == (nt, 10, 3)
## Dataset 2
nt = 12
data = [np.ones([nt, 10, 3])]
time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 1)))
items = [ItemInfo("Foo")]
ds2 = Dataset(data, time, items)
# Act
## interp
dsi = ds1.interp_time(dt=ds2.time)
# Assert
assert dsi.time[0] == ds2.time[0]
assert dsi.time[-1] == ds2.time[-1]
assert len(dsi.time) == len(ds2.time)
assert dsi.data[0].shape[0] == ds2.data[0].shape[0]
# Accept dataset as argument
dsi2 = ds1.interp_time(ds2)
assert dsi2.time[0] == ds2.time[0]
def test_items_data_mismatch():
nt = 100
d = np.zeros([nt, 100, 30]) + 1.0
time = _get_time(nt)
items = [ItemInfo("Foo"), ItemInfo("Bar")] # Two items is not correct!
with pytest.raises(ValueError):
Dataset([d], time, items)
def test_default_name_from_type():
item = ItemInfo(EUMType.Current_Speed)
assert item.name == "Current Speed"
assert item.unit == EUMUnit.meter_per_sec
item2 = ItemInfo(EUMType.Current_Direction, EUMUnit.degree)
assert item2.unit == EUMUnit.degree
item3 = ItemInfo("Current direction (going to)", EUMType.Current_Direction,
EUMUnit.degree)
assert item3.type == EUMType.Current_Direction
assert item3.unit == EUMUnit.degree
def test_subset_with_datetime_is_not_supported():
nt = (24 * 31) + 1
d1 = np.zeros([nt, 100, 30]) + 1.5
d2 = np.zeros([nt, 100, 30]) + 2.0
data = [d1, d2]
time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 1)))
items = [ItemInfo("Foo"), ItemInfo("Bar")]
ds = Dataset(data, time, items)
with pytest.raises(ValueError):
ds[datetime(2000, 1, 1)]
def test_select_item_by_name():
nt = 100
d1 = np.zeros([nt, 100, 30]) + 1.5
d2 = np.zeros([nt, 100, 30]) + 2.0
d1[0, 10, :] = 2.0
d2[0, 10, :] = 3.0
data = [d1, d2]
time = _get_time(nt)
items = [ItemInfo("Foo"), ItemInfo("Bar")]
ds = Dataset(data, time, items)
foo_data = ds["Foo"]
assert foo_data[0, 10, 0] == 2.0
def test_to_dataframe():
nt = 100
d1 = np.zeros([nt])
d2 = np.zeros([nt])
data = [d1, d2]
time = _get_time(nt)
items = [ItemInfo("Foo"), ItemInfo("Bar")]
ds = Dataset(data, time, items)
df = ds.to_dataframe()
assert list(df.columns) == ["Foo", "Bar"]
assert isinstance(df.index, pd.DatetimeIndex)
def __init__(
self,
data: Union[List[np.ndarray], float],
time: Union[pd.DatetimeIndex, str],
items: Union[List[ItemInfo], List[EUMType]],
):
self._deletevalue = Dataset.deletevalue
if isinstance(time, str):
# default single-step time
time = self.create_time(time)
if isinstance(items, int):
# default Undefined items
n_items = items
items = []
for j in range(n_items):
items.append(ItemInfo(f"Item {j+1}"))
if np.isscalar(data):
# create empty dataset
n_elements = data
n_items = len(items)
n_timesteps = len(time)
data = self.create_empty_data(
n_items=n_items, n_timesteps=n_timesteps, n_elements=n_elements
)
n_items = len(data)
n_timesteps = data[0].shape[0]
if len(time) != n_timesteps:
raise ValueError(
f"Number of timesteps in time {len(time)} doesn't match the data {n_timesteps}."
)
if len(items) != n_items:
raise ValueError(
f"Number of items in iteminfo {len(items)} doesn't match the data {n_items}."
)
self.data = data
self.time = pd.DatetimeIndex(time, freq="infer")
for i, item in enumerate(items):
if isinstance(item, EUMType):
items[i] = ItemInfo(item)
self._items = items
def test_create_multiple_item():
start_time = datetime.datetime(2012, 1, 1)
# timeseries_unit = second=1400, minute=1401, hour=1402, day=1403, month=1405, year= 1404
timeseries_unit = 1402
dt = 12
# TODO change int to enum
items = [
ItemInfo("testing water level", 100000, 1000),
ItemInfo("testing rainfall", 100004, 1002),
ItemInfo("testing drain time constant", 100362, 2605),
]
filename = r"multiple.dfs2"
data = []
d = np.zeros([100, 100, 30]) + 1.0
data.append(d)
d = np.zeros([100, 100, 30]) + 2.0
data.append(d)
d = np.zeros([100, 100, 30]) + 3.0
data.append(d)
coordinate = ["LONG/LAT", 12.4387, 55.2257, 0]
length_x = 0.1
length_y = 0.1
title = "test dfs2"
dfs = Dfs2()
dfs.create(
filename=filename,
data=data,
start_time=start_time,
timeseries_unit=timeseries_unit,
dt=dt,
items=items,
coordinate=coordinate,
length_x=length_x,
length_y=length_y,
title=title,
)
assert True
os.remove(filename)
def test_write_accumulated_datatype(tmpdir):
filename = os.path.join(tmpdir.dirname, "simple.dfs2")
data = []
d = np.random.random([100, 2, 3])
data.append(d)
dfs = Dfs2()
dfs.write(
filename=filename,
data=data,
start_time=datetime.datetime(2012, 1, 1),
dt=12,
items=[
ItemInfo(
"testing water level",
EUMType.Water_Level,
EUMUnit.meter,
data_value_type="MeanStepBackward",
)
],
title="test dfs2",
)
newdfs = Dfs2(filename)
assert newdfs.items[0].data_value_type == 3
def test_default_unit_from_type():
item = ItemInfo("Foo", EUMType.Water_Level)
assert item.type == EUMType.Water_Level
assert item.unit == EUMUnit.meter
assert repr(item.unit) == "meter"
item = ItemInfo("Tp", EUMType.Wave_period)
assert item.type == EUMType.Wave_period
assert item.unit == EUMUnit.second
assert repr(item.unit) == "second"
item = ItemInfo("Temperature", EUMType.Temperature)
assert item.type == EUMType.Temperature
assert item.unit == EUMUnit.degree_Celsius
assert repr(item.unit) == "degree Celsius"
def test_write_from_data_frame_monkey_patched_data_value_not_default(tmpdir):
df = pd.read_csv(
"tests/testdata/co2-mm-mlo.csv",
parse_dates=True,
index_col="Date",
na_values=-99.99,
)
filename = os.path.join(tmpdir.dirname, "dataframe.dfs0")
items = []
for col in df.columns:
items.append(
ItemInfo(col,
EUMType.Concentration,
EUMUnit.gram_per_meter_pow_3,
data_value_type="MeanStepBackward"))
df.to_dfs0(filename, items=items)
ds = mikeio.read(filename)
assert len(ds.items) == 5
assert ds.items[0].type == EUMType.Concentration
assert ds.items[0].unit == EUMUnit.gram_per_meter_pow_3
assert np.isnan(ds["Average"][3])
assert ds.time[0].year == 1958
assert ds.items[0].data_value_type == 3
def test_write_single_item(tmpdir):
outfilename = os.path.join(tmpdir.dirname, "simple.dfs3")
start_time = datetime(2012, 1, 1)
items = [ItemInfo(EUMType.Relative_moisture_content)]
data = []
# t , z, y, x
d = np.random.random([20, 2, 5, 10])
d[:, 0, 0, 0] = 0.0
data.append(d)
title = "test dfs3"
dfs = Dfs3()
dfs.write(
filename=outfilename,
data=data,
start_time=start_time,
timeseries_unit=TimeStep.SECOND,
dt=3600.0,
items=items,
coordinate=["UTM-33", 450000, 560000, 0],
length_x=0.1,
length_y=0.1,
length_z=10.0,
title=title,
)
def test_write_big_file(tmpdir):
outfilename = os.path.join(tmpdir.dirname, "big.dfsu")
meshfilename = os.path.join("tests", "testdata", "odense_rough.mesh")
msh = Mesh(meshfilename)
n_elements = msh.n_elements
dfs = Dfsu(meshfilename)
nt = 1000
n_items = 10
items = [ItemInfo(f"Item {i+1}") for i in range(n_items)]
# with dfs.write(outfilename, [], items=items, keep_open=True) as f:
with dfs.write_header(outfilename,
start_time=datetime(2000, 1, 1),
dt=3600,
items=items) as f:
for i in range(nt):
data = []
for i in range(n_items):
d = np.random.random((1, n_elements))
data.append(d)
f.append(data)
dfsu = Dfsu(outfilename)
assert dfsu.n_items == n_items
assert dfsu.n_timesteps == nt
assert dfsu.start_time.year == 2000
def test_write_single_item(tmpdir):
filename = os.path.join(tmpdir.dirname, "random.dfs1")
data = []
d = np.random.random([100, 3])
data.append(d)
items = [
ItemInfo("testing water level", EUMType.Water_Level, EUMUnit.meter)
]
title = "test dfs1"
dfs = Dfs1()
dfs.write(
filename=filename,
data=data,
start_time=datetime.datetime(2012, 1, 1),
dt=12,
dx=100,
items=items,
title=title,
)
assert True
def test_select_subset_isel_multiple_idxs():
nt = 100
d1 = np.zeros([nt, 100, 30]) + 1.5
d2 = np.zeros([nt, 100, 30]) + 2.0
data = [d1, d2]
time = _get_time(nt)
items = [ItemInfo("Foo"), ItemInfo("Bar")]
ds = Dataset(data, time, items)
selds = ds.isel([10, 15], axis=1)
assert len(selds.items) == 2
assert len(selds.data) == 2
assert selds["Foo"].shape == (100, 2, 30)
def test_select_temporal_subset_by_idx():
nt = 100
d1 = np.zeros([nt, 100, 30]) + 1.5
d2 = np.zeros([nt, 100, 30]) + 2.0
d1[0, 10, :] = 2.0
d2[0, 10, :] = 3.0
data = [d1, d2]
time = _get_time(nt)
items = [ItemInfo("Foo"), ItemInfo("Bar")]
ds = Dataset(data, time, items)
selds = ds.isel([0, 1, 2], axis=0)
assert len(selds) == 2
assert selds["Foo"].shape == (3, 100, 30)
def test_time_data_mismatch():
nt = 100
d = np.zeros([nt, 100, 30]) + 1.0
time = _get_time(nt + 1) # 101 timesteps is not correct!
items = [ItemInfo("Foo")]
with pytest.raises(ValueError):
Dataset([d], time, items)
def test_extrapolate_not_allowed():
## Dataset 1
nt = 2
data = [np.zeros([nt, 10, 3])]
time = list(rrule(freq=DAILY, count=nt, dtstart=datetime(2000, 1, 1)))
items = [ItemInfo("Foo")]
ds1 = Dataset(data, time, items)
assert ds1.data[0].shape == (nt, 10, 3)
## Dataset 2 partly overlapping with Dataset 1
nt = 3
data = [np.ones([nt, 10, 3])]
time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 2)))
items = [ItemInfo("Foo")]
ds2 = Dataset(data, time, items)
with pytest.raises(ValueError):
dsi = ds1.interp_time(dt=ds2.time, fill_value=1.0, extrapolate=False)
def test_multiple_write(tmpdir):
filename = os.path.join(tmpdir.dirname, "random.dfs0")
data = []
nt = 10
d1 = np.zeros(nt)
data.append(d1)
d2 = np.ones(nt)
data.append(d2)
items = [ItemInfo("Zeros"), ItemInfo("Ones")]
dfs = Dfs0()
dfs.write(filename=filename, data=data, items=items, title="Zeros and ones")
assert os.path.exists(filename)
def test_get_bad_name():
nt = 100
data = []
d = np.zeros([100, 100, 30]) + 1.0
data.append(d)
time = _get_time(nt)
items = [ItemInfo("Foo")]
ds = Dataset(data, time, items)
with pytest.raises(Exception):
ds["BAR"]
