def test_extrapolate(): # Arrange ## Dataset 1 nt = 2 data = [np.zeros([nt, 10, 3])] time = list(rrule(freq=DAILY, count=nt, dtstart=datetime(2000, 1, 1))) items = [ItemInfo("Foo")] ds1 = Dataset(data, time, items) assert ds1.data[0].shape == (nt, 10, 3) ## Dataset 2 partly overlapping with Dataset 1 nt = 3 data = [np.ones([nt, 10, 3])] time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 2))) items = [ItemInfo("Foo")] ds2 = Dataset(data, time, items) # Act ## interp dsi = ds1.interp_time(dt=ds2.time, fill_value=1.0) # Assert assert dsi.time[0] == ds2.time[0] assert dsi.time[-1] == ds2.time[-1] assert len(dsi.time) == len(ds2.time) assert dsi.data[0][0] == pytest.approx(0.0) assert dsi.data[0][1] == pytest.approx(1.0) # filled assert dsi.data[0][2] == pytest.approx(1.0) # filled
def test_write_timestep_7days(tmpdir): filename = os.path.join(tmpdir.dirname, "random.dfs0") data = [] nt = 10 d1 = np.zeros(nt) data.append(d1) d2 = np.ones(nt) data.append(d2) items = [ItemInfo("Zeros"), ItemInfo("Ones")] dfs = Dfs0() dfs.write( filename=filename, data=data, items=items, title="Zeros and ones", timeseries_unit=TimeStep.DAY, dt=7, ) assert os.path.exists(filename) newdfs = Dfs0(filename) res = newdfs.read() dt = res.time[1] - res.time[0] assert dt == timedelta(days=7) assert res["Zeros"][-1] == 0.0 assert res["Ones"][-1] == 1.0
def test_write_from_data_frame_different_types(tmpdir): df = pd.read_csv( "tests/testdata/co2-mm-mlo.csv", parse_dates=True, index_col="Date", na_values=-99.99, ) df = df[["Average", "Trend"]] filename = os.path.join(tmpdir.dirname, "dataframe.dfs0") items = [ ItemInfo("Average", EUMType.Concentration, EUMUnit.gram_per_meter_pow_3), ItemInfo("Trend", EUMType.Undefined), ] Dfs0.from_dataframe(df, filename, items=items) ds = mikeio.read(filename) assert len(ds.items) == 2 assert ds.items[0].type == EUMType.Concentration assert ds.items[0].unit == EUMUnit.gram_per_meter_pow_3 assert ds.items[1].type == EUMType.Undefined assert ds.items[1].unit == EUMUnit.undefined
def test_temporal_subset_fancy(): nt = (24 * 31) + 1 d1 = np.zeros([nt, 100, 30]) + 1.5 d2 = np.zeros([nt, 100, 30]) + 2.0 data = [d1, d2] time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 1))) items = [ItemInfo("Foo"), ItemInfo("Bar")] ds = Dataset(data, time, items) assert ds.time[0].hour == 0 assert ds.time[-1].hour == 0 selds = ds["2000-01-01 00:00":"2000-01-02 00:00"] assert len(selds) == 2 assert selds["Foo"].shape == (25, 100, 30) selds = ds[:"2000-01-02 00:00"] assert selds["Foo"].shape == (25, 100, 30) selds = ds["2000-01-31 00:00":] assert selds["Foo"].shape == (25, 100, 30) selds = ds["2000-01-30":] assert selds["Foo"].shape == (49, 100, 30)
def test_write_equidistant_calendar(tmpdir): dfs0file = os.path.join(tmpdir.dirname, "random.dfs0") d1 = np.random.random([1000]) d2 = np.random.random([1000]) data = [] data.append(d1) data.append(d2) start_time = datetime.datetime(2017, 1, 1) timeseries_unit = 1402 title = "Hello Test" items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)] data_value_type = [0, 1] # TODO add data_value_type to ItemInfo dt = 5 dfs = Dfs0() dfs.write( filename=dfs0file, data=data, start_time=start_time, timeseries_unit=timeseries_unit, dt=dt, items=items, title=title, data_value_type=data_value_type, )
def test_write_non_equidistant_calendar(tmpdir): dfs0file = os.path.join(tmpdir.dirname, "neq.dfs0") d1 = np.zeros(1000) d2 = np.ones([1000]) data = [] data.append(d1) data.append(d2) start_time = datetime.datetime(2017, 1, 1) time_vector = [] for i in range(1000): time_vector.append(start_time + datetime.timedelta(hours=i * 0.1)) title = "Hello Test" items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)] data_value_type = [0, 1] dfs = Dfs0() dfs.write( filename=dfs0file, data=data, datetimes=time_vector, items=items, title=title, data_value_type=data_value_type, ) assert os.path.exists(dfs0file)
def test_create_timestep_7days(): dfs0File = r"zeros_ones.dfs0" data = [] nt = 10 d1 = np.zeros(nt) data.append(d1) d2 = np.ones(nt) data.append(d2) items = [ItemInfo("Zeros"), ItemInfo("Ones")] dfs = Dfs0() dfs.create( filename=dfs0File, data=data, items=items, title="Zeros and ones", timeseries_unit=TimeStep.DAY, dt=7, ) assert True res = dfs.read(dfs0File) dt = res.time[1] - res.time[0] assert dt == timedelta(days=7) os.remove(dfs0File)
def test_create_non_equidistant_calendar(): dfs0file = r"neq.dfs0" d1 = np.random.random([1000]) d2 = np.random.random([1000]) data = [] data.append(d1) data.append(d2) start_time = datetime.datetime(2017, 1, 1) time_vector = [] for i in range(1000): time_vector.append(start_time + datetime.timedelta(hours=i * 0.1)) title = "Hello Test" items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)] data_value_type = [0, 1] dfs = Dfs0() dfs.create( filename=dfs0file, data=data, datetimes=time_vector, items=items, title=title, data_value_type=data_value_type, ) assert True os.remove(dfs0file)
def test_create_equidistant_calendar(): dfs0file = r"random.dfs0" d1 = np.random.random([1000]) d2 = np.random.random([1000]) data = [] data.append(d1) data.append(d2) start_time = datetime.datetime(2017, 1, 1) timeseries_unit = 1402 title = "Hello Test" items = [ItemInfo("VarFun01", 100000, 1000), ItemInfo("NotFun", 100000, 1000)] data_value_type = [0, 1] # TODO add data_value_type to ItemInfo dt = 5 dfs = Dfs0() dfs.create( filename=dfs0file, data=data, start_time=start_time, timeseries_unit=timeseries_unit, dt=dt, items=items, title=title, data_value_type=data_value_type, ) os.remove(dfs0file) assert True
def test_int_is_valid_type_info(): item = ItemInfo("Foo", 100123) assert item.type == EUMType.Viscosity item = ItemInfo("U", 100002) assert item.type == EUMType.Wind_Velocity
def test_interp_time_to_other_dataset(): # Arrange ## Dataset 1 nt = 4 data = [np.zeros([nt, 10, 3])] time = list(rrule(freq=DAILY, count=nt, dtstart=datetime(2000, 1, 1))) items = [ItemInfo("Foo")] ds1 = Dataset(data, time, items) assert ds1.data[0].shape == (nt, 10, 3) ## Dataset 2 nt = 12 data = [np.ones([nt, 10, 3])] time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 1))) items = [ItemInfo("Foo")] ds2 = Dataset(data, time, items) # Act ## interp dsi = ds1.interp_time(dt=ds2.time) # Assert assert dsi.time[0] == ds2.time[0] assert dsi.time[-1] == ds2.time[-1] assert len(dsi.time) == len(ds2.time) assert dsi.data[0].shape[0] == ds2.data[0].shape[0] # Accept dataset as argument dsi2 = ds1.interp_time(ds2) assert dsi2.time[0] == ds2.time[0]
def test_items_data_mismatch(): nt = 100 d = np.zeros([nt, 100, 30]) + 1.0 time = _get_time(nt) items = [ItemInfo("Foo"), ItemInfo("Bar")] # Two items is not correct! with pytest.raises(ValueError): Dataset([d], time, items)
def test_default_name_from_type(): item = ItemInfo(EUMType.Current_Speed) assert item.name == "Current Speed" assert item.unit == EUMUnit.meter_per_sec item2 = ItemInfo(EUMType.Current_Direction, EUMUnit.degree) assert item2.unit == EUMUnit.degree item3 = ItemInfo("Current direction (going to)", EUMType.Current_Direction, EUMUnit.degree) assert item3.type == EUMType.Current_Direction assert item3.unit == EUMUnit.degree
def test_subset_with_datetime_is_not_supported(): nt = (24 * 31) + 1 d1 = np.zeros([nt, 100, 30]) + 1.5 d2 = np.zeros([nt, 100, 30]) + 2.0 data = [d1, d2] time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 1))) items = [ItemInfo("Foo"), ItemInfo("Bar")] ds = Dataset(data, time, items) with pytest.raises(ValueError): ds[datetime(2000, 1, 1)]
def test_select_item_by_name(): nt = 100 d1 = np.zeros([nt, 100, 30]) + 1.5 d2 = np.zeros([nt, 100, 30]) + 2.0 d1[0, 10, :] = 2.0 d2[0, 10, :] = 3.0 data = [d1, d2] time = _get_time(nt) items = [ItemInfo("Foo"), ItemInfo("Bar")] ds = Dataset(data, time, items) foo_data = ds["Foo"] assert foo_data[0, 10, 0] == 2.0
def test_to_dataframe(): nt = 100 d1 = np.zeros([nt]) d2 = np.zeros([nt]) data = [d1, d2] time = _get_time(nt) items = [ItemInfo("Foo"), ItemInfo("Bar")] ds = Dataset(data, time, items) df = ds.to_dataframe() assert list(df.columns) == ["Foo", "Bar"] assert isinstance(df.index, pd.DatetimeIndex)
def __init__( self, data: Union[List[np.ndarray], float], time: Union[pd.DatetimeIndex, str], items: Union[List[ItemInfo], List[EUMType]], ): self._deletevalue = Dataset.deletevalue if isinstance(time, str): # default single-step time time = self.create_time(time) if isinstance(items, int): # default Undefined items n_items = items items = [] for j in range(n_items): items.append(ItemInfo(f"Item {j+1}")) if np.isscalar(data): # create empty dataset n_elements = data n_items = len(items) n_timesteps = len(time) data = self.create_empty_data( n_items=n_items, n_timesteps=n_timesteps, n_elements=n_elements ) n_items = len(data) n_timesteps = data[0].shape[0] if len(time) != n_timesteps: raise ValueError( f"Number of timesteps in time {len(time)} doesn't match the data {n_timesteps}." ) if len(items) != n_items: raise ValueError( f"Number of items in iteminfo {len(items)} doesn't match the data {n_items}." ) self.data = data self.time = pd.DatetimeIndex(time, freq="infer") for i, item in enumerate(items): if isinstance(item, EUMType): items[i] = ItemInfo(item) self._items = items
def test_create_multiple_item(): start_time = datetime.datetime(2012, 1, 1) # timeseries_unit = second=1400, minute=1401, hour=1402, day=1403, month=1405, year= 1404 timeseries_unit = 1402 dt = 12 # TODO change int to enum items = [ ItemInfo("testing water level", 100000, 1000), ItemInfo("testing rainfall", 100004, 1002), ItemInfo("testing drain time constant", 100362, 2605), ] filename = r"multiple.dfs2" data = [] d = np.zeros([100, 100, 30]) + 1.0 data.append(d) d = np.zeros([100, 100, 30]) + 2.0 data.append(d) d = np.zeros([100, 100, 30]) + 3.0 data.append(d) coordinate = ["LONG/LAT", 12.4387, 55.2257, 0] length_x = 0.1 length_y = 0.1 title = "test dfs2" dfs = Dfs2() dfs.create( filename=filename, data=data, start_time=start_time, timeseries_unit=timeseries_unit, dt=dt, items=items, coordinate=coordinate, length_x=length_x, length_y=length_y, title=title, ) assert True os.remove(filename)
def test_write_accumulated_datatype(tmpdir): filename = os.path.join(tmpdir.dirname, "simple.dfs2") data = [] d = np.random.random([100, 2, 3]) data.append(d) dfs = Dfs2() dfs.write( filename=filename, data=data, start_time=datetime.datetime(2012, 1, 1), dt=12, items=[ ItemInfo( "testing water level", EUMType.Water_Level, EUMUnit.meter, data_value_type="MeanStepBackward", ) ], title="test dfs2", ) newdfs = Dfs2(filename) assert newdfs.items[0].data_value_type == 3
def test_default_unit_from_type(): item = ItemInfo("Foo", EUMType.Water_Level) assert item.type == EUMType.Water_Level assert item.unit == EUMUnit.meter assert repr(item.unit) == "meter" item = ItemInfo("Tp", EUMType.Wave_period) assert item.type == EUMType.Wave_period assert item.unit == EUMUnit.second assert repr(item.unit) == "second" item = ItemInfo("Temperature", EUMType.Temperature) assert item.type == EUMType.Temperature assert item.unit == EUMUnit.degree_Celsius assert repr(item.unit) == "degree Celsius"
def test_write_from_data_frame_monkey_patched_data_value_not_default(tmpdir): df = pd.read_csv( "tests/testdata/co2-mm-mlo.csv", parse_dates=True, index_col="Date", na_values=-99.99, ) filename = os.path.join(tmpdir.dirname, "dataframe.dfs0") items = [] for col in df.columns: items.append( ItemInfo(col, EUMType.Concentration, EUMUnit.gram_per_meter_pow_3, data_value_type="MeanStepBackward")) df.to_dfs0(filename, items=items) ds = mikeio.read(filename) assert len(ds.items) == 5 assert ds.items[0].type == EUMType.Concentration assert ds.items[0].unit == EUMUnit.gram_per_meter_pow_3 assert np.isnan(ds["Average"][3]) assert ds.time[0].year == 1958 assert ds.items[0].data_value_type == 3
def test_write_single_item(tmpdir): outfilename = os.path.join(tmpdir.dirname, "simple.dfs3") start_time = datetime(2012, 1, 1) items = [ItemInfo(EUMType.Relative_moisture_content)] data = [] # t , z, y, x d = np.random.random([20, 2, 5, 10]) d[:, 0, 0, 0] = 0.0 data.append(d) title = "test dfs3" dfs = Dfs3() dfs.write( filename=outfilename, data=data, start_time=start_time, timeseries_unit=TimeStep.SECOND, dt=3600.0, items=items, coordinate=["UTM-33", 450000, 560000, 0], length_x=0.1, length_y=0.1, length_z=10.0, title=title, )
def test_write_big_file(tmpdir): outfilename = os.path.join(tmpdir.dirname, "big.dfsu") meshfilename = os.path.join("tests", "testdata", "odense_rough.mesh") msh = Mesh(meshfilename) n_elements = msh.n_elements dfs = Dfsu(meshfilename) nt = 1000 n_items = 10 items = [ItemInfo(f"Item {i+1}") for i in range(n_items)] # with dfs.write(outfilename, [], items=items, keep_open=True) as f: with dfs.write_header(outfilename, start_time=datetime(2000, 1, 1), dt=3600, items=items) as f: for i in range(nt): data = [] for i in range(n_items): d = np.random.random((1, n_elements)) data.append(d) f.append(data) dfsu = Dfsu(outfilename) assert dfsu.n_items == n_items assert dfsu.n_timesteps == nt assert dfsu.start_time.year == 2000
def test_write_single_item(tmpdir): filename = os.path.join(tmpdir.dirname, "random.dfs1") data = [] d = np.random.random([100, 3]) data.append(d) items = [ ItemInfo("testing water level", EUMType.Water_Level, EUMUnit.meter) ] title = "test dfs1" dfs = Dfs1() dfs.write( filename=filename, data=data, start_time=datetime.datetime(2012, 1, 1), dt=12, dx=100, items=items, title=title, ) assert True
def test_select_subset_isel_multiple_idxs(): nt = 100 d1 = np.zeros([nt, 100, 30]) + 1.5 d2 = np.zeros([nt, 100, 30]) + 2.0 data = [d1, d2] time = _get_time(nt) items = [ItemInfo("Foo"), ItemInfo("Bar")] ds = Dataset(data, time, items) selds = ds.isel([10, 15], axis=1) assert len(selds.items) == 2 assert len(selds.data) == 2 assert selds["Foo"].shape == (100, 2, 30)
def test_select_temporal_subset_by_idx(): nt = 100 d1 = np.zeros([nt, 100, 30]) + 1.5 d2 = np.zeros([nt, 100, 30]) + 2.0 d1[0, 10, :] = 2.0 d2[0, 10, :] = 3.0 data = [d1, d2] time = _get_time(nt) items = [ItemInfo("Foo"), ItemInfo("Bar")] ds = Dataset(data, time, items) selds = ds.isel([0, 1, 2], axis=0) assert len(selds) == 2 assert selds["Foo"].shape == (3, 100, 30)
def test_time_data_mismatch(): nt = 100 d = np.zeros([nt, 100, 30]) + 1.0 time = _get_time(nt + 1) # 101 timesteps is not correct! items = [ItemInfo("Foo")] with pytest.raises(ValueError): Dataset([d], time, items)
def test_extrapolate_not_allowed(): ## Dataset 1 nt = 2 data = [np.zeros([nt, 10, 3])] time = list(rrule(freq=DAILY, count=nt, dtstart=datetime(2000, 1, 1))) items = [ItemInfo("Foo")] ds1 = Dataset(data, time, items) assert ds1.data[0].shape == (nt, 10, 3) ## Dataset 2 partly overlapping with Dataset 1 nt = 3 data = [np.ones([nt, 10, 3])] time = list(rrule(freq=HOURLY, count=nt, dtstart=datetime(2000, 1, 2))) items = [ItemInfo("Foo")] ds2 = Dataset(data, time, items) with pytest.raises(ValueError): dsi = ds1.interp_time(dt=ds2.time, fill_value=1.0, extrapolate=False)
def test_multiple_write(tmpdir): filename = os.path.join(tmpdir.dirname, "random.dfs0") data = [] nt = 10 d1 = np.zeros(nt) data.append(d1) d2 = np.ones(nt) data.append(d2) items = [ItemInfo("Zeros"), ItemInfo("Ones")] dfs = Dfs0() dfs.write(filename=filename, data=data, items=items, title="Zeros and ones") assert os.path.exists(filename)
def test_get_bad_name(): nt = 100 data = [] d = np.zeros([100, 100, 30]) + 1.0 data.append(d) time = _get_time(nt) items = [ItemInfo("Foo")] ds = Dataset(data, time, items) with pytest.raises(Exception): ds["BAR"]