def setUp(self): self.year97 = timetools.Date('01.11.1996') self.year98 = timetools.Date('01.11.1997') self.oneday = timetools.Period('1d') self.onehour = timetools.Period('1h') self.timegrid = timetools.Timegrid(self.year97, self.year98, self.oneday)
def timeofyear(self): """Index values, representing the time of the year. Let us reconsider one of the examples of the documentation on property |Indexer.dayofyear|: >>> from hydpy import pub >>> from hydpy import Timegrids, Timegrid >>> from hydpy.core.indextools import Indexer >>> pub.timegrids = "27.02.2005", "3.03.2005", "1d" Due to the simulation step size of one day, the index arrays calculated by properties |Indexer.dayofyear| and |Indexer.timeofyear| are identical: >>> Indexer().dayofyear array([57, 58, 60, 61]) >>> Indexer().timeofyear array([57, 58, 60, 61]) In the next example, we halve the step size: >>> pub.timegrids = "27.02.2005", "3.03.2005", "12h" Now two subsequent simulation steps associated are with the same day: >>> Indexer().dayofyear array([57, 57, 58, 58, 60, 60, 61, 61]) However, the `timeofyear` array gives the index of the respective simulation steps of the actual year: >>> Indexer().timeofyear array([114, 115, 116, 117, 120, 121, 122, 123]) Note the gap in the returned index array due to 2005 being not a leap year. """ # pylint: disable=no-self-use # pylint does not understand descriptors well enough, so far def _timeofyear(date): date = copy.deepcopy(date) date.year = 2000 return refgrid[date] refgrid = timetools.Timegrid( timetools.Date("2000.01.01"), timetools.Date("2001.01.01"), _get_timegrids(_timeofyear).stepsize, ) return _timeofyear
def test_11_din_style_second(self): self.assertEqual(self.refdate_second, timetools.Date('01.11.1996 12:30:05').datetime)
def test_10_din_style_minute(self): self.assertEqual(self.refdate_minute, timetools.Date('01.11.1996 12:30').datetime)
def test_09_din_style_day(self): self.assertEqual(self.refdate_hour, timetools.Date('01.11.1996 12').datetime)
def setUp(self): self.date = timetools.Date('01.11.1996')
def aggregate_series( series: VectorInput[float], stepsize: Literal["daily", "d", "monthly", "m"] = "monthly", aggregator: Union[str, Callable[[VectorInput[float]], float]] = "mean", subperiod: bool = True, basetime: str = "00:00", ) -> pandas.DataFrame: """Aggregate the time series on a monthly or daily basis. Often, we need some kind of aggregation before analysing deviations between simulation results and observations. Function |aggregate_series| performs such aggregation on a monthly or daily basis. You are free to specify arbitrary aggregation functions. We first show the default behaviour of function |aggregate_series|, which is to calculate monthly averages. Therefore, we first say the hydrological summer half-year 2001 to be our simulation period and define a daily simulation step size: >>> from hydpy import aggregate_series, pub, Node >>> pub.timegrids = "01.11.2000", "01.05.2001", "1d" Next, we prepare a |Node| object and assign some constantly increasing values to its `simulation` series: >>> import numpy >>> node = Node("test") >>> node.prepare_simseries() >>> sim = node.sequences.sim >>> sim.series = numpy.arange(1, 181+1) |aggregate_series| returns the data within index-sorted |pandas.Series| objects (note that the index addresses the left boundary of each time step: >>> aggregate_series(series=sim.series) series 2000-11-01 15.5 2000-12-01 46.0 2001-01-01 77.0 2001-02-01 106.5 2001-03-01 136.0 2001-04-01 166.5 The following example shows how to restrict the considered period via the |Timegrids.eval_| |Timegrid| of the |Timegrids| object available in the |pub| module and how to pass a different aggregation function: >>> pub.timegrids.eval_.dates = "2001-01-01", "2001-03-01" >>> aggregate_series(series=sim.series, aggregator=numpy.sum) series 2001-01-01 2387.0 2001-02-01 2982.0 Functions |aggregate_series| raises errors like the following for unsuitable functions: >>> def wrong(): ... return None >>> aggregate_series(series=sim.series, aggregator=wrong) Traceback (most recent call last): ... TypeError: While trying to aggregate the given series, the following \ error occurred: While trying to perform the aggregation based on method \ `wrong`, the following error occurred: wrong() takes 0 positional arguments \ but 1 was given When passing a string, |aggregate_series| queries it from |numpy|: >>> pub.timegrids.eval_.dates = "2001-01-01", "2001-02-01" >>> aggregate_series(series=sim.series, aggregator="sum") series 2001-01-01 2387.0 |aggregate_series| raises the following error when the requested function does not exist: >>> aggregate_series(series=sim.series, aggregator="Sum") Traceback (most recent call last): ... ValueError: While trying to aggregate the given series, the following \ error occurred: Module `numpy` does not provide a function named `Sum`. To prevent from wrong conclusions, |aggregate_series| generally ignores all data of incomplete intervals: >>> pub.timegrids = "2000-11-30", "2001-04-02", "1d" >>> node.prepare_simseries() >>> sim.series = numpy.arange(30, 152+1) >>> sim = node.sequences.sim >>> aggregate_series(series=sim.series, aggregator="sum") series 2000-12-01 1426.0 2001-01-01 2387.0 2001-02-01 2982.0 2001-03-01 4216.0 >>> pub.timegrids.eval_.dates = "2001-01-02", "2001-02-28" >>> aggregate_series(series=sim.series) Empty DataFrame Columns: [series] Index: [] If you want to analyse the data of the complete initialisation period independently of the state of |Timegrids.eval_|, set argument `subperiod` to |False|: >>> aggregate_series(series=sim.series, aggregator="sum", subperiod=False) series 2000-12-01 1426.0 2001-01-01 2387.0 2001-02-01 2982.0 2001-03-01 4216.0 The following example shows that even with only one missing value at the respective ends of the simulation period, |aggregate_series| does not return any result for the first (November 2000) and the last aggregation interval (April 2001): >>> pub.timegrids = "02.11.2000", "30.04.2001", "1d" >>> node.prepare_simseries() >>> sim.series = numpy.arange(2, 180+1) >>> aggregate_series(series=node.sequences.sim.series) series 2000-12-01 46.0 2001-01-01 77.0 2001-02-01 106.5 2001-03-01 136.0 Now we prepare a time-grid with an hourly simulation step size, to show some examples on daily aggregation: >>> pub.timegrids = "01.01.2000 22:00", "05.01.2000 22:00", "1h" >>> node.prepare_simseries() >>> sim = node.sequences.sim >>> sim.series = numpy.arange(1, 1+4*24) By default, function |aggregate_series| aggregates daily from 0 o'clock to 0 o'clock, which here results in a loss of the first two and the last 22 values of the entire period: >>> aggregate_series(series=sim.series, stepsize="daily") series 2000-01-02 14.5 2000-01-03 38.5 2000-01-04 62.5 If you want the aggregation to start at a different time of the day, use the `basetime` argument. In our example, starting at 22 o'clock fits the defined initialisation time grid and ensures the usage of all available data: >>> aggregate_series(series=sim.series, stepsize="daily", basetime="22:00") series 2000-01-01 22:00:00 12.5 2000-01-02 22:00:00 36.5 2000-01-03 22:00:00 60.5 2000-01-04 22:00:00 84.5 So far, the `basetime` argument works for daily aggregation only: >>> aggregate_series(series=sim.series, stepsize="monthly", basetime="22:00") Traceback (most recent call last): ... ValueError: While trying to aggregate the given series, the following \ error occurred: Use the `basetime` argument in combination with a `daily` \ aggregation step size only. |aggregate_series| does not support aggregation for simulation step sizes larger one day: >>> pub.timegrids = "01.01.2000 22:00", "05.01.2000 22:00", "1d" >>> node.prepare_simseries() >>> sim = node.sequences.sim >>> sim.series = numpy.arange(1, 1+4) >>> aggregate_series(series=sim.series, stepsize="daily") series 2000-01-02 2.0 2000-01-03 3.0 2000-01-04 4.0 >>> pub.timegrids = "01.01.2000 22:00", "05.01.2000 22:00", "2d" >>> node.prepare_simseries() >>> aggregate_series(series=node.sequences.sim.series, stepsize="daily") Traceback (most recent call last): ... ValueError: While trying to aggregate the given series, the following \ error occurred: Data aggregation is not supported for simulation step sizes \ greater one day. We are looking forward supporting other useful aggregation step sizes later: >>> pub.timegrids = "01.01.2000 22:00", "05.01.2000 22:00", "1d" >>> node.prepare_simseries() >>> aggregate_series(series=node.sequences.sim.series, stepsize="yearly") Traceback (most recent call last): ... ValueError: While trying to aggregate the given series, the following \ error occurred: Argument `stepsize` received value `yearly`, but only the \ following ones are supported: `monthly` (default) and `daily`. """ timegrids: timetools.Timegrids = hydpy.pub.timegrids if isinstance(aggregator, str): try: realaggregator = getattr(numpy, aggregator) except AttributeError: raise ValueError( f"Module `numpy` does not provide a function named " f"`{aggregator}`." ) from None else: realaggregator = aggregator tg = timegrids.eval_ if subperiod else timegrids.init if tg.stepsize > "1d": raise ValueError( "Data aggregation is not supported for simulation " "step sizes greater one day." ) if stepsize in ("d", "daily"): rule = "86400s" offset = ( timetools.Date(f"2000-01-01 {basetime}") - timetools.Date("2000-01-01") ).seconds elif basetime != "00:00": raise ValueError( "Use the `basetime` argument in combination with " "a `daily` aggregation step size only." ) elif stepsize in ("m", "monthly"): rule = "MS" offset = 0 else: raise ValueError( f"Argument `stepsize` received value `{stepsize}`, but only the " f"following ones are supported: `monthly` (default) and `daily`." ) dataframe_orig = pandas.DataFrame() idx0, idx1 = timegrids.evalindices if subperiod else timegrids.initindices dataframe_orig["series"] = numpy.asarray(series)[idx0:idx1] dataframe_orig.index = pandas.date_range( start=tg.firstdate.datetime, end=(tg.lastdate - tg.stepsize).datetime, freq=tg.stepsize.timedelta, ) resampler = dataframe_orig.resample( rule=rule, offset=f"{offset}s", ) try: dataframe_resampled = resampler.apply(lambda x: realaggregator(x.values)) except BaseException: objecttools.augment_excmessage( f"While trying to perform the aggregation based " f"on method `{realaggregator.__name__}`" ) for jdx0, date0 in enumerate(dataframe_resampled.index): if date0 >= tg.firstdate: break for jdx1, date1 in enumerate(reversed(dataframe_resampled.index)): date = timetools.Date(date1) if stepsize in ("daily", "d"): date += "1d" else: date = date.beginning_next_month if date <= tg.lastdate: jdx1 = len(dataframe_resampled) - jdx1 break # pylint: disable=undefined-loop-variable # the dataframe index above cannot be empty return dataframe_resampled[jdx0:jdx1]
def test_05_iso_style_day(self): self.assertEqual(self.refdate_hour, timetools.Date('1996.11.01 12').datetime)
def test_06_iso_style_minute(self): self.assertEqual(self.refdate_minute, timetools.Date('1996.11.01 12:30').datetime)
def test_03_os_style_minute(self): self.assertEqual(self.refdate_minute, timetools.Date('1996_11_01_12_30').datetime)
def test_03_os_style_second(self): self.assertEqual(self.refdate_second, timetools.Date('1996_11_01_12_30_05').datetime)
def setUp(self): self.earlydate = timetools.Date('01.11.1996') self.latedate = timetools.Date('01.11.1997') self.period = timetools.Period('365d')
def test_02_os_style_hour(self): self.assertEqual(self.refdate_hour, timetools.Date('1996_11_01_12').datetime)
def setUp(self): self.early1 = timetools.Date('01.11.1996') self.early2 = timetools.Date('01.11.1996') self.late = timetools.Date('01.11.1997')
def test_11_datetime_second(self): self.assertEqual(self.refdate_second, timetools.Date(self.refdate_second).datetime)
def setUp(self): self.year97 = timetools.Date('01.11.1996') self.year98 = timetools.Date('01.11.1997') self.oneday = timetools.Period('1d')
def setUp(self): self.refdate = datetime.datetime(1996, 11, 1, 12, 30, 5) self.testdate = timetools.Date(self.refdate)
def test_07_iso_style_second(self): self.assertEqual(self.refdate_second, timetools.Date('1996.11.01 12:30:05').datetime)
def controlcheck( controldir: str = "default", projectdir: Optional[str] = None, controlfile: Optional[str] = None, firstdate: Optional[timetools.DateConstrArg] = None, stepsize: Optional[timetools.PeriodConstrArg] = None, ) -> None: """Define the corresponding control file within a condition file. Function |controlcheck| serves similar purposes as function |parameterstep|. It is the reason why one can interactively access the state and the log sequences within condition files as `land_dill.py` of the example project `LahnH`. It is called `controlcheck` due to its feature to check for possible inconsistencies between control and condition files. The following test, where we write a number of soil moisture values (|hland_states.SM|) into condition file `land_dill.py`, which does not agree with the number of hydrological response units (|hland_control.NmbZones|) defined in control file `land_dill.py`, verifies that this, in fact, works within a separate Python process: >>> from hydpy.examples import prepare_full_example_1 >>> prepare_full_example_1() >>> import os >>> from hydpy import run_subprocess, TestIO >>> cwd = os.path.join("LahnH", "conditions", "init_1996_01_01_00_00_00") >>> with TestIO(): # doctest: +ELLIPSIS ... os.chdir(cwd) ... with open("land_dill.py") as file_: ... lines = file_.readlines() ... lines[10:12] = "sm(185.13164, 181.18755)", "" ... with open("land_dill.py", "w") as file_: ... _ = file_.write("\\n".join(lines)) ... print() ... result = run_subprocess("hyd.py exec_script land_dill.py") <BLANKLINE> ... While trying to set the value(s) of variable `sm`, the following error \ occurred: While trying to convert the value(s) `(185.13164, 181.18755)` to \ a numpy ndarray with shape `(12,)` and type `float`, the following error \ occurred: could not broadcast input array from shape (2...) into shape (12...) ... With a little trick, we can fake to be "inside" condition file `land_dill.py`. Calling |controlcheck| then, for example, prepares the shape of sequence |hland_states.Ic| as specified by the value of parameter |hland_control.NmbZones| given in the corresponding control file: >>> from hydpy.models.hland_v1 import * >>> __file__ = "land_dill.py" >>> with TestIO(): ... os.chdir(cwd) ... controlcheck(firstdate="1996-01-01", stepsize="1d") >>> ic.shape (12,) In the above example, we use the default names for the project directory (the one containing the executed condition file) and the control directory (`default`). The following example shows how to change them: >>> del model >>> with TestIO(): # doctest: +ELLIPSIS ... os.chdir(cwd) ... controlcheck(projectdir="somewhere", controldir="nowhere") Traceback (most recent call last): ... FileNotFoundError: While trying to load the control file `land_dill.py` \ from directory `...hydpy/tests/iotesting/somewhere/control/nowhere`, \ the following error occurred: ... For some models, the suitable states may depend on the initialisation date. One example is the interception storage (|lland_states.Inzp|) of application model |lland_v1|, which should not exceed the interception capacity (|lland_derived.KInz|). However, |lland_derived.KInz| itself depends on the leaf area index parameter |lland_control.LAI|, which offers varying values both for different land-use types and months. Hence, one can assign higher values to state |lland_states.Inzp| during periods with high leaf area indices than during periods with small leaf area indices. To show the related functionalities, we first replace the |hland_v1| application model of element `land_dill` with a |lland_v1| model object, define some of its parameter values, and write its control and condition files. Note that the |lland_control.LAI| value of the only relevant land-use (|lland_constants.ACKER|) is 0.5 during January and 5.0 during July: >>> from hydpy import HydPy, prepare_model, pub >>> from hydpy.models.lland_v1 import ACKER >>> pub.timegrids = "2000-06-01", "2000-07-01", "1d" >>> with TestIO(): ... hp = HydPy("LahnH") ... hp.prepare_network() ... land_dill = hp.elements["land_dill"] ... with pub.options.usedefaultvalues(True): ... land_dill.model = prepare_model("lland_v1") ... control = land_dill.model.parameters.control ... control.nhru(2) ... control.ft(1.0) ... control.fhru(0.5) ... control.hnn(100.0) ... control.lnk(ACKER) ... control.lai.acker_jan = 0.5 ... control.lai.acker_jul = 5.0 ... land_dill.model.parameters.update() ... land_dill.model.sequences.states.inzp(1.0) ... land_dill.model.parameters.save_controls() ... land_dill.model.sequences.save_conditions() Unfortunately, state |lland_states.Inzp| does not define a |trim| method taking the actual value of parameter |lland_derived.KInz| into account (due to compatibility with the original LARSIM model). As an auxiliary solution, we define such a function within the `land_dill.py` condition file (and additionally modify some warning settings in favour of the next examples): >>> cwd = os.path.join("LahnH", "conditions", "init_2000_07_01_00_00_00") >>> with TestIO(): ... os.chdir(cwd) ... with open("land_dill.py") as file_: ... lines = file_.readlines() ... with open("land_dill.py", "w") as file_: ... file_.writelines([ ... "from hydpy import pub\\n", ... "pub.options.warnsimulationstep = False\\n", ... "import warnings\\n", ... 'warnings.filterwarnings("error", message="For variable")\\n']) ... file_.writelines(lines[:5]) ... file_.writelines([ ... "from hydpy.core.variabletools import trim as trim_\\n", ... "def trim(self, lower=None, upper=None):\\n", ... " der = self.subseqs.seqs.model.parameters.derived\\n", ... " trim_(self, 0.0, der.kinz.acker[der.moy[0]])\\n", ... "type(inzp).trim = trim\\n"]) ... file_.writelines(lines[5:]) Now, executing the condition file (and thereby calling function |controlcheck|) does not raise any warnings due to extracting the initialisation date from the name of the condition directory: >>> with TestIO(): ... os.chdir(cwd) ... result = run_subprocess("hyd.py exec_script land_dill.py") If the directory name does imply the initialisation date to be within January 2000 instead of July 2000, we correctly get the following warning: >>> cwd_old = cwd >>> cwd_new = os.path.join("LahnH", "conditions", "init_2000_01_01") >>> with TestIO(): # doctest: +ELLIPSIS ... os.rename(cwd_old, cwd_new) ... os.chdir(cwd_new) ... result = run_subprocess("hyd.py exec_script land_dill.py") Invoking hyd.py with arguments `exec_script, land_dill.py` resulted \ in the following error: For variable `inzp` at least one value needed to be trimmed. \ The old and the new value(s) are `1.0, 1.0` and `0.1, 0.1`, respectively. ... One can define an alternative initialisation date via argument `firstdate`: >>> text_old = ('controlcheck(projectdir=r"LahnH", ' ... 'controldir="default", stepsize="1d")') >>> text_new = ('controlcheck(projectdir=r"LahnH", controldir="default", ' ... 'firstdate="2100-07-15", stepsize="1d")') >>> with TestIO(): ... os.chdir(cwd_new) ... with open("land_dill.py") as file_: ... text = file_.read() ... text = text.replace(text_old, text_new) ... with open("land_dill.py", "w") as file_: ... _ = file_.write(text) ... result = run_subprocess("hyd.py exec_script land_dill.py") Default condition directory names do not contain any information about the simulation step size. Hence, one needs to define it explicitly for all application modelsrelying on the functionalities of class |Indexer|: >>> with TestIO(): # doctest: +ELLIPSIS ... os.chdir(cwd_new) ... with open("land_dill.py") as file_: ... text = file_.read() ... text = text.replace('stepsize="1d"', "") ... with open("land_dill.py", "w") as file_: ... _ = file_.write(text) ... result = run_subprocess("hyd.py exec_script land_dill.py") Invoking hyd.py with arguments `exec_script, land_dill.py` resulted \ in the following error: To apply function `controlcheck` requires time information for some \ model types. Please define the `Timegrids` object of module `pub` manually \ or pass the required information (`stepsize` and eventually `firstdate`) \ as function arguments. ... The same error occurs we do not use the argument `firstdate` to define the initialisation time point, and method |controlcheck| cannot extract it from the directory name: >>> cwd_old = cwd_new >>> cwd_new = os.path.join("LahnH", "conditions", "init") >>> with TestIO(): # doctest: +ELLIPSIS ... os.rename(cwd_old, cwd_new) ... os.chdir(cwd_new) ... with open("land_dill.py") as file_: ... text = file_.read() ... text = text.replace('firstdate="2100-07-15"', 'stepsize="1d"') ... with open("land_dill.py", "w") as file_: ... _ = file_.write(text) ... result = run_subprocess("hyd.py exec_script land_dill.py") Invoking hyd.py with arguments `exec_script, land_dill.py` resulted \ in the following error: To apply function `controlcheck` requires time information for some \ model types. Please define the `Timegrids` object of module `pub` manually \ or pass the required information (`stepsize` and eventually `firstdate`) \ as function arguments. ... Note that the functionalities of function |controlcheck| do not come into action if there is a `model` variable in the namespace, which is the case when a condition file is executed within the context of a complete *HydPy* project. """ namespace = inspect.currentframe().f_back.f_locals model = namespace.get("model") if model is None: if not controlfile: controlfile = os.path.split(namespace["__file__"])[-1] if projectdir is None: projectdir = os.path.split( os.path.split(os.path.split(os.getcwd())[0])[0])[-1] dirpath = os.path.abspath( os.path.join("..", "..", "..", projectdir, "control", controldir)) if not (exceptiontools.attrready(hydpy.pub, "timegrids") or (stepsize is None)): if firstdate is None: try: firstdate = timetools.Date.from_string( os.path.split(os.getcwd())[-1].partition("_")[-1]) except (ValueError, TypeError): pass else: firstdate = timetools.Date(firstdate) if firstdate is not None: stepsize = timetools.Period(stepsize) hydpy.pub.timegrids = (firstdate, firstdate + 1000 * stepsize, stepsize) class CM(filetools.ControlManager): """Tempory |ControlManager| class.""" currentpath = dirpath cwd = os.getcwd() try: os.chdir(dirpath) model = CM().load_file(filename=controlfile)["model"] except BaseException: objecttools.augment_excmessage( f"While trying to load the control file `{controlfile}` " f"from directory `{objecttools.repr_(dirpath)}`") finally: os.chdir(cwd) try: model.parameters.update() except exceptiontools.AttributeNotReady as exc: raise RuntimeError( "To apply function `controlcheck` requires time " "information for some model types. Please define " "the `Timegrids` object of module `pub` manually " "or pass the required information (`stepsize` and " "eventually `firstdate`) as function arguments.") from exc namespace["model"] = model for name in ("states", "logs"): subseqs = getattr(model.sequences, name, None) if subseqs is not None: for seq in subseqs: namespace[seq.name] = seq
def test_08_din_style_day(self): self.assertEqual(self.refdate_day, timetools.Date('01.11.1996').datetime)
def _gettimeofyear(self): """Time of the year index (first simulation step of each year = 0...). The property |Indexer.timeofyear| is best explained through comparing it with property |Indexer.dayofyear|: Let us reconsider one of the examples of the documentation on property |Indexer.dayofyear|: >>> from hydpy import pub >>> from hydpy import Timegrids, Timegrid >>> from hydpy.core.indextools import Indexer >>> pub.timegrids = Timegrids(Timegrid('27.02.2005', ... '3.03.2005', ... '1d')) Due to the simulation stepsize being one day, the index arrays calculated by both properties are identical: >>> Indexer().dayofyear array([57, 58, 60, 61]) >>> Indexer().timeofyear array([57, 58, 60, 61]) In the next example the step size is halved: >>> pub.timegrids = Timegrids(Timegrid('27.02.2005', ... '3.03.2005', ... '12h')) Now the there a generally two subsequent simulation steps associated with the same day: >>> Indexer().dayofyear array([57, 57, 58, 58, 60, 60, 61, 61]) However, the `timeofyear` array gives the index of the respective simulation steps of the actual year: >>> Indexer().timeofyear array([114, 115, 116, 117, 120, 121, 122, 123]) Note the gap in the returned index array due to 2005 being not a leap year. """ if ((self._timeofyear is None) or (hash(pub.timegrids) != self._timeofyear_hash)): if pub.timegrids is None: refgrid = None else: refgrid = timetools.Timegrid(timetools.Date('2000.01.01'), timetools.Date('2001.01.01'), pub.timegrids.stepsize) def timeofyear(date): date = date.copy() date.year = 2000 return refgrid[date] self._timeofyear = self._calcidxs(timeofyear) self._timeofyear_hash = hash(pub.timegrids) return self._timeofyear
def test_01_os_style_day(self): self.assertEqual(self.refdate_day, timetools.Date('1996_11_01').datetime)