def test_write(self): anp = np.array([ [parse_date("2005-08-23 18:53"), 93, ""], [parse_date("2005-08-24 19:52"), 108.7, ""], [parse_date("2005-08-25 23:59"), 28.3, "HEARTS SPADES"], [parse_date("2005-08-26 00:02"), float("NaN"), ""], [parse_date("2005-08-27 00:02"), float("NaN"), "DIAMONDS"], ]) data = pd.DataFrame(anp[:, [1, 2]], index=anp[:, 0], columns=("value", "flags")) ts = HTimeseries(data=data) s = StringIO() ts.write(s) self.assertEqual( s.getvalue(), textwrap.dedent("""\ 2005-08-23 18:53,93,\r 2005-08-24 19:52,108.7,\r 2005-08-25 23:59,28.3,HEARTS SPADES\r 2005-08-26 00:02,,\r 2005-08-27 00:02,,DIAMONDS\r """), )
def test_execute(self): application = cli.App(self.config_file) # Check that the two files don't exist yet self.assertFalse(os.path.exists(os.path.join(self.tempdir, "file1"))) self.assertFalse(os.path.exists(os.path.join(self.tempdir, "file2"))) # Execute the application application.run() # Check that it has created two files self.assertTrue(os.path.exists(os.path.join(self.tempdir, "file1"))) self.assertTrue(os.path.exists(os.path.join(self.tempdir, "file2"))) # Check that the files are what they should be with open("file1", newline="\n") as f: ts1_before = HTimeseries(f) self.assertEqual(ts1_before.time_step, "D") c = StringIO() ts1_before.write(c) self.assertEqual(c.getvalue().replace("\r", ""), self.timeseries1_top) with open("file2", newline="\n") as f: ts2_before = HTimeseries(f) self.assertEqual(ts2_before.time_step, "D") c = StringIO() ts2_before.write(c) self.assertEqual(c.getvalue().replace("\r", ""), self.timeseries2_top) # Append a record to the database for each timeseries self.api_client.post_tsdata( self.station1_id, self.timeseries1_id, HTimeseries(StringIO(self.timeseries1_bottom)), ) self.api_client.post_tsdata( self.station2_id, self.timeseries2_id, HTimeseries(StringIO(self.timeseries2_bottom)), ) # Execute the application again application.run() # Check that the files are what they should be with open("file1", newline="\n") as f: ts1_after = HTimeseries(f) self.assertEqual(ts1_after.time_step, "D") c = StringIO() ts1_after.write(c) self.assertEqual(c.getvalue().replace("\r", ""), self.test_timeseries1) with open("file2", newline="\n") as f: ts2_after = HTimeseries(f) self.assertEqual(ts2_after.time_step, "D") c = StringIO() ts2_after.write(c) self.assertEqual(c.getvalue().replace("\r", ""), self.test_timeseries2) # Check that the time series comments are the same before and after self.assertEqual(ts1_before.comment, ts1_after.comment) self.assertEqual(ts2_before.comment, ts2_after.comment)
class HTimeseriesWriteFileTestCase(TestCase): def setUp(self): data = pd.read_csv( StringIO(tenmin_test_timeseries), parse_dates=[0], usecols=["date", "value", "flags"], index_col=0, header=None, names=("date", "value", "flags"), dtype={ "value": np.float64, "flags": str }, ).asfreq("10T") self.reference_ts = HTimeseries(data=data) self.reference_ts.unit = "°C" self.reference_ts.title = "A test 10-min time series" self.reference_ts.precision = 1 self.reference_ts.time_step = "10min" self.reference_ts.timezone = "EET (UTC+0200)" self.reference_ts.variable = "temperature" self.reference_ts.comment = ("This timeseries is extremely important\n" "because the comment that describes it\n" "spans five lines.\n\n" "These five lines form two paragraphs.") self.reference_ts.location = { "abscissa": 24.6789, "ordinate": 38.12345, "srid": 4326, "altitude": 219.22, "asrid": None, } def test_version_2(self): outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=2) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_version_2) def test_version_3(self): outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=3) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_version_3) def test_version_4(self): outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_version_4) def test_version_5(self): outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=5) self.assertEqual( outstring.getvalue(), tenmin_test_timeseries_file_version_4.replace( "Time_step=10,0", "Time_step=10min"), ) def test_version_latest(self): outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE) self.assertEqual( outstring.getvalue(), tenmin_test_timeseries_file_version_4.replace( "Time_step=10,0", "Time_step=10min"), ) def test_altitude_none(self): self.reference_ts.location["altitude"] = None outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_no_altitude) def test_no_altitude(self): del self.reference_ts.location["altitude"] outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_no_altitude) def test_altitude_zero(self): self.reference_ts.location["altitude"] = 0 outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertIn("Altitude=0", outstring.getvalue()) def test_location_none(self): self.reference_ts.location = None outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_no_location) def test_no_location(self): delattr(self.reference_ts, "location") outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_no_location) def test_precision_none(self): self.reference_ts.precision = None outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_no_precision) def test_no_precision(self): delattr(self.reference_ts, "precision") outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_no_precision) def test_precision_zero(self): self.reference_ts.precision = 0 outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_zero_precision) def test_negative_precision(self): self.reference_ts.precision = -1 outstring = StringIO() self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4) self.assertEqual(outstring.getvalue(), tenmin_test_timeseries_file_negative_precision)
def test_write_empty(self): ts = HTimeseries() s = StringIO() ts.write(s) self.assertEqual(s.getvalue(), "")
class ProcessAtPoint: def __init__(self, config): self.config = config def execute(self): self._read_input_timeseries() self._setup_attrs() self._check_all_timeseries_are_in_same_location_and_timezone() self._get_location_in_wgs84() self._prepare_penman_monteith_parameters() self._prepare_resulting_htimeseries_object() self._determine_variables_to_use_in_calculation() self._calculate_evaporation() self._save_result() def _setup_attrs(self): atimeseries = self.input_timeseries["wind_speed"] self.location = atimeseries.location self.timezone = getattr(atimeseries, "timezone", None) def _read_input_timeseries(self): vars = { "temperature_min", "temperature_max", "temperature", "humidity", "humidity_min", "humidity_max", "wind_speed", "pressure", "solar_radiation", "sunshine_duration", } self.input_timeseries = {} for var in vars: self._get_input_timeseries_for_var(var) def _get_input_timeseries_for_var(self, var): filename = os.path.join(self.config.base_dir, getattr(self.config, var + "_prefix") + ".hts") if not os.path.exists(filename): return with open(filename, "r") as f: self.input_timeseries[var] = HTimeseries(f) def _check_all_timeseries_are_in_same_location_and_timezone(self): for i, (name, hts) in enumerate(self.input_timeseries.items()): if i == 0: reference_hts = hts else: self._compare_location_and_timezone_of(hts, reference_hts) def _compare_location_and_timezone_of(self, hts1, hts2): self._compare_locations_of(hts1, hts2) self._compare_altitudes_of(hts1, hts2) self._compare_timezones_of(hts1, hts2) def _compare_locations_of(self, hts1, hts2): loc1 = hts1.location loc2 = hts2.location abscissas_differ = self._numbers_are_wrong_or_differ( loc1.get("abscissa"), loc2.get("abscissa")) ordinates_differ = self._numbers_are_wrong_or_differ( loc1.get("ordinate"), loc2.get("ordinate")) srids_differ = loc1.get("srid") != loc2.get("srid") if abscissas_differ or ordinates_differ or srids_differ: raise ValueError( "Incorrect or unspecified or inconsistent locations in the time series " "files.") def _numbers_are_wrong_or_differ(self, num1, num2, tolerance=1e7): if num1 is None or num2 is None: return True return abs(num1 - num2) > tolerance def _compare_altitudes_of(self, hts1, hts2): altitude1 = hts1.location.get("altitude") altitude2 = hts2.location.get("altitude") if self._numbers_are_wrong_or_differ(altitude1, altitude2, 1e-2): raise ValueError( "Incorrect or unspecified or inconsistent altitudes in the time series " "files.") def _compare_timezones_of(self, hts1, hts2): timezone1 = getattr(hts1, "timezone", "") timezone2 = getattr(hts2, "timezone", "") if timezone1 != timezone2: raise ValueError( "Incorrect or unspecified or inconsistent time zones in the time " "series files.") def _get_location_in_wgs84(self): source_projection = osr.SpatialReference() source_projection.ImportFromEPSG(self.location["srid"]) wgs84 = osr.SpatialReference() wgs84.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(source_projection, wgs84) apoint = ogr.Geometry(ogr.wkbPoint) apoint.AddPoint(self.location["abscissa"], self.location["ordinate"]) apoint.Transform(transform) self.location["latitude"] = apoint.GetY() self.location["longitude"] = apoint.GetX() def _prepare_penman_monteith_parameters(self): nsrr = self.config.nighttime_solar_radiation_ratio self.penman_monteith = PenmanMonteith( albedo=self.config.albedo, nighttime_solar_radiation_ratio=nsrr, elevation=self.location["altitude"], latitude=self.location["latitude"], longitude=self.location["longitude"], step_length=self.config.step, unit_converters=self.config.unit_converters, ) def _prepare_resulting_htimeseries_object(self): self.pet = HTimeseries() minutes = int(self.config.step.total_seconds() / 60) self.pet.time_step = str(minutes) + ",0" self.pet.unit = "mm" self.pet.timezone = self.timezone self.pet.variable = "Potential Evapotranspiration" self.pet.precision = 2 if self.config.step == dt.timedelta( hours=1) else 1 self.pet.location = self.location def _determine_variables_to_use_in_calculation(self): if self.config.step == dt.timedelta(hours=1): vars = ["temperature", "humidity", "wind_speed", "solar_radiation"] if "pressure" in self.input_timeseries: vars.append("pressure") elif self.config.step == dt.timedelta(days=1): vars = ( "temperature_max", "temperature_min", "humidity_max", "humidity_min", "wind_speed", ("solar_radiation" if "solar_radiation" in self.input_timeseries else "sunshine_duration"), ) self.input_vars = vars def _calculate_evaporation(self): for adatetime in self.input_timeseries["wind_speed"].data.index: self._calculate_evaporation_for(adatetime) def _calculate_evaporation_for(self, adatetime): try: kwargs = { v: self.input_timeseries[v].data.loc[adatetime, "value"] for v in self.input_vars } except (IndexError, KeyError): return kwargs["adatetime"] = self._datetime64_to_aware_datetime(adatetime) self.pet.data.loc[adatetime, "value"] = self.penman_monteith.calculate(**kwargs) def _datetime64_to_aware_datetime(self, adatetime): result = adatetime.to_pydatetime() if self.timezone: result = result.replace(tzinfo=TzinfoFromString(self.timezone)) return result def _save_result(self): outfilename = self.config.evaporation_prefix + ".hts" outpathname = os.path.join(self.config.base_dir, outfilename) with open(outpathname, "w") as f: self.pet.write(f, format=HTimeseries.FILE)
class RoccTestCase(TestCase): test_data = textwrap.dedent("""\ 2020-10-06 14:30,24.0, 2020-10-06 14:40,25.0, 2020-10-06 14:50,36.0,SOMEFLAG 2020-10-06 15:01,51.0, 2020-10-06 15:21,55.0, 2020-10-06 15:31,65.0, 2020-10-06 15:41,75.0, 2020-10-06 15:51,70.0, """) def setUp(self): self.ahtimeseries = HTimeseries(StringIO(self.test_data)) self.ahtimeseries.precision = 1 def _run_rocc(self, flag): self.return_value = rocc( timeseries=self.ahtimeseries, thresholds=( Threshold("10min", 10), Threshold("20min", 15), Threshold("H", 40), ), flag=flag, ) def test_calculation(self): self._run_rocc(flag="TEMPORAL") result = StringIO() self.ahtimeseries.write(result) result = result.getvalue().replace("\r\n", "\n") self.assertEqual( result, textwrap.dedent("""\ 2020-10-06 14:30,24.0, 2020-10-06 14:40,25.0, 2020-10-06 14:50,36.0,SOMEFLAG TEMPORAL 2020-10-06 15:01,51.0, 2020-10-06 15:21,55.0, 2020-10-06 15:31,65.0, 2020-10-06 15:41,75.0,TEMPORAL 2020-10-06 15:51,70.0, """), ) def test_return_value(self): self._run_rocc(flag="TEMPORAL") self.assertEqual(len(self.return_value), 2) self.assertEqual(self.return_value[0], "2020-10-06T14:50 +11.0 in 10min (> 10.0)") self.assertEqual(self.return_value[1], "2020-10-06T15:41 +20.0 in 20min (> 15.0)") def test_value_dtype(self): self._run_rocc(flag="TEMPORAL") expected_dtype = HTimeseries().data["value"].dtype self.assertEqual(self.ahtimeseries.data["value"].dtype, expected_dtype) def test_flags_dtype(self): self._run_rocc(flag="TEMPORAL") expected_dtype = HTimeseries().data["flags"].dtype self.assertEqual(self.ahtimeseries.data["flags"].dtype, expected_dtype) def test_empty_flag(self): self._run_rocc(flag=None) result = StringIO() self.ahtimeseries.write(result) result = result.getvalue().replace("\r\n", "\n") self.assertEqual(result, self.test_data)
class RoccSymmetricCase(TestCase): test_data = textwrap.dedent("""\ 2020-10-06 14:30,76.0, 2020-10-06 14:40,75.0,SOMEFLAG 2020-10-06 14:50,64.0,SOMEFLAG 2020-10-06 15:01,49.0, 2020-10-06 15:21,45.0, 2020-10-06 15:31,35.0, 2020-10-06 15:41,25.0, 2020-10-06 15:51,30.0, """) def setUp(self): self.ahtimeseries = HTimeseries(StringIO(self.test_data)) self.ahtimeseries.precision = 1 def test_without_symmetric(self): rocc( timeseries=self.ahtimeseries, thresholds=( Threshold("10min", 10), Threshold("20min", 15), Threshold("H", 40), ), ) result = StringIO() self.ahtimeseries.write(result) result = result.getvalue().replace("\r\n", "\n") self.assertEqual( result, textwrap.dedent("""\ 2020-10-06 14:30,76.0, 2020-10-06 14:40,75.0,SOMEFLAG 2020-10-06 14:50,64.0,SOMEFLAG 2020-10-06 15:01,49.0, 2020-10-06 15:21,45.0, 2020-10-06 15:31,35.0, 2020-10-06 15:41,25.0, 2020-10-06 15:51,30.0, """), ) def test_with_symmetric(self): rocc( timeseries=self.ahtimeseries, thresholds=( Threshold("10min", 10), Threshold("20min", 15), Threshold("H", 40), ), symmetric=True, ) result = StringIO() self.ahtimeseries.write(result) result = result.getvalue().replace("\r\n", "\n") self.assertEqual( result, textwrap.dedent("""\ 2020-10-06 14:30,76.0, 2020-10-06 14:40,75.0,SOMEFLAG 2020-10-06 14:50,64.0,SOMEFLAG TEMPORAL 2020-10-06 15:01,49.0, 2020-10-06 15:21,45.0, 2020-10-06 15:31,35.0, 2020-10-06 15:41,25.0,TEMPORAL 2020-10-06 15:51,30.0, """), ) def test_symmetric_return_value(self): return_value = rocc( timeseries=self.ahtimeseries, thresholds=( Threshold("10min", 10), Threshold("20min", 15), Threshold("H", 40), ), symmetric=True, ) self.assertEqual(len(return_value), 2) self.assertEqual(return_value[0], "2020-10-06T14:50 -11.0 in 10min (< -10.0)") self.assertEqual(return_value[1], "2020-10-06T15:41 -20.0 in 20min (< -15.0)")
def test_update(self, mock_api_client): timeseries_group = [ { "base_url": "https://mydomain.com", "station_id": 2, "timeseries_id": 42, "user": "******", "password": "******", "file": "file1", }, { "base_url": "https://mydomain.com", "station_id": 3, "timeseries_id": 43, "user": "******", "password": "******", "file": "file2", }, ] # Cache the two timeseries cache = TimeseriesCache(timeseries_group) cache.update() # Check that the cached stuff is what it should be with open("file1", newline="\n") as f: ts1_before = HTimeseries(f) self.assertEqual(ts1_before.time_step, "D") c = StringIO() ts1_before.write(c) self.assertEqual(c.getvalue().replace("\r", ""), test_timeseries["42_top"]) with open("file2", newline="\n") as f: ts2_before = HTimeseries(f) self.assertEqual(ts2_before.time_step, "D") c = StringIO() ts2_before.write(c) self.assertEqual(c.getvalue().replace("\r", ""), test_timeseries["43_top"]) # Update the cache cache.update() # Check that the cached stuff is what it should be with open("file1", newline="\n") as f: ts1_after = HTimeseries(f) self.assertEqual(ts1_after.time_step, "D") c = StringIO() ts1_after.write(c) self.assertEqual(c.getvalue().replace("\r", ""), test_timeseries["42_all"]) with open("file2", newline="\n") as f: ts2_after = HTimeseries(f) self.assertEqual(ts2_after.time_step, "D") c = StringIO() ts2_after.write(c) self.assertEqual(c.getvalue().replace("\r", ""), test_timeseries["43_all"]) # Check that the time series comments are the same before and after self.assertEqual(ts1_before.comment, ts1_after.comment) self.assertEqual(ts2_before.comment, ts2_after.comment)