def test_write(self):
anp = np.array([
[parse_date("2005-08-23 18:53"), 93, ""],
[parse_date("2005-08-24 19:52"), 108.7, ""],
[parse_date("2005-08-25 23:59"), 28.3, "HEARTS SPADES"],
[parse_date("2005-08-26 00:02"),
float("NaN"), ""],
[parse_date("2005-08-27 00:02"),
float("NaN"), "DIAMONDS"],
])
data = pd.DataFrame(anp[:, [1, 2]],
index=anp[:, 0],
columns=("value", "flags"))
ts = HTimeseries(data=data)
s = StringIO()
ts.write(s)
self.assertEqual(
s.getvalue(),
textwrap.dedent("""\
2005-08-23 18:53,93,\r
2005-08-24 19:52,108.7,\r
2005-08-25 23:59,28.3,HEARTS SPADES\r
2005-08-26 00:02,,\r
2005-08-27 00:02,,DIAMONDS\r
"""),
)
def test_execute(self):
application = cli.App(self.config_file)
# Check that the two files don't exist yet
self.assertFalse(os.path.exists(os.path.join(self.tempdir, "file1")))
self.assertFalse(os.path.exists(os.path.join(self.tempdir, "file2")))
# Execute the application
application.run()
# Check that it has created two files
self.assertTrue(os.path.exists(os.path.join(self.tempdir, "file1")))
self.assertTrue(os.path.exists(os.path.join(self.tempdir, "file2")))
# Check that the files are what they should be
with open("file1", newline="\n") as f:
ts1_before = HTimeseries(f)
self.assertEqual(ts1_before.time_step, "D")
c = StringIO()
ts1_before.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.timeseries1_top)
with open("file2", newline="\n") as f:
ts2_before = HTimeseries(f)
self.assertEqual(ts2_before.time_step, "D")
c = StringIO()
ts2_before.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.timeseries2_top)
# Append a record to the database for each timeseries
self.api_client.post_tsdata(
self.station1_id,
self.timeseries1_id,
HTimeseries(StringIO(self.timeseries1_bottom)),
)
self.api_client.post_tsdata(
self.station2_id,
self.timeseries2_id,
HTimeseries(StringIO(self.timeseries2_bottom)),
)
# Execute the application again
application.run()
# Check that the files are what they should be
with open("file1", newline="\n") as f:
ts1_after = HTimeseries(f)
self.assertEqual(ts1_after.time_step, "D")
c = StringIO()
ts1_after.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.test_timeseries1)
with open("file2", newline="\n") as f:
ts2_after = HTimeseries(f)
self.assertEqual(ts2_after.time_step, "D")
c = StringIO()
ts2_after.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.test_timeseries2)
# Check that the time series comments are the same before and after
self.assertEqual(ts1_before.comment, ts1_after.comment)
self.assertEqual(ts2_before.comment, ts2_after.comment)
class HTimeseriesWriteFileTestCase(TestCase):
def setUp(self):
data = pd.read_csv(
StringIO(tenmin_test_timeseries),
parse_dates=[0],
usecols=["date", "value", "flags"],
index_col=0,
header=None,
names=("date", "value", "flags"),
dtype={
"value": np.float64,
"flags": str
},
).asfreq("10T")
self.reference_ts = HTimeseries(data=data)
self.reference_ts.unit = "°C"
self.reference_ts.title = "A test 10-min time series"
self.reference_ts.precision = 1
self.reference_ts.time_step = "10min"
self.reference_ts.timezone = "EET (UTC+0200)"
self.reference_ts.variable = "temperature"
self.reference_ts.comment = ("This timeseries is extremely important\n"
"because the comment that describes it\n"
"spans five lines.\n\n"
"These five lines form two paragraphs.")
self.reference_ts.location = {
"abscissa": 24.6789,
"ordinate": 38.12345,
"srid": 4326,
"altitude": 219.22,
"asrid": None,
}
def test_version_2(self):
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=2)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_version_2)
def test_version_3(self):
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=3)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_version_3)
def test_version_4(self):
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_version_4)
def test_version_5(self):
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=5)
self.assertEqual(
outstring.getvalue(),
tenmin_test_timeseries_file_version_4.replace(
"Time_step=10,0", "Time_step=10min"),
)
def test_version_latest(self):
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE)
self.assertEqual(
outstring.getvalue(),
tenmin_test_timeseries_file_version_4.replace(
"Time_step=10,0", "Time_step=10min"),
)
def test_altitude_none(self):
self.reference_ts.location["altitude"] = None
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_no_altitude)
def test_no_altitude(self):
del self.reference_ts.location["altitude"]
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_no_altitude)
def test_altitude_zero(self):
self.reference_ts.location["altitude"] = 0
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertIn("Altitude=0", outstring.getvalue())
def test_location_none(self):
self.reference_ts.location = None
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_no_location)
def test_no_location(self):
delattr(self.reference_ts, "location")
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_no_location)
def test_precision_none(self):
self.reference_ts.precision = None
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_no_precision)
def test_no_precision(self):
delattr(self.reference_ts, "precision")
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_no_precision)
def test_precision_zero(self):
self.reference_ts.precision = 0
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_zero_precision)
def test_negative_precision(self):
self.reference_ts.precision = -1
outstring = StringIO()
self.reference_ts.write(outstring, format=HTimeseries.FILE, version=4)
self.assertEqual(outstring.getvalue(),
tenmin_test_timeseries_file_negative_precision)
def test_write_empty(self):
ts = HTimeseries()
s = StringIO()
ts.write(s)
self.assertEqual(s.getvalue(), "")
class ProcessAtPoint:
def __init__(self, config):
self.config = config
def execute(self):
self._read_input_timeseries()
self._setup_attrs()
self._check_all_timeseries_are_in_same_location_and_timezone()
self._get_location_in_wgs84()
self._prepare_penman_monteith_parameters()
self._prepare_resulting_htimeseries_object()
self._determine_variables_to_use_in_calculation()
self._calculate_evaporation()
self._save_result()
def _setup_attrs(self):
atimeseries = self.input_timeseries["wind_speed"]
self.location = atimeseries.location
self.timezone = getattr(atimeseries, "timezone", None)
def _read_input_timeseries(self):
vars = {
"temperature_min",
"temperature_max",
"temperature",
"humidity",
"humidity_min",
"humidity_max",
"wind_speed",
"pressure",
"solar_radiation",
"sunshine_duration",
}
self.input_timeseries = {}
for var in vars:
self._get_input_timeseries_for_var(var)
def _get_input_timeseries_for_var(self, var):
filename = os.path.join(self.config.base_dir,
getattr(self.config, var + "_prefix") + ".hts")
if not os.path.exists(filename):
return
with open(filename, "r") as f:
self.input_timeseries[var] = HTimeseries(f)
def _check_all_timeseries_are_in_same_location_and_timezone(self):
for i, (name, hts) in enumerate(self.input_timeseries.items()):
if i == 0:
reference_hts = hts
else:
self._compare_location_and_timezone_of(hts, reference_hts)
def _compare_location_and_timezone_of(self, hts1, hts2):
self._compare_locations_of(hts1, hts2)
self._compare_altitudes_of(hts1, hts2)
self._compare_timezones_of(hts1, hts2)
def _compare_locations_of(self, hts1, hts2):
loc1 = hts1.location
loc2 = hts2.location
abscissas_differ = self._numbers_are_wrong_or_differ(
loc1.get("abscissa"), loc2.get("abscissa"))
ordinates_differ = self._numbers_are_wrong_or_differ(
loc1.get("ordinate"), loc2.get("ordinate"))
srids_differ = loc1.get("srid") != loc2.get("srid")
if abscissas_differ or ordinates_differ or srids_differ:
raise ValueError(
"Incorrect or unspecified or inconsistent locations in the time series "
"files.")
def _numbers_are_wrong_or_differ(self, num1, num2, tolerance=1e7):
if num1 is None or num2 is None:
return True
return abs(num1 - num2) > tolerance
def _compare_altitudes_of(self, hts1, hts2):
altitude1 = hts1.location.get("altitude")
altitude2 = hts2.location.get("altitude")
if self._numbers_are_wrong_or_differ(altitude1, altitude2, 1e-2):
raise ValueError(
"Incorrect or unspecified or inconsistent altitudes in the time series "
"files.")
def _compare_timezones_of(self, hts1, hts2):
timezone1 = getattr(hts1, "timezone", "")
timezone2 = getattr(hts2, "timezone", "")
if timezone1 != timezone2:
raise ValueError(
"Incorrect or unspecified or inconsistent time zones in the time "
"series files.")
def _get_location_in_wgs84(self):
source_projection = osr.SpatialReference()
source_projection.ImportFromEPSG(self.location["srid"])
wgs84 = osr.SpatialReference()
wgs84.ImportFromEPSG(4326)
transform = osr.CoordinateTransformation(source_projection, wgs84)
apoint = ogr.Geometry(ogr.wkbPoint)
apoint.AddPoint(self.location["abscissa"], self.location["ordinate"])
apoint.Transform(transform)
self.location["latitude"] = apoint.GetY()
self.location["longitude"] = apoint.GetX()
def _prepare_penman_monteith_parameters(self):
nsrr = self.config.nighttime_solar_radiation_ratio
self.penman_monteith = PenmanMonteith(
albedo=self.config.albedo,
nighttime_solar_radiation_ratio=nsrr,
elevation=self.location["altitude"],
latitude=self.location["latitude"],
longitude=self.location["longitude"],
step_length=self.config.step,
unit_converters=self.config.unit_converters,
)
def _prepare_resulting_htimeseries_object(self):
self.pet = HTimeseries()
minutes = int(self.config.step.total_seconds() / 60)
self.pet.time_step = str(minutes) + ",0"
self.pet.unit = "mm"
self.pet.timezone = self.timezone
self.pet.variable = "Potential Evapotranspiration"
self.pet.precision = 2 if self.config.step == dt.timedelta(
hours=1) else 1
self.pet.location = self.location
def _determine_variables_to_use_in_calculation(self):
if self.config.step == dt.timedelta(hours=1):
vars = ["temperature", "humidity", "wind_speed", "solar_radiation"]
if "pressure" in self.input_timeseries:
vars.append("pressure")
elif self.config.step == dt.timedelta(days=1):
vars = (
"temperature_max",
"temperature_min",
"humidity_max",
"humidity_min",
"wind_speed",
("solar_radiation" if "solar_radiation"
in self.input_timeseries else "sunshine_duration"),
)
self.input_vars = vars
def _calculate_evaporation(self):
for adatetime in self.input_timeseries["wind_speed"].data.index:
self._calculate_evaporation_for(adatetime)
def _calculate_evaporation_for(self, adatetime):
try:
kwargs = {
v: self.input_timeseries[v].data.loc[adatetime, "value"]
for v in self.input_vars
}
except (IndexError, KeyError):
return
kwargs["adatetime"] = self._datetime64_to_aware_datetime(adatetime)
self.pet.data.loc[adatetime,
"value"] = self.penman_monteith.calculate(**kwargs)
def _datetime64_to_aware_datetime(self, adatetime):
result = adatetime.to_pydatetime()
if self.timezone:
result = result.replace(tzinfo=TzinfoFromString(self.timezone))
return result
def _save_result(self):
outfilename = self.config.evaporation_prefix + ".hts"
outpathname = os.path.join(self.config.base_dir, outfilename)
with open(outpathname, "w") as f:
self.pet.write(f, format=HTimeseries.FILE)
class RoccTestCase(TestCase):
test_data = textwrap.dedent("""\
2020-10-06 14:30,24.0,
2020-10-06 14:40,25.0,
2020-10-06 14:50,36.0,SOMEFLAG
2020-10-06 15:01,51.0,
2020-10-06 15:21,55.0,
2020-10-06 15:31,65.0,
2020-10-06 15:41,75.0,
2020-10-06 15:51,70.0,
""")
def setUp(self):
self.ahtimeseries = HTimeseries(StringIO(self.test_data))
self.ahtimeseries.precision = 1
def _run_rocc(self, flag):
self.return_value = rocc(
timeseries=self.ahtimeseries,
thresholds=(
Threshold("10min", 10),
Threshold("20min", 15),
Threshold("H", 40),
),
flag=flag,
)
def test_calculation(self):
self._run_rocc(flag="TEMPORAL")
result = StringIO()
self.ahtimeseries.write(result)
result = result.getvalue().replace("\r\n", "\n")
self.assertEqual(
result,
textwrap.dedent("""\
2020-10-06 14:30,24.0,
2020-10-06 14:40,25.0,
2020-10-06 14:50,36.0,SOMEFLAG TEMPORAL
2020-10-06 15:01,51.0,
2020-10-06 15:21,55.0,
2020-10-06 15:31,65.0,
2020-10-06 15:41,75.0,TEMPORAL
2020-10-06 15:51,70.0,
"""),
)
def test_return_value(self):
self._run_rocc(flag="TEMPORAL")
self.assertEqual(len(self.return_value), 2)
self.assertEqual(self.return_value[0],
"2020-10-06T14:50 +11.0 in 10min (> 10.0)")
self.assertEqual(self.return_value[1],
"2020-10-06T15:41 +20.0 in 20min (> 15.0)")
def test_value_dtype(self):
self._run_rocc(flag="TEMPORAL")
expected_dtype = HTimeseries().data["value"].dtype
self.assertEqual(self.ahtimeseries.data["value"].dtype, expected_dtype)
def test_flags_dtype(self):
self._run_rocc(flag="TEMPORAL")
expected_dtype = HTimeseries().data["flags"].dtype
self.assertEqual(self.ahtimeseries.data["flags"].dtype, expected_dtype)
def test_empty_flag(self):
self._run_rocc(flag=None)
result = StringIO()
self.ahtimeseries.write(result)
result = result.getvalue().replace("\r\n", "\n")
self.assertEqual(result, self.test_data)
class RoccSymmetricCase(TestCase):
test_data = textwrap.dedent("""\
2020-10-06 14:30,76.0,
2020-10-06 14:40,75.0,SOMEFLAG
2020-10-06 14:50,64.0,SOMEFLAG
2020-10-06 15:01,49.0,
2020-10-06 15:21,45.0,
2020-10-06 15:31,35.0,
2020-10-06 15:41,25.0,
2020-10-06 15:51,30.0,
""")
def setUp(self):
self.ahtimeseries = HTimeseries(StringIO(self.test_data))
self.ahtimeseries.precision = 1
def test_without_symmetric(self):
rocc(
timeseries=self.ahtimeseries,
thresholds=(
Threshold("10min", 10),
Threshold("20min", 15),
Threshold("H", 40),
),
)
result = StringIO()
self.ahtimeseries.write(result)
result = result.getvalue().replace("\r\n", "\n")
self.assertEqual(
result,
textwrap.dedent("""\
2020-10-06 14:30,76.0,
2020-10-06 14:40,75.0,SOMEFLAG
2020-10-06 14:50,64.0,SOMEFLAG
2020-10-06 15:01,49.0,
2020-10-06 15:21,45.0,
2020-10-06 15:31,35.0,
2020-10-06 15:41,25.0,
2020-10-06 15:51,30.0,
"""),
)
def test_with_symmetric(self):
rocc(
timeseries=self.ahtimeseries,
thresholds=(
Threshold("10min", 10),
Threshold("20min", 15),
Threshold("H", 40),
),
symmetric=True,
)
result = StringIO()
self.ahtimeseries.write(result)
result = result.getvalue().replace("\r\n", "\n")
self.assertEqual(
result,
textwrap.dedent("""\
2020-10-06 14:30,76.0,
2020-10-06 14:40,75.0,SOMEFLAG
2020-10-06 14:50,64.0,SOMEFLAG TEMPORAL
2020-10-06 15:01,49.0,
2020-10-06 15:21,45.0,
2020-10-06 15:31,35.0,
2020-10-06 15:41,25.0,TEMPORAL
2020-10-06 15:51,30.0,
"""),
)
def test_symmetric_return_value(self):
return_value = rocc(
timeseries=self.ahtimeseries,
thresholds=(
Threshold("10min", 10),
Threshold("20min", 15),
Threshold("H", 40),
),
symmetric=True,
)
self.assertEqual(len(return_value), 2)
self.assertEqual(return_value[0],
"2020-10-06T14:50 -11.0 in 10min (< -10.0)")
self.assertEqual(return_value[1],
"2020-10-06T15:41 -20.0 in 20min (< -15.0)")
def test_update(self, mock_api_client):
timeseries_group = [
{
"base_url": "https://mydomain.com",
"station_id": 2,
"timeseries_id": 42,
"user": "******",
"password": "******",
"file": "file1",
},
{
"base_url": "https://mydomain.com",
"station_id": 3,
"timeseries_id": 43,
"user": "******",
"password": "******",
"file": "file2",
},
]
# Cache the two timeseries
cache = TimeseriesCache(timeseries_group)
cache.update()
# Check that the cached stuff is what it should be
with open("file1", newline="\n") as f:
ts1_before = HTimeseries(f)
self.assertEqual(ts1_before.time_step, "D")
c = StringIO()
ts1_before.write(c)
self.assertEqual(c.getvalue().replace("\r", ""),
test_timeseries["42_top"])
with open("file2", newline="\n") as f:
ts2_before = HTimeseries(f)
self.assertEqual(ts2_before.time_step, "D")
c = StringIO()
ts2_before.write(c)
self.assertEqual(c.getvalue().replace("\r", ""),
test_timeseries["43_top"])
# Update the cache
cache.update()
# Check that the cached stuff is what it should be
with open("file1", newline="\n") as f:
ts1_after = HTimeseries(f)
self.assertEqual(ts1_after.time_step, "D")
c = StringIO()
ts1_after.write(c)
self.assertEqual(c.getvalue().replace("\r", ""),
test_timeseries["42_all"])
with open("file2", newline="\n") as f:
ts2_after = HTimeseries(f)
self.assertEqual(ts2_after.time_step, "D")
c = StringIO()
ts2_after.write(c)
self.assertEqual(c.getvalue().replace("\r", ""),
test_timeseries["43_all"])
# Check that the time series comments are the same before and after
self.assertEqual(ts1_before.comment, ts1_after.comment)
self.assertEqual(ts2_before.comment, ts2_after.comment)