def test_execute(self):
application = AggregateApp()
application.read_command_line()
application.read_configuration()
# Verify the output files don't exist yet
self.assertFalse(os.path.exists(self.output_filenames[0]))
self.assertFalse(os.path.exists(self.output_filenames[1]))
# Execute
application.run()
# Check that it has created two files
self.assertTrue(os.path.exists(self.output_filenames[0]))
self.assertTrue(os.path.exists(self.output_filenames[1]))
# Check that the created time series are correct
t = Timeseries()
with open(self.output_filenames[0]) as f:
t.read_file(f)
self.assertEqual(t.timezone, 'EET (UTC+0200)')
self.assertEqual(len(t), 1)
self.assertAlmostEqual(t['2014-06-16 16:00'], 114.9, places=5)
t = Timeseries()
with open(self.output_filenames[1]) as f:
t.read_file(f)
self.assertEqual(t.timezone, '')
self.assertEqual(len(t), 1)
self.assertAlmostEqual(t['2014-06-17 16:00'], 50.8167, places=5)
def read_timeseries_from_cache_file(self):
result = Timeseries()
if os.path.exists(self.filename):
with open(self.filename) as f:
try:
result.read_file(f)
except ValueError:
# File may be corrupted; continue with empty time series
result = Timeseries()
return result
def execute_item(self, item):
source_ts = Timeseries()
with open(item['source_file']) as f:
source_ts.read_file(f)
self.target_step.interval_type = item['interval_type']
target_ts, missing = source_ts.aggregate(
self.target_step,
missing_allowed=self.missing_allowed,
missing_flag=self.missing_flag)
with open(item['target_file'], 'w') as f:
target_ts.write_file(f, version=3)
def append_newer_timeseries(self, start_date, ts1):
self.session_cookies = enhydris_api.login(self.base_url, self.user,
self.password)
url = self.base_url + 'timeseries/d/{}/download/{}/?version=3'.format(
self.timeseries_id, start_date.isoformat())
r = requests.get(url, cookies=self.session_cookies)
if r.status_code != 200:
raise HTTPError('Error {} while getting {}'.format(r.status_code,
url))
responseio = StringIO(r.text)
ts2 = Timeseries()
ts2.read_file(responseio)
responseio.seek(0)
ts1.read_meta(responseio)
ts1.append(ts2)
def h_integrate(mask, stations_layer, date, output_filename_prefix, date_fmt,
funct, kwargs):
date_fmt_for_filename = date.strftime(date_fmt).replace(' ', '-').replace(
':', '-')
output_filename = '{}-{}.tif'.format(output_filename_prefix,
date.strftime(date_fmt_for_filename))
if not _needs_calculation(output_filename, date, stations_layer):
return
# Read the time series values and add the 'value' attribute to
# stations_layer
stations_layer.CreateField(ogr.FieldDefn('value', ogr.OFTReal))
input_files = []
stations_layer.ResetReading()
for station in stations_layer:
filename = station.GetField('filename')
t = Timeseries()
with open(filename) as f:
t.read_file(f)
value = t.get(date, float('NaN'))
station.SetField('value', value)
if not isnan(value):
input_files.append(filename)
stations_layer.SetFeature(station)
if not input_files:
return
# Create destination data source
output = gdal.GetDriverByName('GTiff').Create(
output_filename, mask.RasterXSize, mask.RasterYSize, 1,
gdal.GDT_Float32)
output.SetMetadataItem('TIMESTAMP', date.strftime(date_fmt))
output.SetMetadataItem('INPUT_FILES', '\n'.join(input_files))
try:
# Set geotransform and projection in the output data source
output.SetGeoTransform(mask.GetGeoTransform())
output.SetProjection(mask.GetProjection())
# Do the integration
integrate(mask, stations_layer, output.GetRasterBand(1), funct, kwargs)
finally:
# Close the dataset
output = None
def _needs_calculation(output_filename, date, stations_layer):
"""
Used by h_integrate to check whether the output file needs to be calculated
or not. It does not need to be calculated if it already exists and has been
calculated from all available data.
"""
# Return immediately if output file does not exist
if not os.path.exists(output_filename):
return True
# Get list of files which were used to calculate the output file
fp = gdal.Open(output_filename)
try:
actual_input_files = fp.GetMetadataItem('INPUT_FILES')
if actual_input_files is None:
raise IOError('{} does not contain the metadata item INPUT_FILES'
.format(output_filename))
finally:
fp = None # Close file
actual_input_files = set(actual_input_files.split('\n'))
# Get list of files available for calculating the output file
stations_layer.ResetReading()
available_input_files = set(
[station.GetField('filename') for station in stations_layer
if os.path.exists(station.GetField('filename'))])
# Which of these files have not been used?
unused_files = available_input_files - actual_input_files
# For each one of these files, check whether it has newly available data.
# Upon finding one that does, the verdict is made: return True
for filename in unused_files:
t = Timeseries()
with open(filename) as f:
t.read_file(f)
value = t.get(date, float('NaN'))
if not isnan(value):
return True
# We were unable to find data that had not already been used
return False
def update_one_timeseries(base_url, id, user=None, password=None):
if base_url[-1] != '/':
base_url += '/'
# Read timeseries from cache file
cache_filename = os.path.join(
settings.BITIA_TIMESERIES_CACHE_DIR, '{}.hts'.format(id))
ts1 = Timeseries()
if os.path.exists(cache_filename):
with open(cache_filename) as f:
try:
ts1.read_file(f)
except ValueError:
# File may be corrupted; continue with empty time series
ts1 = Timeseries()
# Get its end date
try:
end_date = ts1.bounding_dates()[1]
except TypeError:
# Timeseries is totally empty; no start and end date
end_date = datetime(1, 1, 1, 0, 0)
start_date = end_date + timedelta(minutes=1)
# Get newer timeseries and append it
session_cookies = enhydris_api.login(base_url, user, password)
url = base_url + 'timeseries/d/{}/download/{}/'.format(
id, start_date.isoformat())
r = requests.get(url, cookies=session_cookies)
r.raise_for_status()
ts2 = Timeseries()
ts2.read_file(StringIO(r.text))
ts1.append(ts2)
# Save it
with open(cache_filename, 'w') as f:
ts1.write(f)
class SoilWaterBalanceDailyTestCase(TestCase):
# Case Study: IRMA_DW_Balance3.ods
def setUp(self):
# Water Balance 3
self.precip = Timeseries(time_step=TimeStep(1440, 0))
instring = StringIO(precipitation_test_daily_timeseries)
self.precip.read_file(instring)
self.evap = Timeseries(time_step=Timeseries(1440, 0))
instring = StringIO(evapotranspiration_test_daily_timeseries)
self.evap.read_file(instring)
# Parameters
self.fc = 0.287
self.wp = 0.140
self.rd = 0.5
self.kc = 0.7
self.p = 0.5
self.peff = 0.8
self.irr_eff = 0.8
self.theta_s = 0.425
self.rd_factor = 1000
# swb instance
self.swb = SoilWaterBalance(self.fc, self.wp, self.rd,
self.kc, self.p, self.peff, self.irr_eff,
self.theta_s, self.precip, self.evap,
self.rd_factor)
def test_swb_daily_general(self):
start_date = datetime(2008, 8, 1)
end_date = datetime(2008, 8, 30)
theta_init = self.swb.fc_mm
irr_event_days = []
self.swb.water_balance(theta_init, irr_event_days,
start_date, end_date, FC_IRT=1)
# Parameters
self.assertEqual(self.swb.fc_mm, 143.5)
self.assertEqual(self.swb.wp_mm, 70.0)
self.assertEqual(self.swb.theta_s_mm, 212.5)
self.assertAlmostEqual(self.swb.taw, 0.147)
self.assertEqual(self.swb.taw_mm, 73.5)
self.assertAlmostEqual(self.swb.raw, 0.0735)
self.assertEqual(self.swb.raw_mm, 36.75)
self.assertEqual(self.swb.lowlim_theta, 0.2135)
self.assertEqual(self.swb.lowlim_theta_mm, 106.75)
# Check timedelta
self.assertEqual(self.swb.__get_timedelta__(), timedelta(1))
# i = 1
values1 = self.swb.wbm_report[0]
self.assertEqual(values1['ETc'], 4.837)
self.assertAlmostEqual(values1['Dr_i'], 4.837)
self.assertAlmostEqual(values1['Dr_1_next'], 4.837)
self.assertEqual(values1['theta'], 138.663)
self.assertEqual(values1['Peff'], 0.0)
self.assertEqual(values1['RO'], 0.0)
self.assertEqual(values1['Inet'], 0.0)
self.assertEqual(values1['Ks'], 1)
self.assertEqual(values1['SWB'], 4.837)
# Catch Inet
values2 = self.swb.wbm_report[9]
self.assertEqual(values2['date'], datetime(2008, 8, 10, 0, 0))
self.assertEqual(values2['irrigate'], 1.0)
self.assertAlmostEqual(values2['Inet'], 40.124)
self.assertEqual(values2['theta'], 143.500)
values3 = self.swb.wbm_report[19]
self.assertEqual(values3['date'], datetime(2008, 8, 20, 0, 0))
self.assertEqual(values3['irrigate'], 1.0)
self.assertAlmostEqual(values3['Inet'], 38.3810)
self.assertEqual(values3['theta'], 143.500)
# end_date
values4 = self.swb.wbm_report[29]
self.assertEqual(values4['date'], datetime(2008, 8, 30, 0, 0))
self.assertEqual(values3['irrigate'], 1.0)
self.assertEqual(values4['Dr_i'], 36.169)
self.assertEqual(values4['theta'], 107.3310)
def test_swb_daily_with_no_irrigation_date(self):
start_date = datetime(2008, 8, 1)
end_date = datetime(2008, 8, 30)
theta_init = self.swb.fc_mm - 0.75 * self.swb.raw_mm
self.assertEqual(theta_init, 115.9375)
irr_event_days = []
self.swb.water_balance(theta_init, irr_event_days,
start_date, end_date, FC_IRT=1)
# i = 1
values1 = self.swb.wbm_report[0]
self.assertEqual(values1['ETc'], 4.837)
self.assertAlmostEqual(values1['Dr_i'], 32.39950)
self.assertAlmostEqual(values1['Dr_1_next'], 32.39950)
self.assertEqual(values1['theta'], 111.100500)
self.assertEqual(values1['Peff'], 0.0)
self.assertEqual(values1['RO'], 0.0)
self.assertEqual(values1['Inet'], 0.0)
self.assertEqual(values1['Ks'], 1)
self.assertEqual(values1['SWB'], 4.837)
# Catch Inet
values2 = self.swb.wbm_report[11]
self.assertEqual(values2['date'], datetime(2008, 8, 12, 0, 0))
self.assertEqual(values2['irrigate'], 1.0)
self.assertAlmostEqual(values2['Inet'], 38.626000)
self.assertEqual(values2['theta'], 143.500)
values3 = self.swb.wbm_report[21]
self.assertEqual(values3['date'], datetime(2008, 8, 22, 0, 0))
self.assertEqual(values3['irrigate'], 1.0)
self.assertAlmostEqual(values3['Inet'], 38.3810)
self.assertEqual(values3['theta'], 143.500)
# end_date
values4 = self.swb.wbm_report[29]
self.assertEqual(values4['date'], datetime(2008, 8, 30, 0, 0))
self.assertEqual(values3['irrigate'], 1.0)
self.assertAlmostEqual(values4['Dr_i'], 28.46200)
self.assertAlmostEqual(values4['theta'], 115.038)
def test_swb_daily_with_irrigations_dates(self):
start_date = datetime(2008, 8, 1)
end_date = datetime(2008, 8, 30)
theta_init = 131.93750
irr_event_days = [datetime(2008, 8, 8, 0, 0),
datetime(2008, 8, 15, 0, 0)]
self.swb.water_balance(theta_init, irr_event_days,
start_date, end_date, FC_IRT=1)
values1 = self.swb.wbm_report[0]
self.assertEqual(values1['ETc'], 4.837)
self.assertEqual(values1['Inet'], 0.0)
self.assertEqual(values1['SWB'], 4.837)
self.assertAlmostEqual(values1['Dr_i'], 16.39950)
self.assertAlmostEqual(values1['Dr_1_next'], 16.39950)
self.assertEqual(values1['Peff'], 0.0)
self.assertEqual(values1['RO'], 0.0)
self.assertEqual(values1['Inet'], 0.0)
self.assertEqual(values1['Ks'], 1)
self.assertEqual(values1['theta'], 127.1005)
# Catch Inet
values2 = self.swb.wbm_report[14]
self.assertEqual(values2['date'], datetime(2008, 8, 15, 0, 0))
self.assertEqual(values2['irrigate'], 0.0)
self.assertAlmostEqual(values2['Dr_i'], 26.634999999999998)
self.assertEqual(values2['Inet'], 26.634999999999998)
self.assertEqual(values2['theta'], 143.5)
values3 = self.swb.wbm_report[-1]
self.assertEqual(values3['date'], datetime(2008, 8, 30, 0, 0))
self.assertEqual(values3['irrigate'], 1.0)
self.assertAlmostEqual(values3['Dr_i'], 51.83859483)
self.assertAlmostEqual(values3['theta'], 91.66140517)
def test_swb_daily_Inet_in_wrong_input(self):
start_date = datetime(2008, 8, 1)
end_date = datetime(2008, 8, 30)
theta_init = 131.93750
irr_event_days = [datetime(2008, 8, 8, 0, 0),
datetime(2008, 8, 15, 0, 0)]
with self.assertRaises(ValueError):
self.swb.water_balance(theta_init, irr_event_days,
start_date, end_date,
FC_IRT=1, Inet_in="Something Else")
def test_update(self):
self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
timeseries_group = [{'base_url': self.parms['base_url'],
'id': self.ts1_id,
'user': self.parms['user'],
'password': self.parms['password'],
'file': 'file1',
},
{'base_url': self.parms['base_url'],
'id': self.ts2_id,
'user': self.parms['user'],
'password': self.parms['password'],
'file': 'file2',
},
]
# Cache the two timeseries
cache = TimeseriesCache(timeseries_group)
cache.update()
# Check that the cached stuff is what it should be
with open('file1') as f:
ts1_before = Timeseries()
ts1_before.read_file(f)
self.assertEqual(ts1_before.time_step.length_minutes, 1440)
self.assertEqual(ts1_before.time_step.length_months, 0)
c = StringIO()
ts1_before.write(c)
self.assertEqual(c.getvalue().replace('\r', ''),
self.timeseries1_top)
with open('file2') as f:
ts2_before = Timeseries()
ts2_before.read_file(f)
self.assertEqual(ts2_before.time_step.length_minutes, 1440)
self.assertEqual(ts2_before.time_step.length_months, 0)
c = StringIO()
ts2_before.write(c)
self.assertEqual(c.getvalue().replace('\r', ''),
self.timeseries2_top)
# Append a record to the database for each timeseries
ts = Timeseries(self.ts1_id)
ts.read(StringIO(self.timeseries1_bottom))
enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
ts = Timeseries(self.ts2_id)
ts.read(StringIO(self.timeseries2_bottom))
enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
# Update the cache
cache.update()
# Check that the cached stuff is what it should be
with open('file1') as f:
ts1_after = Timeseries()
ts1_after.read_file(f)
self.assertEqual(ts1_after.time_step.length_minutes, 1440)
self.assertEqual(ts1_after.time_step.length_months, 0)
c = StringIO()
ts1_after.write(c)
self.assertEqual(c.getvalue().replace('\r', ''),
self.test_timeseries1)
with open('file2') as f:
ts2_after = Timeseries()
ts2_after.read_file(f)
self.assertEqual(ts2_after.time_step.length_minutes, 1440)
self.assertEqual(ts2_after.time_step.length_months, 0)
c = StringIO()
ts2_after.write(c)
self.assertEqual(c.getvalue().replace('\r', ''),
self.test_timeseries2)
# Check that the time series comments are the same before and after
self.assertEqual(ts1_before.comment, ts1_after.comment)
self.assertEqual(ts2_before.comment, ts2_after.comment)
def execute_point(self):
# Make sure elevation has not been specified
if self.elevation is not None:
raise InvalidOptionError(
'elevation should only be specified in the configuration file '
'in spatial calculation')
# Read input time series
input_timeseries = {}
for name in ('temperature_min', 'temperature_max', 'temperature',
'humidity', 'humidity_min', 'humidity_max', 'wind_speed',
'pressure', 'solar_radiation', 'sunshine_duration'):
filename = self.config['General'][name + '_prefix'] + '.hts'
if not os.path.exists(filename):
continue
t = Timeseries()
with open(filename, 'r') as f:
t.read_file(f)
input_timeseries[name] = t
# Make sure all are in the same location and timezone
first = True
for index in input_timeseries:
t = input_timeseries[index]
if first:
abscissa = t.location['abscissa']
ordinate = t.location['ordinate']
srid = t.location['srid']
altitude = t.location['altitude']
timezone = t.timezone
first = False
continue
if abs(abscissa - t.location['abscissa']) > 1e-7 or \
abs(ordinate - t.location['ordinate']) > 1e-7 or \
srid != t.location['srid'] or \
abs(altitude - t.location['altitude']) > 1e-2 or \
timezone != t.timezone:
raise ValueError('Incorrect or unspecified or inconsistent '
'locations or time zones in the time series '
'files.')
# Convert location to WGS84
source_projection = osr.SpatialReference()
source_projection.ImportFromEPSG(srid)
wgs84 = osr.SpatialReference()
wgs84.ImportFromEPSG(4326)
transform = osr.CoordinateTransformation(source_projection, wgs84)
apoint = ogr.Geometry(ogr.wkbPoint)
apoint.AddPoint(abscissa, ordinate)
apoint.Transform(transform)
latitude, longitude = apoint.GetY(), apoint.GetY()
# Prepare the Penman Monteith method
nsrr = self.nighttime_solar_radiation_ratio
self.penman_monteith = PenmanMonteith(
albedo=self.albedo,
nighttime_solar_radiation_ratio=nsrr,
elevation=altitude,
latitude=latitude,
longitude=longitude,
step_length=self.step,
unit_converters=self.unit_converters)
# Create output timeseries object
precision = 2 if self.step == timedelta(hours=1) else 1
pet = Timeseries(
time_step=TimeStep(length_minutes=self.step.total_seconds() / 60),
unit='mm',
timezone=timezone,
variable='Potential Evapotranspiration',
precision=precision,
location={'abscissa': abscissa, 'ordinate': ordinate,
'srid': srid, 'altitude': altitude})
# Let's see what variables we are going to use in the calculation,
# based mainly on the step.
if self.step == timedelta(hours=1):
variables = ('temperature', 'humidity', 'wind_speed',
'solar_radiation')
elif self.step == timedelta(days=1):
variables = (
'temperature_max', 'temperature_min', 'humidity_max',
'humidity_min', 'wind_speed',
'solar_radiation' if 'solar_radiation' in input_timeseries
else 'sunshine_duration')
else:
raise Exception(
'Internal error: step should have been checked already')
# Calculate evaporation
for adatetime in input_timeseries['wind_speed']:
try:
kwargs = {v: input_timeseries[v][adatetime]
for v in variables}
except (IndexError, KeyError):
continue
kwargs['adatetime'] = adatetime
pet[adatetime] = self.penman_monteith.calculate(**kwargs)
# Save result
outfilename = self.config['General']['evaporation_prefix'] + '.hts'
with open(outfilename, 'w') as f:
pet.write_file(f)
def test_daily(self):
self.setup_input_file('temperature_max',
textwrap.dedent("""\
Title=Temperature Max
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,21.5,
"""))
self.setup_input_file('temperature_min',
textwrap.dedent("""\
Title=Temperature Min
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,12.3,
"""))
self.setup_input_file('humidity_max',
textwrap.dedent("""\
Title=Humidity Max
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,84.0,
"""))
self.setup_input_file('humidity_min',
textwrap.dedent("""\
Title=Humidity Min
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,63.0,
"""))
self.setup_input_file('wind_speed',
textwrap.dedent("""\
Title=Wind speed
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,2.078,
"""))
self.setup_input_file('sunshine_duration',
textwrap.dedent("""\
Title=Sunshine duration
Location=-16.25 16.217 4326
Altitude=100
2014-07-06,9.25,
"""))
with open(self.config_file, 'w') as f:
f.write(textwrap.dedent('''\
base_dir = {self.tempdir}
albedo = 0.23
step_length = 1440
''').format(self=self))
application = VaporizeApp()
application.read_command_line()
application.read_configuration()
# Verify the output file doesn't exist yet
result_filename = os.path.join(self.tempdir, 'evaporation.hts')
self.assertFalse(os.path.exists(result_filename))
# Execute
application.run()
# Check that it has created a file and that the file is correct
t = Timeseries()
with open(result_filename) as f:
t.read_file(f)
self.assertEqual(len(t), 1)
adate = datetime(2014, 7, 6)
self.assertEqual(t.bounding_dates(), (adate, adate))
self.assertAlmostEqual(t[adate], 3.9)
def test_hourly(self):
self.setup_input_file('temperature',
textwrap.dedent("""\
Title=Temperature
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,38,
"""))
self.setup_input_file('humidity',
textwrap.dedent("""\
Title=Humidity
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,52,
"""))
self.setup_input_file('wind_speed',
textwrap.dedent("""\
Title=Wind speed
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,3.3,
"""))
self.setup_input_file('pressure',
textwrap.dedent("""\
Title=Pressure
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,1013.0,
"""))
self.setup_input_file('solar_radiation',
textwrap.dedent("""\
Title=Solar radiation
Location=-16.25 16.217 4326
Altitude=8
Timezone=CVT (UTC-0100)
2014-10-01 15:00,681.0,
"""))
with open(self.config_file, 'w') as f:
f.write(textwrap.dedent('''\
base_dir = {self.tempdir}
albedo = 0.23
nighttime_solar_radiation_ratio = 0.8
step_length = 60
unit_converter_pressure = x / 10.0
unit_converter_solar_radiation = x * 3600 / 1e6
''').format(self=self))
application = VaporizeApp()
application.read_command_line()
application.read_configuration()
# Verify the output file doesn't exist yet
result_filename = os.path.join(self.tempdir, 'evaporation.hts')
self.assertFalse(os.path.exists(result_filename))
# Execute
application.run()
# Check that it has created a file and that the file is correct
t = Timeseries()
with open(result_filename) as f:
t.read_file(f)
self.assertEqual(len(t), 1)
adate = datetime(2014, 10, 1, 15, 0, tzinfo=senegal_tzinfo)
self.assertEqual(t.bounding_dates(), (adate, adate))
self.assertAlmostEqual(t[adate], 0.63)
