def test_execute(self): application = AggregateApp() application.read_command_line() application.read_configuration() # Verify the output files don't exist yet self.assertFalse(os.path.exists(self.output_filenames[0])) self.assertFalse(os.path.exists(self.output_filenames[1])) # Execute application.run() # Check that it has created two files self.assertTrue(os.path.exists(self.output_filenames[0])) self.assertTrue(os.path.exists(self.output_filenames[1])) # Check that the created time series are correct t = Timeseries() with open(self.output_filenames[0]) as f: t.read_file(f) self.assertEqual(t.timezone, 'EET (UTC+0200)') self.assertEqual(len(t), 1) self.assertAlmostEqual(t['2014-06-16 16:00'], 114.9, places=5) t = Timeseries() with open(self.output_filenames[1]) as f: t.read_file(f) self.assertEqual(t.timezone, '') self.assertEqual(len(t), 1) self.assertAlmostEqual(t['2014-06-17 16:00'], 50.8167, places=5)
def read_timeseries_from_cache_file(self): result = Timeseries() if os.path.exists(self.filename): with open(self.filename) as f: try: result.read_file(f) except ValueError: # File may be corrupted; continue with empty time series result = Timeseries() return result
def execute_item(self, item): source_ts = Timeseries() with open(item['source_file']) as f: source_ts.read_file(f) self.target_step.interval_type = item['interval_type'] target_ts, missing = source_ts.aggregate( self.target_step, missing_allowed=self.missing_allowed, missing_flag=self.missing_flag) with open(item['target_file'], 'w') as f: target_ts.write_file(f, version=3)
def append_newer_timeseries(self, start_date, ts1): self.session_cookies = enhydris_api.login(self.base_url, self.user, self.password) url = self.base_url + 'timeseries/d/{}/download/{}/?version=3'.format( self.timeseries_id, start_date.isoformat()) r = requests.get(url, cookies=self.session_cookies) if r.status_code != 200: raise HTTPError('Error {} while getting {}'.format(r.status_code, url)) responseio = StringIO(r.text) ts2 = Timeseries() ts2.read_file(responseio) responseio.seek(0) ts1.read_meta(responseio) ts1.append(ts2)
def h_integrate(mask, stations_layer, date, output_filename_prefix, date_fmt, funct, kwargs): date_fmt_for_filename = date.strftime(date_fmt).replace(' ', '-').replace( ':', '-') output_filename = '{}-{}.tif'.format(output_filename_prefix, date.strftime(date_fmt_for_filename)) if not _needs_calculation(output_filename, date, stations_layer): return # Read the time series values and add the 'value' attribute to # stations_layer stations_layer.CreateField(ogr.FieldDefn('value', ogr.OFTReal)) input_files = [] stations_layer.ResetReading() for station in stations_layer: filename = station.GetField('filename') t = Timeseries() with open(filename) as f: t.read_file(f) value = t.get(date, float('NaN')) station.SetField('value', value) if not isnan(value): input_files.append(filename) stations_layer.SetFeature(station) if not input_files: return # Create destination data source output = gdal.GetDriverByName('GTiff').Create( output_filename, mask.RasterXSize, mask.RasterYSize, 1, gdal.GDT_Float32) output.SetMetadataItem('TIMESTAMP', date.strftime(date_fmt)) output.SetMetadataItem('INPUT_FILES', '\n'.join(input_files)) try: # Set geotransform and projection in the output data source output.SetGeoTransform(mask.GetGeoTransform()) output.SetProjection(mask.GetProjection()) # Do the integration integrate(mask, stations_layer, output.GetRasterBand(1), funct, kwargs) finally: # Close the dataset output = None
def _needs_calculation(output_filename, date, stations_layer): """ Used by h_integrate to check whether the output file needs to be calculated or not. It does not need to be calculated if it already exists and has been calculated from all available data. """ # Return immediately if output file does not exist if not os.path.exists(output_filename): return True # Get list of files which were used to calculate the output file fp = gdal.Open(output_filename) try: actual_input_files = fp.GetMetadataItem('INPUT_FILES') if actual_input_files is None: raise IOError('{} does not contain the metadata item INPUT_FILES' .format(output_filename)) finally: fp = None # Close file actual_input_files = set(actual_input_files.split('\n')) # Get list of files available for calculating the output file stations_layer.ResetReading() available_input_files = set( [station.GetField('filename') for station in stations_layer if os.path.exists(station.GetField('filename'))]) # Which of these files have not been used? unused_files = available_input_files - actual_input_files # For each one of these files, check whether it has newly available data. # Upon finding one that does, the verdict is made: return True for filename in unused_files: t = Timeseries() with open(filename) as f: t.read_file(f) value = t.get(date, float('NaN')) if not isnan(value): return True # We were unable to find data that had not already been used return False
def update_one_timeseries(base_url, id, user=None, password=None): if base_url[-1] != '/': base_url += '/' # Read timeseries from cache file cache_filename = os.path.join( settings.BITIA_TIMESERIES_CACHE_DIR, '{}.hts'.format(id)) ts1 = Timeseries() if os.path.exists(cache_filename): with open(cache_filename) as f: try: ts1.read_file(f) except ValueError: # File may be corrupted; continue with empty time series ts1 = Timeseries() # Get its end date try: end_date = ts1.bounding_dates()[1] except TypeError: # Timeseries is totally empty; no start and end date end_date = datetime(1, 1, 1, 0, 0) start_date = end_date + timedelta(minutes=1) # Get newer timeseries and append it session_cookies = enhydris_api.login(base_url, user, password) url = base_url + 'timeseries/d/{}/download/{}/'.format( id, start_date.isoformat()) r = requests.get(url, cookies=session_cookies) r.raise_for_status() ts2 = Timeseries() ts2.read_file(StringIO(r.text)) ts1.append(ts2) # Save it with open(cache_filename, 'w') as f: ts1.write(f)
class SoilWaterBalanceDailyTestCase(TestCase): # Case Study: IRMA_DW_Balance3.ods def setUp(self): # Water Balance 3 self.precip = Timeseries(time_step=TimeStep(1440, 0)) instring = StringIO(precipitation_test_daily_timeseries) self.precip.read_file(instring) self.evap = Timeseries(time_step=Timeseries(1440, 0)) instring = StringIO(evapotranspiration_test_daily_timeseries) self.evap.read_file(instring) # Parameters self.fc = 0.287 self.wp = 0.140 self.rd = 0.5 self.kc = 0.7 self.p = 0.5 self.peff = 0.8 self.irr_eff = 0.8 self.theta_s = 0.425 self.rd_factor = 1000 # swb instance self.swb = SoilWaterBalance(self.fc, self.wp, self.rd, self.kc, self.p, self.peff, self.irr_eff, self.theta_s, self.precip, self.evap, self.rd_factor) def test_swb_daily_general(self): start_date = datetime(2008, 8, 1) end_date = datetime(2008, 8, 30) theta_init = self.swb.fc_mm irr_event_days = [] self.swb.water_balance(theta_init, irr_event_days, start_date, end_date, FC_IRT=1) # Parameters self.assertEqual(self.swb.fc_mm, 143.5) self.assertEqual(self.swb.wp_mm, 70.0) self.assertEqual(self.swb.theta_s_mm, 212.5) self.assertAlmostEqual(self.swb.taw, 0.147) self.assertEqual(self.swb.taw_mm, 73.5) self.assertAlmostEqual(self.swb.raw, 0.0735) self.assertEqual(self.swb.raw_mm, 36.75) self.assertEqual(self.swb.lowlim_theta, 0.2135) self.assertEqual(self.swb.lowlim_theta_mm, 106.75) # Check timedelta self.assertEqual(self.swb.__get_timedelta__(), timedelta(1)) # i = 1 values1 = self.swb.wbm_report[0] self.assertEqual(values1['ETc'], 4.837) self.assertAlmostEqual(values1['Dr_i'], 4.837) self.assertAlmostEqual(values1['Dr_1_next'], 4.837) self.assertEqual(values1['theta'], 138.663) self.assertEqual(values1['Peff'], 0.0) self.assertEqual(values1['RO'], 0.0) self.assertEqual(values1['Inet'], 0.0) self.assertEqual(values1['Ks'], 1) self.assertEqual(values1['SWB'], 4.837) # Catch Inet values2 = self.swb.wbm_report[9] self.assertEqual(values2['date'], datetime(2008, 8, 10, 0, 0)) self.assertEqual(values2['irrigate'], 1.0) self.assertAlmostEqual(values2['Inet'], 40.124) self.assertEqual(values2['theta'], 143.500) values3 = self.swb.wbm_report[19] self.assertEqual(values3['date'], datetime(2008, 8, 20, 0, 0)) self.assertEqual(values3['irrigate'], 1.0) self.assertAlmostEqual(values3['Inet'], 38.3810) self.assertEqual(values3['theta'], 143.500) # end_date values4 = self.swb.wbm_report[29] self.assertEqual(values4['date'], datetime(2008, 8, 30, 0, 0)) self.assertEqual(values3['irrigate'], 1.0) self.assertEqual(values4['Dr_i'], 36.169) self.assertEqual(values4['theta'], 107.3310) def test_swb_daily_with_no_irrigation_date(self): start_date = datetime(2008, 8, 1) end_date = datetime(2008, 8, 30) theta_init = self.swb.fc_mm - 0.75 * self.swb.raw_mm self.assertEqual(theta_init, 115.9375) irr_event_days = [] self.swb.water_balance(theta_init, irr_event_days, start_date, end_date, FC_IRT=1) # i = 1 values1 = self.swb.wbm_report[0] self.assertEqual(values1['ETc'], 4.837) self.assertAlmostEqual(values1['Dr_i'], 32.39950) self.assertAlmostEqual(values1['Dr_1_next'], 32.39950) self.assertEqual(values1['theta'], 111.100500) self.assertEqual(values1['Peff'], 0.0) self.assertEqual(values1['RO'], 0.0) self.assertEqual(values1['Inet'], 0.0) self.assertEqual(values1['Ks'], 1) self.assertEqual(values1['SWB'], 4.837) # Catch Inet values2 = self.swb.wbm_report[11] self.assertEqual(values2['date'], datetime(2008, 8, 12, 0, 0)) self.assertEqual(values2['irrigate'], 1.0) self.assertAlmostEqual(values2['Inet'], 38.626000) self.assertEqual(values2['theta'], 143.500) values3 = self.swb.wbm_report[21] self.assertEqual(values3['date'], datetime(2008, 8, 22, 0, 0)) self.assertEqual(values3['irrigate'], 1.0) self.assertAlmostEqual(values3['Inet'], 38.3810) self.assertEqual(values3['theta'], 143.500) # end_date values4 = self.swb.wbm_report[29] self.assertEqual(values4['date'], datetime(2008, 8, 30, 0, 0)) self.assertEqual(values3['irrigate'], 1.0) self.assertAlmostEqual(values4['Dr_i'], 28.46200) self.assertAlmostEqual(values4['theta'], 115.038) def test_swb_daily_with_irrigations_dates(self): start_date = datetime(2008, 8, 1) end_date = datetime(2008, 8, 30) theta_init = 131.93750 irr_event_days = [datetime(2008, 8, 8, 0, 0), datetime(2008, 8, 15, 0, 0)] self.swb.water_balance(theta_init, irr_event_days, start_date, end_date, FC_IRT=1) values1 = self.swb.wbm_report[0] self.assertEqual(values1['ETc'], 4.837) self.assertEqual(values1['Inet'], 0.0) self.assertEqual(values1['SWB'], 4.837) self.assertAlmostEqual(values1['Dr_i'], 16.39950) self.assertAlmostEqual(values1['Dr_1_next'], 16.39950) self.assertEqual(values1['Peff'], 0.0) self.assertEqual(values1['RO'], 0.0) self.assertEqual(values1['Inet'], 0.0) self.assertEqual(values1['Ks'], 1) self.assertEqual(values1['theta'], 127.1005) # Catch Inet values2 = self.swb.wbm_report[14] self.assertEqual(values2['date'], datetime(2008, 8, 15, 0, 0)) self.assertEqual(values2['irrigate'], 0.0) self.assertAlmostEqual(values2['Dr_i'], 26.634999999999998) self.assertEqual(values2['Inet'], 26.634999999999998) self.assertEqual(values2['theta'], 143.5) values3 = self.swb.wbm_report[-1] self.assertEqual(values3['date'], datetime(2008, 8, 30, 0, 0)) self.assertEqual(values3['irrigate'], 1.0) self.assertAlmostEqual(values3['Dr_i'], 51.83859483) self.assertAlmostEqual(values3['theta'], 91.66140517) def test_swb_daily_Inet_in_wrong_input(self): start_date = datetime(2008, 8, 1) end_date = datetime(2008, 8, 30) theta_init = 131.93750 irr_event_days = [datetime(2008, 8, 8, 0, 0), datetime(2008, 8, 15, 0, 0)] with self.assertRaises(ValueError): self.swb.water_balance(theta_init, irr_event_days, start_date, end_date, FC_IRT=1, Inet_in="Something Else")
def test_update(self): self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API')) timeseries_group = [{'base_url': self.parms['base_url'], 'id': self.ts1_id, 'user': self.parms['user'], 'password': self.parms['password'], 'file': 'file1', }, {'base_url': self.parms['base_url'], 'id': self.ts2_id, 'user': self.parms['user'], 'password': self.parms['password'], 'file': 'file2', }, ] # Cache the two timeseries cache = TimeseriesCache(timeseries_group) cache.update() # Check that the cached stuff is what it should be with open('file1') as f: ts1_before = Timeseries() ts1_before.read_file(f) self.assertEqual(ts1_before.time_step.length_minutes, 1440) self.assertEqual(ts1_before.time_step.length_months, 0) c = StringIO() ts1_before.write(c) self.assertEqual(c.getvalue().replace('\r', ''), self.timeseries1_top) with open('file2') as f: ts2_before = Timeseries() ts2_before.read_file(f) self.assertEqual(ts2_before.time_step.length_minutes, 1440) self.assertEqual(ts2_before.time_step.length_months, 0) c = StringIO() ts2_before.write(c) self.assertEqual(c.getvalue().replace('\r', ''), self.timeseries2_top) # Append a record to the database for each timeseries ts = Timeseries(self.ts1_id) ts.read(StringIO(self.timeseries1_bottom)) enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts) ts = Timeseries(self.ts2_id) ts.read(StringIO(self.timeseries2_bottom)) enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts) # Update the cache cache.update() # Check that the cached stuff is what it should be with open('file1') as f: ts1_after = Timeseries() ts1_after.read_file(f) self.assertEqual(ts1_after.time_step.length_minutes, 1440) self.assertEqual(ts1_after.time_step.length_months, 0) c = StringIO() ts1_after.write(c) self.assertEqual(c.getvalue().replace('\r', ''), self.test_timeseries1) with open('file2') as f: ts2_after = Timeseries() ts2_after.read_file(f) self.assertEqual(ts2_after.time_step.length_minutes, 1440) self.assertEqual(ts2_after.time_step.length_months, 0) c = StringIO() ts2_after.write(c) self.assertEqual(c.getvalue().replace('\r', ''), self.test_timeseries2) # Check that the time series comments are the same before and after self.assertEqual(ts1_before.comment, ts1_after.comment) self.assertEqual(ts2_before.comment, ts2_after.comment)
def execute_point(self): # Make sure elevation has not been specified if self.elevation is not None: raise InvalidOptionError( 'elevation should only be specified in the configuration file ' 'in spatial calculation') # Read input time series input_timeseries = {} for name in ('temperature_min', 'temperature_max', 'temperature', 'humidity', 'humidity_min', 'humidity_max', 'wind_speed', 'pressure', 'solar_radiation', 'sunshine_duration'): filename = self.config['General'][name + '_prefix'] + '.hts' if not os.path.exists(filename): continue t = Timeseries() with open(filename, 'r') as f: t.read_file(f) input_timeseries[name] = t # Make sure all are in the same location and timezone first = True for index in input_timeseries: t = input_timeseries[index] if first: abscissa = t.location['abscissa'] ordinate = t.location['ordinate'] srid = t.location['srid'] altitude = t.location['altitude'] timezone = t.timezone first = False continue if abs(abscissa - t.location['abscissa']) > 1e-7 or \ abs(ordinate - t.location['ordinate']) > 1e-7 or \ srid != t.location['srid'] or \ abs(altitude - t.location['altitude']) > 1e-2 or \ timezone != t.timezone: raise ValueError('Incorrect or unspecified or inconsistent ' 'locations or time zones in the time series ' 'files.') # Convert location to WGS84 source_projection = osr.SpatialReference() source_projection.ImportFromEPSG(srid) wgs84 = osr.SpatialReference() wgs84.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(source_projection, wgs84) apoint = ogr.Geometry(ogr.wkbPoint) apoint.AddPoint(abscissa, ordinate) apoint.Transform(transform) latitude, longitude = apoint.GetY(), apoint.GetY() # Prepare the Penman Monteith method nsrr = self.nighttime_solar_radiation_ratio self.penman_monteith = PenmanMonteith( albedo=self.albedo, nighttime_solar_radiation_ratio=nsrr, elevation=altitude, latitude=latitude, longitude=longitude, step_length=self.step, unit_converters=self.unit_converters) # Create output timeseries object precision = 2 if self.step == timedelta(hours=1) else 1 pet = Timeseries( time_step=TimeStep(length_minutes=self.step.total_seconds() / 60), unit='mm', timezone=timezone, variable='Potential Evapotranspiration', precision=precision, location={'abscissa': abscissa, 'ordinate': ordinate, 'srid': srid, 'altitude': altitude}) # Let's see what variables we are going to use in the calculation, # based mainly on the step. if self.step == timedelta(hours=1): variables = ('temperature', 'humidity', 'wind_speed', 'solar_radiation') elif self.step == timedelta(days=1): variables = ( 'temperature_max', 'temperature_min', 'humidity_max', 'humidity_min', 'wind_speed', 'solar_radiation' if 'solar_radiation' in input_timeseries else 'sunshine_duration') else: raise Exception( 'Internal error: step should have been checked already') # Calculate evaporation for adatetime in input_timeseries['wind_speed']: try: kwargs = {v: input_timeseries[v][adatetime] for v in variables} except (IndexError, KeyError): continue kwargs['adatetime'] = adatetime pet[adatetime] = self.penman_monteith.calculate(**kwargs) # Save result outfilename = self.config['General']['evaporation_prefix'] + '.hts' with open(outfilename, 'w') as f: pet.write_file(f)
def test_daily(self): self.setup_input_file('temperature_max', textwrap.dedent("""\ Title=Temperature Max Location=-16.25 16.217 4326 Altitude=100 2014-07-06,21.5, """)) self.setup_input_file('temperature_min', textwrap.dedent("""\ Title=Temperature Min Location=-16.25 16.217 4326 Altitude=100 2014-07-06,12.3, """)) self.setup_input_file('humidity_max', textwrap.dedent("""\ Title=Humidity Max Location=-16.25 16.217 4326 Altitude=100 2014-07-06,84.0, """)) self.setup_input_file('humidity_min', textwrap.dedent("""\ Title=Humidity Min Location=-16.25 16.217 4326 Altitude=100 2014-07-06,63.0, """)) self.setup_input_file('wind_speed', textwrap.dedent("""\ Title=Wind speed Location=-16.25 16.217 4326 Altitude=100 2014-07-06,2.078, """)) self.setup_input_file('sunshine_duration', textwrap.dedent("""\ Title=Sunshine duration Location=-16.25 16.217 4326 Altitude=100 2014-07-06,9.25, """)) with open(self.config_file, 'w') as f: f.write(textwrap.dedent('''\ base_dir = {self.tempdir} albedo = 0.23 step_length = 1440 ''').format(self=self)) application = VaporizeApp() application.read_command_line() application.read_configuration() # Verify the output file doesn't exist yet result_filename = os.path.join(self.tempdir, 'evaporation.hts') self.assertFalse(os.path.exists(result_filename)) # Execute application.run() # Check that it has created a file and that the file is correct t = Timeseries() with open(result_filename) as f: t.read_file(f) self.assertEqual(len(t), 1) adate = datetime(2014, 7, 6) self.assertEqual(t.bounding_dates(), (adate, adate)) self.assertAlmostEqual(t[adate], 3.9)
def test_hourly(self): self.setup_input_file('temperature', textwrap.dedent("""\ Title=Temperature Location=-16.25 16.217 4326 Altitude=8 Timezone=CVT (UTC-0100) 2014-10-01 15:00,38, """)) self.setup_input_file('humidity', textwrap.dedent("""\ Title=Humidity Location=-16.25 16.217 4326 Altitude=8 Timezone=CVT (UTC-0100) 2014-10-01 15:00,52, """)) self.setup_input_file('wind_speed', textwrap.dedent("""\ Title=Wind speed Location=-16.25 16.217 4326 Altitude=8 Timezone=CVT (UTC-0100) 2014-10-01 15:00,3.3, """)) self.setup_input_file('pressure', textwrap.dedent("""\ Title=Pressure Location=-16.25 16.217 4326 Altitude=8 Timezone=CVT (UTC-0100) 2014-10-01 15:00,1013.0, """)) self.setup_input_file('solar_radiation', textwrap.dedent("""\ Title=Solar radiation Location=-16.25 16.217 4326 Altitude=8 Timezone=CVT (UTC-0100) 2014-10-01 15:00,681.0, """)) with open(self.config_file, 'w') as f: f.write(textwrap.dedent('''\ base_dir = {self.tempdir} albedo = 0.23 nighttime_solar_radiation_ratio = 0.8 step_length = 60 unit_converter_pressure = x / 10.0 unit_converter_solar_radiation = x * 3600 / 1e6 ''').format(self=self)) application = VaporizeApp() application.read_command_line() application.read_configuration() # Verify the output file doesn't exist yet result_filename = os.path.join(self.tempdir, 'evaporation.hts') self.assertFalse(os.path.exists(result_filename)) # Execute application.run() # Check that it has created a file and that the file is correct t = Timeseries() with open(result_filename) as f: t.read_file(f) self.assertEqual(len(t), 1) adate = datetime(2014, 10, 1, 15, 0, tzinfo=senegal_tzinfo) self.assertEqual(t.bounding_dates(), (adate, adate)) self.assertAlmostEqual(t[adate], 0.63)