def setUp(self):
parms = settings.BITIA_TEST_ENHYDRIS_INSTALLATION
self.cookies = enhydris_api.login(parms['base_url'],
parms['user'],
parms['password'])
# Create two time series
j = {
'gentity': parms['station_id'],
'variable': parms['variable_id'],
'unit_of_measurement': parms['unit_of_measurement_id'],
'time_zone': parms['time_zone_id'],
}
self.ts1_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Timeseries', j)
self.ts2_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Timeseries', j)
assert self.ts1_id != self.ts2_id
# Add some data (all but the last record) to the database
ts = Timeseries(self.ts1_id)
ts.read(StringIO(self.timeseries1_top))
enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)
ts = Timeseries(self.ts2_id)
ts.read(StringIO(self.timeseries2_top))
enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)
# Temporary directory for cache files
self.tempdir = tempfile.mkdtemp()
def setUp(self):
get_server_from_env(self.__dict__)
if not self.class_being_tested or not self.base_url:
return
# First 60 nonempty lines of test go to self.file1
(fd, self.file1) = tempfile.mkstemp(text=True)
fp = os.fdopen(fd, 'w')
i = 0
with open(full_testdata_filename(self.datafilename)) as f:
for line in f:
if i >= 60:
break
if not line.strip():
continue
i += 1
fp.write(line)
fp.close()
# Login
self.cookies = enhydris_api.login(self.base_url,
self.user,
self.password)
# Create two timeseries
self.timeseries_id1 = create_timeseries(self.cookies, self.__dict__)
self.timeseries_id2 = create_timeseries(self.cookies, self.__dict__)
self.datafile_fields = "0,{0},{1}".format(self.timeseries_id1,
self.timeseries_id2)
self.ts1 = Timeseries(self.timeseries_id1)
self.ts2 = Timeseries(self.timeseries_id2)
def setUp(self):
self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
self.cookies = enhydris_api.login(self.parms['base_url'],
self.parms['user'],
self.parms['password'])
# Create two time series
j = {
'gentity': self.parms['station_id'],
'variable': self.parms['variable_id'],
'unit_of_measurement': self.parms['unit_of_measurement_id'],
'time_zone': self.parms['time_zone_id'],
'time_step': 3,
'timestamp_offset_minutes': 0,
'timestamp_offset_months': 0,
'remarks': 'Très importante',
}
self.ts1_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Timeseries', j)
self.ts2_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Timeseries', j)
assert self.ts1_id != self.ts2_id
# Add some data (all but the last record) to the database
ts = Timeseries(self.ts1_id)
ts.read(StringIO(self.timeseries1_top))
enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
ts = Timeseries(self.ts2_id)
ts.read(StringIO(self.timeseries2_top))
enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
# Temporary directory for cache files
self.tempdir = tempfile.mkdtemp()
self.savedcwd = os.getcwd()
os.chdir(self.tempdir)
def test_get_ts_end_date(self):
v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
cookies = enhydris_api.login(v['base_url'], v['user'], v['password'])
# Create a time series in the database
j = {
'gentity': v['station_id'],
'variable': v['variable_id'],
'unit_of_measurement': v['unit_of_measurement_id'],
'time_zone': v['time_zone_id'],
}
ts_id = enhydris_api.post_model(v['base_url'], cookies, 'Timeseries',
j)
# Get its last date while it has no data
date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
self.assertEqual(date.isoformat(), '0001-01-01T00:00:00')
# Now upload some data
ts = Timeseries(ts_id)
ts.read(StringIO(self.test_timeseries))
enhydris_api.post_tsdata(v['base_url'], cookies, ts)
# Get its last date
date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
self.assertEqual(date.isoformat(), '2014-01-05T08:00:00')
# Get the last date of a nonexistent time series
self.assertRaises(requests.HTTPError, enhydris_api.get_ts_end_date,
v['base_url'], cookies, ts_id + 1)
def setUp(self):
get_server_from_env(self.__dict__)
self.ref_ts = Timeseries(0)
if not self.base_url:
return
self.cookies = enhydris_api.login(self.base_url, self.user,
self.password)
self.timeseries_id = create_timeseries(self.cookies, self.__dict__)
self.ts = Timeseries(self.timeseries_id)
def setUp(self):
get_server_from_env(self.__dict__)
# Connect to server
self.cookies = enhydris_api.login(self.base_url, self.user,
self.password)
# Create as many timeseries as there are not-to-ignore parameters
for parm in self.parameters:
parm['ts_id'] = create_timeseries(self.cookies, self.__dict__
) if parm['expname'] else 0
def append_newer_timeseries(self, start_date, ts1):
self.session_cookies = enhydris_api.login(self.base_url, self.user,
self.password)
url = self.base_url + 'timeseries/d/{}/download/{}/?version=3'.format(
self.timeseries_id, start_date.isoformat())
r = requests.get(url, cookies=self.session_cookies)
if r.status_code != 200:
raise HTTPError('Error {} while getting {}'.format(r.status_code,
url))
responseio = StringIO(r.text)
ts2 = Timeseries()
ts2.read_file(responseio)
responseio.seek(0)
ts1.read_meta(responseio)
ts1.append(ts2)
def test_login(self):
v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
base_url, user, password = v['base_url'], v['user'], v['password']
# Verify we are logged out
r = requests.get(base_url)
self.assertEqual(r.status_code, 200)
self.assertTrue('Login' in r.text)
self.assertFalse('Logout' in r.text)
# Now login and verify we're logged on
cookies = enhydris_api.login(base_url, user, password)
r = requests.get(base_url, cookies=cookies)
self.assertEqual(r.status_code, 200)
self.assertFalse('Login' in r.text)
self.assertTrue('Logout' in r.text)
def setUp(self):
get_server_from_env(self.__dict__)
if not self.class_being_tested or not self.base_url:
return
# Login
self.cookies = enhydris_api.login(self.base_url,
self.user,
self.password)
# Create two timeseries
self.timeseries_id1 = create_timeseries(self.cookies, self.__dict__)
self.timeseries_id2 = create_timeseries(self.cookies, self.__dict__)
self.datafile_fields = "{0},{1}".format(self.timeseries_id1,
self.timeseries_id2)
self.ts1 = Timeseries(self.timeseries_id1)
self.ts2 = Timeseries(self.timeseries_id2)
def execute(self):
cookies = enhydris_api.login(self.config['General']['base_url'],
self.config['General']['user'],
self.config['General']['password'])
for section in self.config:
if section == 'General':
continue
datafileclass = eval('Datafile_{}'.format(
self.config[section]['datafile_format']))
adatafile = datafileclass(self.config['General']['base_url'],
cookies,
self.config[section],
self.logger)
try:
adatafile.update_database()
except MeteologgerError as e:
msg = 'Error while processing item {0}: {1}'.format(section,
str(e))
sys.stderr.write(msg + '\n')
self.logger.error(msg)
self.logger.debug(traceback.format_exc())
def test_post_tsdata(self):
v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
cookies = enhydris_api.login(v['base_url'], v['user'], v['password'])
# Create a time series in the database
j = {
'gentity': v['station_id'],
'variable': v['variable_id'],
'unit_of_measurement': v['unit_of_measurement_id'],
'time_zone': v['time_zone_id'],
}
ts_id = enhydris_api.post_model(v['base_url'], cookies, 'Timeseries',
j)
# Now upload some data
ts = Timeseries(ts_id)
ts.read(StringIO(self.test_timeseries_top))
enhydris_api.post_tsdata(v['base_url'], cookies, ts)
# Read and check the time series
url = enhydris_api.urljoin(v['base_url'],
'timeseries/d/{}/download/'.format(ts.id))
r = requests.get(url, cookies=cookies)
r.raise_for_status()
self.assertEqual(get_after_blank_line(r.text),
self.test_timeseries_top)
# Upload more data
ts = Timeseries(ts_id)
ts.read(StringIO(self.test_timeseries_bottom))
enhydris_api.post_tsdata(v['base_url'], cookies, ts)
# Read and check the time series
url = enhydris_api.urljoin(v['base_url'],
'timeseries/d/{}/download/'.format(ts.id))
r = requests.get(url, cookies=cookies)
r.raise_for_status()
self.assertEqual(get_after_blank_line(r.text),
self.test_timeseries)
def setUp(self):
# Create two stations, each one with a time series
parms = settings.BITIA_TEST_ENHYDRIS_INSTALLATION
self.cookies = enhydris_api.login(parms['base_url'],
parms['user'],
parms['password'])
self.station1_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Station',
{'name': 'station1',
'srid': 4326,
'point': 'POINT (23.78743 37.97385)',
'copyright_holder': 'Joe User',
'copyright_years': '2014',
'stype': 1,
'owner': parms['owner_id'],
})
self.timeseries1_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Timeseries',
{'gentity': self.station1_id,
'variable': parms['variable_id'],
'unit_of_measurement': parms['unit_of_measurement_id'],
'time_zone': parms['time_zone_id']})
self.station2_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Station',
{'name': 'station1',
'srid': 4326,
'point': 'POINT (24.56789 38.76543)',
'copyright_holder': 'Joe User',
'copyright_years': '2014',
'stype': 1,
'owner': parms['owner_id'],
})
self.timeseries2_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Timeseries',
{'gentity': self.station2_id,
'variable': parms['variable_id'],
'unit_of_measurement': parms['unit_of_measurement_id'],
'time_zone': parms['time_zone_id']})
def update_one_timeseries(base_url, id, user=None, password=None):
if base_url[-1] != '/':
base_url += '/'
# Read timeseries from cache file
cache_filename = os.path.join(
settings.BITIA_TIMESERIES_CACHE_DIR, '{}.hts'.format(id))
ts1 = Timeseries()
if os.path.exists(cache_filename):
with open(cache_filename) as f:
try:
ts1.read_file(f)
except ValueError:
# File may be corrupted; continue with empty time series
ts1 = Timeseries()
# Get its end date
try:
end_date = ts1.bounding_dates()[1]
except TypeError:
# Timeseries is totally empty; no start and end date
end_date = datetime(1, 1, 1, 0, 0)
start_date = end_date + timedelta(minutes=1)
# Get newer timeseries and append it
session_cookies = enhydris_api.login(base_url, user, password)
url = base_url + 'timeseries/d/{}/download/{}/'.format(
id, start_date.isoformat())
r = requests.get(url, cookies=session_cookies)
r.raise_for_status()
ts2 = Timeseries()
ts2.read_file(StringIO(r.text))
ts1.append(ts2)
# Save it
with open(cache_filename, 'w') as f:
ts1.write(f)
def create_ogr_layer_from_stations(group, data_source):
"""
Create and return an ogr.Layer with stations as its points.
settings.BITIA_TIMESERIES_GROUPS[group] is a list of time series,
each time series referring to a station. This function retrieves
the co-ordinates of each station from Enhydris (unless we have
them cached in Django's cache) and creates a layer in the
specified ogr data_source whose features are points; as many
points as there are stations/timeseries; each point is also given
a timeseries_id attribute.
"""
layer = data_source.CreateLayer('test')
layer.CreateField(ogr.FieldDefn('timeseries_id', ogr.OFTInteger))
for item in settings.BITIA_TIMESERIES_GROUPS[group]:
# Get the point from the cache, or fetch it anew on cache miss
cache_key = 'timeseries_{}_point'.format(item['id'])
pointwkt = cache.get(cache_key)
if not pointwkt:
cookies = enhydris_api.login(item['base_url'],
item.get('user', None),
item.get('password', None))
tsd = enhydris_api.get_model(item['base_url'], cookies,
'Timeseries', item['id'])
gpoint = enhydris_api.get_model(item['base_url'], cookies,
'Gpoint', tsd['gentity'])
pointwkt = gpoint['point']
cache.set(cache_key, pointwkt)
# Create feature and put it on the layer
point = ogr.CreateGeometryFromWkt(pointwkt)
f = ogr.Feature(layer.GetLayerDefn())
f.SetGeometry(point)
f.SetField('timeseries_id', item['id'])
layer.CreateFeature(f)
return layer
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.config_file = os.path.join(self.tempdir, 'enhydris_cache.conf')
self.saved_argv = sys.argv
sys.argv = ['enhydris_cache', '--traceback', self.config_file]
self.savedcwd = os.getcwd()
# Create two stations, each one with a time series
self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
self.cookies = enhydris_api.login(self.parms['base_url'],
self.parms['user'],
self.parms['password'])
self.station1_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Station',
{'name': 'station1',
'srid': 4326,
'point': 'POINT (23.78743 37.97385)',
'copyright_holder': 'Joe User',
'copyright_years': '2014',
'stype': 1,
'owner': self.parms['owner_id'],
})
self.timeseries1_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Timeseries',
{'gentity': self.station1_id,
'variable': self.parms['variable_id'],
'unit_of_measurement': self.parms['unit_of_measurement_id'],
'time_zone': self.parms['time_zone_id']})
self.station2_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Station',
{'name': 'station1',
'srid': 4326,
'point': 'POINT (24.56789 38.76543)',
'copyright_holder': 'Joe User',
'copyright_years': '2014',
'stype': 1,
'owner': self.parms['owner_id'],
})
self.timeseries2_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Timeseries',
{'gentity': self.station2_id,
'variable': self.parms['variable_id'],
'unit_of_measurement': self.parms['unit_of_measurement_id'],
'time_zone': self.parms['time_zone_id']})
# Prepare a configuration file (some tests override it)
with open(self.config_file, 'w') as f:
f.write(textwrap.dedent('''\
[General]
cache_dir = {self.tempdir}
[timeseries1]
base_url = {base_url}
id = {self.timeseries1_id}
file = file1
user = {self.parms[user]}
password = {self.parms[password]}
[timeseries2]
base_url = {base_url}
id = {self.timeseries2_id}
file = file2
user = {self.parms[user]}
password = {self.parms[password]}
''').format(self=self, base_url=self.parms['base_url']))
