class Enhydris:
def __init__(self, configuration):
self.base_url = configuration.base_url
self.auth_token = configuration.auth_token
self.client = EnhydrisApiClient(self.base_url, self.auth_token)
def upload(self, meteologger_storage):
self._meteologger_storage = meteologger_storage
self._get_composite_timeseries_ids()
self._get_ts_end_dates()
self._upload_all_new_data()
def _get_composite_timeseries_ids(self):
"""Create a list of (timeseries_group_id, initial_timeseries_id) pairs."""
station_id = self._meteologger_storage.station_id
self._composite_timeseries_ids = []
for timeseries_group_id in self._meteologger_storage.timeseries_group_ids:
timeseries_id = self._get_timeseries_id(station_id,
timeseries_group_id)
self._composite_timeseries_ids.append(
CompositeTimeseriesId(timeseries_group_id, timeseries_id))
def _get_timeseries_id(self, station_id, timeseries_group_id):
timeseries = self.client.list_timeseries(station_id,
timeseries_group_id)
for item in timeseries:
if item["type"] == "Initial":
return item["id"]
return self._create_timeseries(station_id, timeseries_group_id)
def _create_timeseries(self, station_id, timeseries_group_id):
return self.client.post_timeseries(
station_id,
timeseries_group_id,
data={
"type": "Initial",
"time_step": "",
"timeseries_group": timeseries_group_id,
},
)
def _get_ts_end_dates(self):
station_id = self._meteologger_storage.station_id
start_of_time = dt.datetime(1700, 1, 1)
self._ts_end_dates = {
cts_id: self.client.get_ts_end_date(station_id, *cts_id)
or start_of_time
for cts_id in self._composite_timeseries_ids
}
def _upload_all_new_data(self):
station_id = self._meteologger_storage.station_id
sorted_ts_end_dates = sorted(self._ts_end_dates.items(),
key=lambda x: x[1])
for cts_id, ts_end_date in sorted_ts_end_dates:
new_data = self._meteologger_storage.get_recent_data(
cts_id.timeseries_group_id, ts_end_date)
if len(new_data):
self.client.post_tsdata(station_id, *cts_id,
HTimeseries(new_data))
def upgrade(self):
self._read_config()
self.api_client = EnhydrisApiClient(self.base_url)
self._convert_username_and_password_to_api_token()
self._convert_fields()
self._backup_file()
self._write_upgraded_file()
def setUp(self):
self.api_client = EnhydrisApiClient("https://mydomain.com")
# Temporary directory for cache files
self.tempdir = tempfile.mkdtemp()
self.savedcwd = os.getcwd()
os.chdir(self.tempdir)
def _append_newer_timeseries(self, start_date, old_ts):
with EnhydrisApiClient(self.base_url) as api_client:
api_client.login(self.user, self.password)
ts = api_client.read_tsdata(self.station_id,
self.timeseries_id,
start_date=start_date)
new_data = ts.data
ts.data = old_ts.data.append(new_data,
verify_integrity=True,
sort=False)
return ts
def __init__(self, configuration):
self.base_url = configuration.base_url
self.auth_token = configuration.auth_token
self.client = EnhydrisApiClient(self.base_url, self.auth_token)
class ConfigFile:
def __init__(self, filename):
self.filename = filename
def upgrade(self):
self._read_config()
self.api_client = EnhydrisApiClient(self.base_url)
self._convert_username_and_password_to_api_token()
self._convert_fields()
self._backup_file()
self._write_upgraded_file()
def _read_config(self):
self.config = configparser.ConfigParser(interpolation=None)
with open(self.filename) as f:
self.config.read_file(f)
self.base_url = self.config.get("General", "base_url")
def _convert_username_and_password_to_api_token(self):
username = self.config.get("General", "username")
password = self.config.get("General", "password")
token = self.api_client.get_token(username, password)
self.config.set("General", "auth_token", token)
self.config.remove_option("General", "username")
self.config.remove_option("General", "password")
def _convert_fields(self):
station_section_names = [
n for n in self.config.sections() if n != "General"
]
for section_name in station_section_names:
self._convert_section(section_name)
def _convert_section(self, section_name):
self.section_name = section_name
self.station_id = self.config.get(section_name, "station_id")
for parameter in convertable_parameters:
self._convert_parameter(parameter)
def _convert_parameter(self, parameter):
value = self.config.get(self.section_name, parameter, fallback=None)
if value is None:
return
timeseries_ids = [int(x.strip()) for x in value.split(",")]
timeseries_group_ids = [
self._get_timeseries_group(timeseries_id)
for timeseries_id in timeseries_ids
]
self.config.set(
self.section_name,
parameter,
",".join([str(x) for x in timeseries_group_ids]),
)
def _get_timeseries_group(self, timeseries_id):
if timeseries_id == 0:
return 0
url = urljoin(
self.api_client.base_url,
f"api/stations/{self.station_id}/timeseries/{timeseries_id}/",
)
response = self.api_client.session.get(url)
self._check_response(response)
return response.json()["timeseries_group"]
def _check_response(self, response):
try:
response.raise_for_status()
except requests.HTTPError as e:
if response.text:
raise requests.HTTPError(
f"{str(e)}. Server response: {response.text}")
else:
raise
@property
def backup_filename(self):
return self.filename + ".bak"
def _backup_file(self):
backup_file_is_identical = os.path.exists(
self.backup_filename) and filecmp.cmp(
self.filename, self.backup_filename, shallow=False)
if backup_file_is_identical:
return
if os.path.exists(self.backup_filename):
raise RuntimeError(
f"Cannot backup configuration file; {self.backup_filename} exists"
)
shutil.copy(self.filename, self.backup_filename)
def _write_upgraded_file(self):
with open(self.filename, "w") as f:
self.config.write(f)
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.config_file = os.path.join(self.tempdir, "enhydris_cache.conf")
self.saved_argv = sys.argv
sys.argv = ["enhydris_cache", "--traceback", self.config_file]
self.savedcwd = os.getcwd()
# Create two stations, each one with a time series
self.parms = json.loads(os.getenv("ENHYDRIS_CACHE_E2E_TEST"))
self.api_client = EnhydrisApiClient(self.parms["base_url"])
self.api_client.__enter__()
with self.api_client.session:
self.api_client.login(self.parms["username"], self.parms["password"])
self.station1_id = self.api_client.post_station(
{
"name": "station1",
"original_srid": 4326,
"geom": "POINT (23.78743 37.97385)",
"copyright_holder": "Joe User",
"copyright_years": "2014",
"owner": self.parms["owner_id"],
}
)
self.timeseries1_id = self.api_client.post_timeseries(
self.station1_id,
{
"gentity": self.station1_id,
"variable": self.parms["variable_id"],
"unit_of_measurement": self.parms["unit_of_measurement_id"],
"time_zone": self.parms["time_zone_id"],
"precision": 0,
"time_step": "D",
},
)
self.station2_id = self.api_client.post_station(
{
"name": "station1",
"original_srid": 4326,
"geom": "POINT (24.56789 38.76543)",
"copyright_holder": "Joe User",
"copyright_years": "2014",
"owner": self.parms["owner_id"],
}
)
self.timeseries2_id = self.api_client.post_timeseries(
self.station2_id,
{
"gentity": self.station2_id,
"variable": self.parms["variable_id"],
"unit_of_measurement": self.parms["unit_of_measurement_id"],
"time_zone": self.parms["time_zone_id"],
"precision": 2,
"time_step": "D",
},
)
# Add some data (all but the last record) to the database
self.api_client.post_tsdata(
self.station1_id,
self.timeseries1_id,
HTimeseries(StringIO(self.timeseries1_top)),
)
self.api_client.post_tsdata(
self.station2_id,
self.timeseries2_id,
HTimeseries(StringIO(self.timeseries2_top)),
)
# Prepare a configuration file (some tests override it)
with open(self.config_file, "w") as f:
f.write(
textwrap.dedent(
"""\
[General]
cache_dir = {self.tempdir}
[timeseries1]
base_url = {base_url}
station_id = {self.station1_id}
timeseries_id = {self.timeseries1_id}
file = file1
user = {self.parms[username]}
password = {self.parms[password]}
[timeseries2]
base_url = {base_url}
station_id = {self.station2_id}
timeseries_id = {self.timeseries2_id}
file = file2
user = {self.parms[username]}
password = {self.parms[password]}
"""
).format(self=self, base_url=self.parms["base_url"])
)
class EnhydrisCacheE2eTestCase(TestCase):
test_timeseries1 = textwrap.dedent(
"""\
2014-01-01 08:00,11,
2014-01-02 08:00,12,
2014-01-03 08:00,13,
2014-01-04 08:00,14,
2014-01-05 08:00,15,
"""
)
test_timeseries2 = textwrap.dedent(
"""\
2014-07-01 08:00,9.11,
2014-07-02 08:00,9.12,
2014-07-03 08:00,9.13,
2014-07-04 08:00,9.14,
2014-07-05 08:00,9.15,
"""
)
timeseries1_top = "".join(test_timeseries1.splitlines(True)[:-1])
timeseries2_top = "".join(test_timeseries2.splitlines(True)[:-1])
timeseries1_bottom = test_timeseries1.splitlines(True)[-1]
timeseries2_bottom = test_timeseries2.splitlines(True)[-1]
def setUp(self):
self.tempdir = tempfile.mkdtemp()
self.config_file = os.path.join(self.tempdir, "enhydris_cache.conf")
self.saved_argv = sys.argv
sys.argv = ["enhydris_cache", "--traceback", self.config_file]
self.savedcwd = os.getcwd()
# Create two stations, each one with a time series
self.parms = json.loads(os.getenv("ENHYDRIS_CACHE_E2E_TEST"))
self.api_client = EnhydrisApiClient(self.parms["base_url"])
self.api_client.__enter__()
with self.api_client.session:
self.api_client.login(self.parms["username"], self.parms["password"])
self.station1_id = self.api_client.post_station(
{
"name": "station1",
"original_srid": 4326,
"geom": "POINT (23.78743 37.97385)",
"copyright_holder": "Joe User",
"copyright_years": "2014",
"owner": self.parms["owner_id"],
}
)
self.timeseries1_id = self.api_client.post_timeseries(
self.station1_id,
{
"gentity": self.station1_id,
"variable": self.parms["variable_id"],
"unit_of_measurement": self.parms["unit_of_measurement_id"],
"time_zone": self.parms["time_zone_id"],
"precision": 0,
"time_step": "D",
},
)
self.station2_id = self.api_client.post_station(
{
"name": "station1",
"original_srid": 4326,
"geom": "POINT (24.56789 38.76543)",
"copyright_holder": "Joe User",
"copyright_years": "2014",
"owner": self.parms["owner_id"],
}
)
self.timeseries2_id = self.api_client.post_timeseries(
self.station2_id,
{
"gentity": self.station2_id,
"variable": self.parms["variable_id"],
"unit_of_measurement": self.parms["unit_of_measurement_id"],
"time_zone": self.parms["time_zone_id"],
"precision": 2,
"time_step": "D",
},
)
# Add some data (all but the last record) to the database
self.api_client.post_tsdata(
self.station1_id,
self.timeseries1_id,
HTimeseries(StringIO(self.timeseries1_top)),
)
self.api_client.post_tsdata(
self.station2_id,
self.timeseries2_id,
HTimeseries(StringIO(self.timeseries2_top)),
)
# Prepare a configuration file (some tests override it)
with open(self.config_file, "w") as f:
f.write(
textwrap.dedent(
"""\
[General]
cache_dir = {self.tempdir}
[timeseries1]
base_url = {base_url}
station_id = {self.station1_id}
timeseries_id = {self.timeseries1_id}
file = file1
user = {self.parms[username]}
password = {self.parms[password]}
[timeseries2]
base_url = {base_url}
station_id = {self.station2_id}
timeseries_id = {self.timeseries2_id}
file = file2
user = {self.parms[username]}
password = {self.parms[password]}
"""
).format(self=self, base_url=self.parms["base_url"])
)
def tearDown(self):
os.chdir(self.savedcwd)
shutil.rmtree(self.tempdir)
sys.argv = self.saved_argv
self.api_client.__exit__()
def test_execute(self):
application = cli.App(self.config_file)
# Check that the two files don't exist yet
self.assertFalse(os.path.exists(os.path.join(self.tempdir, "file1")))
self.assertFalse(os.path.exists(os.path.join(self.tempdir, "file2")))
# Execute the application
application.run()
# Check that it has created two files
self.assertTrue(os.path.exists(os.path.join(self.tempdir, "file1")))
self.assertTrue(os.path.exists(os.path.join(self.tempdir, "file2")))
# Check that the files are what they should be
with open("file1", newline="\n") as f:
ts1_before = HTimeseries(f)
self.assertEqual(ts1_before.time_step, "D")
c = StringIO()
ts1_before.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.timeseries1_top)
with open("file2", newline="\n") as f:
ts2_before = HTimeseries(f)
self.assertEqual(ts2_before.time_step, "D")
c = StringIO()
ts2_before.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.timeseries2_top)
# Append a record to the database for each timeseries
self.api_client.post_tsdata(
self.station1_id,
self.timeseries1_id,
HTimeseries(StringIO(self.timeseries1_bottom)),
)
self.api_client.post_tsdata(
self.station2_id,
self.timeseries2_id,
HTimeseries(StringIO(self.timeseries2_bottom)),
)
# Execute the application again
application.run()
# Check that the files are what they should be
with open("file1", newline="\n") as f:
ts1_after = HTimeseries(f)
self.assertEqual(ts1_after.time_step, "D")
c = StringIO()
ts1_after.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.test_timeseries1)
with open("file2", newline="\n") as f:
ts2_after = HTimeseries(f)
self.assertEqual(ts2_after.time_step, "D")
c = StringIO()
ts2_after.write(c)
self.assertEqual(c.getvalue().replace("\r", ""), self.test_timeseries2)
# Check that the time series comments are the same before and after
self.assertEqual(ts1_before.comment, ts1_after.comment)
self.assertEqual(ts2_before.comment, ts2_after.comment)