def save(self, commit=True, *args, **kwargs):
tseries = super(TimeseriesForm, self).save(commit=False)
# This may not be a good idea !
if commit:
tseries.save()
# Handle pushing additional data if present back to the db
if 'data' in self.cleaned_data and self.cleaned_data['data']:
data = self.cleaned_data['data']
ts = timeseries.Timeseries(int(self.instance.id))
data.seek(0)
ts.read_file(data)
if self.cleaned_data['data_policy'] == 'A':
pass
# ts.append_to_db(db.connection, commit=False)
else:
afilename = os.path.join(settings.ENHYDRIS_TS_GRAPH_CACHE_DIR,
'%d.hts' % int(self.instance.id))
if os.path.exists(afilename):
os.remove(afilename)
ts.write_to_db(db.connection, commit=False)
return tseries
def clean_data(self):
# Check if file contains valid timeseries data.
if ('data' not in self.cleaned_data) or not self.cleaned_data['data']:
return None
self.cleaned_data['data'].seek(0)
ts = timeseries.Timeseries()
data = self.cleaned_data['data']
try:
ts.read_file(self.cleaned_data['data'])
except Exception as e:
raise forms.ValidationError(str(e))
return self.cleaned_data['data']
def end_date(self):
try:
ts = timeseries.Timeseries(int(self.id))
except:
return None
# This should be removed if ticket #112 gets resolved
c = db_connection.cursor()
c.execute("SELECT timeseries_end_date(%s)", (ts.id, ))
try:
r = c.fetchone()
except:
return None
c.close()
return r[0]
def testTimeseriesData(self):
"""Test that the timeseries data upload/download is correct"""
from pthelma import timeseries
# check uploading
f = open("enhydris/hcore/tests/tsdata.hts", "r")
file_dict = {'data': SimpleUploadedFile(f.name, f.read())}
post_dict = {'gentity': self.station.pk, 'variable': self.var.pk,
'unit_of_measurement': self.unit.pk,
'time_zone': self.tz.pk
}
form = TimeseriesDataForm(post_dict, file_dict, instance=self.ts)
self.assertEqual(form.is_valid(), True)
ts = form.save()
ts.save()
pts = timeseries.Timeseries(ts.id)
pts.read_from_db(dj_connection)
self.assertEqual(len(pts.items()), 12872)
#check downloading
url = "/timeseries/d/%d/download/" % self.ts.pk
response = self.client.get(url)
if settings.ENHYDRIS_TSDATA_AVAILABLE_FOR_ANONYMOUS_USERS:
self.assertEqual(response.status_code, 200)
else:
self.assertEqual(response.status_code, 302)
self.assertEquals(self.client.login(username='******',
password='******'), True)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
# check fiLe
lines = response.content.splitlines()
linecount = len(lines)
headerlinecount = sum([1 for x in takewhile(lambda x: x != '',
lines)]) + 1
datalinecount = linecount - headerlinecount
self.assertEqual(datalinecount, 12872)
self.client.logout()
def clean(self):
"""
This function checks the timestep and offset values and reports
inconsistencies.
"""
time_step = self.cleaned_data.get('time_step', None)
nominal_offset_minutes = self.cleaned_data.get(
'nominal_offset_minutes', None)
nominal_offset_months = self.cleaned_data.get('nominal_offset_months',
None)
actual_offset_minutes = self.cleaned_data.get('actual_offset_minutes',
None)
actual_offset_months = self.cleaned_data.get('actual_offset_months',
None)
if not time_step:
if nominal_offset_minutes or nominal_offset_months or \
actual_offset_minutes or actual_offset_months:
raise forms.ValidationError(
_("Invalid Timestep: If time step is"
" null, the offsets must also be null!"))
else:
if actual_offset_minutes is None or actual_offset_months is None:
raise forms.ValidationError(
_("Invalid offset: If time step is"
" not null, actual offset values must be provided!"))
if (nominal_offset_minutes is None
and nominal_offset_months is not None) \
or (nominal_offset_minutes is not None
and nominal_offset_months is None):
raise forms.ValidationError(
_("Invalid offsets: Nominal"
" offsets must be both null or both not null!"))
#add a validation test for instrument in station:
instr = self.cleaned_data.get('instrument', None)
if instr:
stat = self.cleaned_data.get('gentity', None)
assert (stat)
if Instrument.objects.filter(id=instr.id,
station__id=stat.id).count() < 1:
raise forms.ValidationError(
_("Selected instrument "
"not in selected station"))
# XXX: This is not a good idea but it's the only way to handle append
# errors. We save the data in the clean function instead of the save
# and if an error occurs we show the error in the form. Another way
# would be to write a try_append() function in pthelma.timeseries but
# this is specific to enhydris and should not be part of the pthelma
# module.
if 'data' in self.cleaned_data and self.cleaned_data['data']:
data = self.cleaned_data['data']
ts = timeseries.Timeseries(int(self.instance.id))
data.seek(0)
ts.read_file(data)
if self.cleaned_data['data_policy'] == 'A':
try:
ts.append_to_db(db.connection, commit=False)
except Exception, e:
raise forms.ValidationError(_(e.message))
def test_timeseries_data(self):
# Upload
with open("enhydris/hcore/tests/tsdata.hts", "r") as f:
file_dict = {'data': SimpleUploadedFile(f.name, f.read())}
post_dict = {'gentity': self.station.pk, 'variable': self.var.pk,
'unit_of_measurement': self.unit.pk,
'time_zone': self.tz.pk
}
form = TimeseriesDataForm(post_dict, file_dict, instance=self.ts)
self.assertTrue(form.is_valid())
ts = form.save()
ts.save()
pts = timeseries.Timeseries(ts.id)
pts.read_from_db(dj_connection)
self.assertEqual(len(pts.items()), 12872)
# Download
def nrecords():
lines = response.content.splitlines()
linecount = len(lines)
headerlinecount = sum([1 for x in takewhile(lambda x: x != '',
lines)]) + 1
return linecount - headerlinecount
if not settings.ENHYDRIS_TSDATA_AVAILABLE_FOR_ANONYMOUS_USERS:
self.client.login(username='******', password='******')
url = "/timeseries/d/{}/download/".format(self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content.splitlines()[0].strip(), 'Version=2')
self.assertEqual(nrecords(), 12872)
url = "/timeseries/d/{}/download/?version=3".format(self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
ats = timeseries.Timeseries()
ats.read_file(StringIO(response.content))
self.assertAlmostEqual(ats.location['abscissa'], 24.67890, places=6)
self.assertAlmostEqual(ats.location['ordinate'], 38.12345, places=6)
self.assertEqual(ats.location['srid'], 4326)
self.assertAlmostEqual(ats.location['altitude'], 219.22, places=2)
self.assertTrue(ats.location['asrid'] is None)
self.assertEqual(nrecords(), 12872)
url = "/timeseries/d/{}/download/1960-11-04/".format(self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nrecords(), 12870)
url = "/timeseries/d/{}/download/1960-11-04/1960-11-08T08:00/".format(
self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nrecords(), 4)
url = "/timeseries/d/{}/download//1960-11-08T08:00/".format(self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nrecords(), 6)
url = "/timeseries/d/{}/download//1960-11-08T08:00:00/".format(
self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nrecords(), 6)
url = "/timeseries/d/{}/download/1950-02-02/1960-11-08T08:00/".format(
self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nrecords(), 6)
url = "/timeseries/d/{}/download/1950-02-02/1960-01-01T08:00/".format(
self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nrecords(), 0)
url = "/timeseries/d/{}/download/1998-02-02//".format(
self.ts.pk)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(nrecords(), 0)
self.client.logout()