def testUploadTsData(self): self.assert_(self.client.login(username='******', password='******')) response = self.client.put( "/api/tsdata/1/", encode_multipart(BOUNDARY, {'timeseries_records': '2012-11-06 18:17,20,\n'}), content_type=MULTIPART_CONTENT) t = Timeseries(1) t.read_from_db(connection) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, '1') self.assertEqual(len(t), 1) self.assertEqual(t.items(0)[0], datetime(2012, 11, 06, 18, 17, 0)) self.assertEqual(t.items(0)[1], 20) self.client.logout() # Append two more records self.assert_(self.client.login(username='******', password='******')) response = self.client.put( "/api/tsdata/1/", encode_multipart( BOUNDARY, { 'timeseries_records': '2012-11-06 18:18,21,\n2012-11-07 18:18,23,\n' }), content_type=MULTIPART_CONTENT) t = Timeseries(1) t.read_from_db(connection) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, '2') self.assertEqual(len(t), 3) self.assertEqual(t.items(0)[0], datetime(2012, 11, 06, 18, 17, 0)) self.assertEqual(t.items(0)[1], 20) self.assertEqual(t.items(1)[0], datetime(2012, 11, 06, 18, 18, 0)) self.assertEqual(t.items(1)[1], 21) self.assertEqual(t.items(2)[0], datetime(2012, 11, 07, 18, 18, 0)) self.assertEqual(t.items(2)[1], 23) self.client.logout() # Try to append an earlier record; should fail self.assert_(self.client.login(username='******', password='******')) response = self.client.put( "/api/tsdata/1/", encode_multipart(BOUNDARY, {'timeseries_records': '2012-11-05 18:18,21,\n'}), content_type=MULTIPART_CONTENT) self.client.logout() t = Timeseries(1) t.read_from_db(connection) self.assertEqual(response.status_code, 409) self.assertEqual(len(t), 3) self.client.logout()
def testUploadTsData(self): self.assert_(self.client.login(username='******', password='******')) response = self.client.put( "/api/tsdata/1/", encode_multipart(BOUNDARY, {'timeseries_records': '2012-11-06 18:17,20,\n'}), content_type=MULTIPART_CONTENT) t = Timeseries(1) t.read_from_db(connection) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, '1') self.assertEqual(len(t), 1) self.assertEqual(t.items(0)[0], datetime(2012, 11, 06, 18, 17, 0)) self.assertEqual(t.items(0)[1], 20) self.client.logout() # Append two more records self.assert_(self.client.login(username='******', password='******')) response = self.client.put( "/api/tsdata/1/", encode_multipart(BOUNDARY, {'timeseries_records': '2012-11-06 18:18,21,\n2012-11-07 18:18,23,\n'}), content_type=MULTIPART_CONTENT) t = Timeseries(1) t.read_from_db(connection) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, '2') self.assertEqual(len(t), 3) self.assertEqual(t.items(0)[0], datetime(2012, 11, 06, 18, 17, 0)) self.assertEqual(t.items(0)[1], 20) self.assertEqual(t.items(1)[0], datetime(2012, 11, 06, 18, 18, 0)) self.assertEqual(t.items(1)[1], 21) self.assertEqual(t.items(2)[0], datetime(2012, 11, 07, 18, 18, 0)) self.assertEqual(t.items(2)[1], 23) self.client.logout() # Try to append an earlier record; should fail self.assert_(self.client.login(username='******', password='******')) response = self.client.put( "/api/tsdata/1/", encode_multipart(BOUNDARY, {'timeseries_records': '2012-11-05 18:18,21,\n'}), content_type=MULTIPART_CONTENT) self.client.logout() t = Timeseries(1) t.read_from_db(connection) self.assertEqual(response.status_code, 409) self.assertEqual(len(t), 3) self.client.logout()
def regularize_raw_series(raw_series_db, proc_series_db, rs, re, ps, pe ): """ This function regularize raw_series_db object from database and writes a processed proc_series_db in database. Raw series is a continuously increasing values time series, aggregating the water consumption. Resulting processed timeseries contains water consumption for each of its interval. I.e. if the timeseries is of 15 minutes time step, then each record contains the water consumption for each record period. """ raw_series = TSeries(id=raw_series_db.id) raw_series.read_from_db(db.connection) # We keep the last value for x-checking reasons, see last print # command test_value = raw_series[raw_series.bounding_dates()[1]] time_step = ReadTimeStep(proc_series_db.id, proc_series_db) proc_series = TSeries(id=proc_series_db.id, time_step = time_step) # The following code can be used in real conditions to append only # new records to db, in a next version #if not pe: # start = proc_series.time_step.down(rs) #else: # start = proc_series.time_step.up(pe) # Instead of the above we use now: start = proc_series.time_step.down(rs) end = proc_series.time_step.up(re) pointer = start # Pass 1: Initialize proc_series while pointer<=end: proc_series[pointer] = float('nan') pointer = proc_series.time_step.next(pointer) # Pass 2: Transfer cummulative raw series to differences series: prev_s = 0 for i in xrange(len(raw_series)): dat, value = raw_series.items(pos=i) if not math.isnan(value): raw_series[dat] = value-prev_s prev_s = value # Pass 3: Regularize step: loop over raw series records and distribute # floating point values to processed series for i in xrange(len(raw_series)): dat, value = raw_series.items(pos=i) if not math.isnan(value): # find previous, next timestamp of the proc time series d1 = proc_series.time_step.down(dat) d2 = proc_series.time_step.up(dat) if math.isnan(proc_series[d1]): proc_series[d1] = 0 if math.isnan(proc_series[d2]): proc_series[d2] = 0 if d1==d2: # if dat on proc step then d1=d2 proc_series[d1] += value continue dif1 = _dif_in_secs(d1, dat) dif2 = _dif_in_secs(dat, d2) dif = dif1+dif2 # Distribute value to d1, d2 proc_series[d1] += (dif2/dif)*value proc_series[d2] += (dif1/dif)*value # Uncomment the following line in order to show debug information. # Usually the three following sums are consistent by equality. If # not equality is satisfied then there is a likelyhood of algorith # error print raw_series.sum(), proc_series.sum(), test_value proc_series.write_to_db(db=db.connection, commit=True) #False) #return the full timeseries return proc_series
def regularize(raw_series_db, proc_series_db, rs, re): """ This function regularize raw_series_db object from database and writes a processed proc_series_db in database. Raw series is a continuously increasing values time series, aggregating the water consumption. Resulting processed timeseries contains water consumption for each of its interval. I.e. if the timeseries is of 15 minutes time step, then each record contains the water consumption for each record period. """ raw_series = TSeries(id=raw_series_db.id) raw_series.read_from_db(db.connection) # We keep the last value for x-checking reasons, see last print # command try: test_value = raw_series[raw_series.bounding_dates()[1]] except Exception as e: #log.debug("Trying to get test value for raw series %s failed with %s. " # "Skipping!" % (raw_series_db.id, repr(e))) return None time_step = ReadTimeStep(proc_series_db.id, proc_series_db) proc_series = TSeries(id=proc_series_db.id, time_step=time_step) # The following code can be used in real conditions to append only # new records to db, in a next version #if not pe: # start = proc_series.time_step.down(rs) #else: # start = proc_series.time_step.up(pe) # Instead of the above we use now: start = proc_series.time_step.down(rs) end = proc_series.time_step.up(re) pointer = start # Pass 1: Initialize proc_series while pointer <= end: proc_series[pointer] = float('nan') pointer = proc_series.time_step.next(pointer) # Pass 2: Transfer cummulative raw series to differences series: prev_s = 0 for i in xrange(len(raw_series)): dat, value = raw_series.items(pos=i) d = datetime.today() d = d.replace(month=11).replace(day=5) if dat.date() == d.date(): pass if not isnan(value): # "if" Added by Chris Pantazis, because sometimes # We get a negative small value by the meter if prev_s > value: prev_s = value raw_series[dat] = value - prev_s prev_s = value # Pass 3: Regularize step: loop over raw series records and distribute # floating point values to processed series for i in xrange(len(raw_series)): dat, value = raw_series.items(pos=i) if not isnan(value): # find previous, next timestamp of the proc time series d1 = proc_series.time_step.down(dat) d2 = proc_series.time_step.up(dat) if isnan(proc_series[d1]): proc_series[d1] = 0 if isnan(proc_series[d2]): proc_series[d2] = 0 if d1 == d2: # if dat on proc step then d1=d2 proc_series[d1] += value continue dif1 = _dif_in_secs(d1, dat) dif2 = _dif_in_secs(dat, d2) dif = dif1 + dif2 # Distribute value to d1, d2 proc_series[d1] += (dif2 / dif) * value proc_series[d2] += (dif1 / dif) * value # Uncomment the following line in order to show debug information. # Usually the three following sums are consistent by equality. If # not equality is satisfied then there is a likelyhood of algorith # error # log.info("%s = %s = %s ?" % (raw_series.sum(), # proc_series.sum(), test_value)) proc_series.write_to_db(db=db.connection, commit=True) #return the full timeseries return proc_series
def readseries(self,timeseries): time_step = ReadTimeStep(timeseries.id, timeseries) timeseries = Timeseries(time_step=time_step,id=timeseries.id) timeseries.read_from_db(connection) return timeseries.items()
Assuming that "dir" is the openmeteo directory, run as follows: export PYTHONPATH=dir:dir/enhydris export DJANGO_SETTINGS=settings ./oldopenmeteo2enhydris.sql """ import sys from datetime import timedelta from django.db import connection, transaction from enhydris.hcore import models from pthelma.timeseries import Timeseries transaction.enter_transaction_management() tms = models.Timeseries.objects.filter(time_step__id__in=[4,5]) for tm in tms: sys.stderr.write("Doing timeseries %d..." % (tm.id,)) t = Timeseries(id=tm.id) nt = Timeseries(id=tm.id) t.read_from_db(connection) for (d, value) in t.items(): d += timedelta(hours=1) assert(not d.minute and not d.hour and not d.second and d.day==1, "Invalid date "+str(d)) nt[d] = value nt.write_to_db(connection, transaction=transaction, commit=False) sys.stderr.write(" Done\n") transaction.commit()