def test_get_ts_end_date(self):
v = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
cookies = enhydris_api.login(v['base_url'], v['user'], v['password'])
# Create a time series in the database
j = {
'gentity': v['station_id'],
'variable': v['variable_id'],
'unit_of_measurement': v['unit_of_measurement_id'],
'time_zone': v['time_zone_id'],
}
ts_id = enhydris_api.post_model(v['base_url'], cookies, 'Timeseries',
j)
# Get its last date while it has no data
date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
self.assertEqual(date.isoformat(), '0001-01-01T00:00:00')
# Now upload some data
ts = Timeseries(ts_id)
ts.read(StringIO(self.test_timeseries))
enhydris_api.post_tsdata(v['base_url'], cookies, ts)
# Get its last date
date = enhydris_api.get_ts_end_date(v['base_url'], cookies, ts_id)
self.assertEqual(date.isoformat(), '2014-01-05T08:00:00')
# Get the last date of a nonexistent time series
self.assertRaises(requests.HTTPError, enhydris_api.get_ts_end_date,
v['base_url'], cookies, ts_id + 1)
def handle(self, *args, **options):
try:
username = args[0]
except IndexError:
print "I need a username!"
return -1
try:
if username:
user = User.objects.get(username=username)
out = []
print "output for {x}".format(x=username)
household = Household.objects.get(user=user)
timeseries = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
series = TSeries(id=timeseries.id)
series.read_from_db(db.connection)
timestamps = sorted(series.keys())
values = np.array([])
for ts in timestamps:
val = series[ts]
if isnan(val) or val == 0:
continue
values = np.append(values, val)
perc = np.percentile(values, 90)
out.append([ts, val, perc])
_outfile = "timeseries_%s.csv" % username
_path = "data/"
with open(path.join(_path, _outfile), 'w') as of:
a = csv.writer(of, delimiter=',',
quotechar='"',
quoting=csv.QUOTE_ALL)
a.writerows(out)
except Exception as e:
print "failed with %s" % repr(e)
def get(self, request, pk, format=None):
ts = Timeseries(id=int(pk))
self.check_object_permissions(request, ts)
ts.read_from_db(connection)
result = StringIO()
ts.write(result)
return HttpResponse(result.getvalue(), content_type="text/plain")
def read_timeseries_from_cache_file(self):
result = Timeseries()
if os.path.exists(self.filename):
with open(self.filename) as f:
try:
result.read_file(f)
except ValueError:
# File may be corrupted; continue with empty time series
result = Timeseries()
return result
def execute_item(self, item):
source_ts = Timeseries()
with open(item['source_file']) as f:
source_ts.read_file(f)
self.target_step.interval_type = item['interval_type']
target_ts, missing = source_ts.aggregate(
self.target_step,
missing_allowed=self.missing_allowed,
missing_flag=self.missing_flag)
with open(item['target_file'], 'w') as f:
target_ts.write_file(f, version=3)
def testUploadTsDataUnauthenticated(self):
# Attempt to upload some timeseries data, unauthenticated
response = self.client.put(
"/api/tsdata/1/",
encode_multipart(BOUNDARY, {"timeseries_records": "2012-11-06 18:17,20,\n"}),
content_type=MULTIPART_CONTENT,
)
t = Timeseries(1)
t.read_from_db(connection)
self.assertEqual(response.status_code, 403)
self.assertEqual(len(t), 0)
def testUploadTsDataGarbage(self):
self.assert_(self.client.login(username='******', password='******'))
response = self.client.put(
"/api/tsdata/{}/".format(self.timeseries1.id),
encode_multipart(BOUNDARY,
{'timeseries_records': '2012-aa-06 18:17,20,\n'}),
content_type=MULTIPART_CONTENT)
self.assertEqual(response.status_code, 400)
t = Timeseries(self.timeseries1.id)
t.read_from_db(connection)
self.assertEqual(len(t), 0)
self.client.logout()
def testUploadTsDataGarbage(self):
self.assert_(self.client.login(username="******", password="******"))
response = self.client.put(
"/api/tsdata/1/",
encode_multipart(BOUNDARY, {"timeseries_records": "2012-aa-06 18:17,20,\n"}),
content_type=MULTIPART_CONTENT,
)
t = Timeseries(1)
t.read_from_db(connection)
self.assertEqual(response.status_code, 400)
self.assertEqual(len(t), 0)
self.client.logout()
def check(self, datadir):
for parm in self.parameters:
if not parm['ts_id']:
continue
actual_ts = Timeseries(parm['ts_id'])
enhydris_api.read_tsdata(self.base_url, self.cookies, actual_ts)
reference_ts = Timeseries()
with open(os.path.join(
datadir, 'generated', parm['expname'] + '.txt')) as f:
reference_ts.read(f)
precision = self.guess_precision(f)
self.assertTimeseriesEqual(actual_ts, reference_ts, precision,
parm['expname'] + '.txt')
def put(self, request, pk, format=None):
try:
ts = Timeseries(id=int(pk))
self.check_object_permissions(request, ts)
result_if_error = status.HTTP_400_BAD_REQUEST
ts.read(StringIO(request.DATA['timeseries_records']))
result_if_error = status.HTTP_409_CONFLICT
ts.append_to_db(connection, commit=False)
return HttpResponse(str(len(ts)), content_type="text/plain")
except ValueError as e:
return HttpResponse(status=result_if_error,
content=str(e),
content_type="text/plain")
def testUploadTsDataAsWrongUser(self):
# Attempt to upload some timeseries data as user 2; should deny
self.assert_(self.client.login(username="******", password="******"))
response = self.client.put(
"/api/tsdata/1/",
encode_multipart(BOUNDARY, {"timeseries_records": "2012-11-06 18:17,20,\n"}),
content_type=MULTIPART_CONTENT,
)
t = Timeseries(1)
t.read_from_db(connection)
self.assertEqual(response.status_code, 403)
self.assertEqual(len(t), 0)
self.client.logout()
def testUploadTsDataAsWrongUser(self):
# Attempt to upload some timeseries data as user 2; should deny
self.assert_(self.client.login(username='******', password='******'))
response = self.client.put(
"/api/tsdata/{}/".format(self.timeseries1.id),
encode_multipart(BOUNDARY,
{'timeseries_records': '2012-11-06 18:17,20,\n'}),
content_type=MULTIPART_CONTENT)
t = Timeseries(self.timeseries1.id)
t.read_from_db(connection)
self.assertEqual(response.status_code, 403)
self.assertEqual(len(t), 0)
self.client.logout()
def append_newer_timeseries(self, start_date, ts1):
self.session_cookies = enhydris_api.login(self.base_url, self.user,
self.password)
url = self.base_url + 'timeseries/d/{}/download/{}/?version=3'.format(
self.timeseries_id, start_date.isoformat())
r = requests.get(url, cookies=self.session_cookies)
if r.status_code != 200:
raise HTTPError('Error {} while getting {}'.format(r.status_code,
url))
responseio = StringIO(r.text)
ts2 = Timeseries()
ts2.read_file(responseio)
responseio.seek(0)
ts1.read_meta(responseio)
ts1.append(ts2)
def process():
for household in Household.objects.all():
daily_series_db = household.timeseries.get(
time_step__id=TSTEP_DAILY)
series = TSeries(id=daily_series_db.id)
series.read_from_db(db.connection)
m = 1000.000*series.average()
if math.isnan(m):
continue
num_of_occupants = max(1,
int(round(m/AVERAGE_UNIT_WATER_CONSUMPTION)))
print 'Household with id=%s, average daily consumption %.1f, '\
'number of occupants set to %s'%(household.id, m,
num_of_occupants,)
household.num_of_occupants = num_of_occupants
household.save()
def time_step(self):
"""
Return time step of all time series. If time step is not the same
for all time series, raises exception.
"""
time_step = None
for filename in self.files:
with open(filename) as f:
t = Timeseries()
t.read_meta(f)
item_time_step = (t.time_step.length_minutes,
t.time_step.length_months)
if time_step and (item_time_step != time_step):
raise WrongValueError(
'Not all time series have the same step')
time_step = item_time_step
return time_step
def has_burst_old(household):
"""
We won't be using this algorithm any more
:param household:
:return:
"""
name = household.user.username
if not name.startswith('GR'):
return 0, 0
timeseries = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
series = TSeries(id=timeseries.id)
series.read_from_db(db.connection)
timestamps = sorted(series.keys())
today = [] # all today's values
_all = []
for i in range(1, len(timestamps)):
ts = timestamps[i]
if household.user.username == "GR006047" \
and ts.year == 2015 and ts.month == 2 and ts.day == 9 \
and ts.hour == 17:
pass
prev_ts = timestamps[i-1]
# if previous value is NaN we don't take this value into consideration
# Because it might have all consumption of all the previous NaN times
val = series[ts]
prev_val = series[prev_ts]
if isnan(prev_val):
continue
if i < len(timestamps) - 100:
if not isnan(val) and not val == 0:
_all.append(series[ts])
else:
tm = "%s:%s" % (ts.time().hour, ts.time().minute)
if not isnan(val) and not val == 0:
today.append((val, tm))
if _all and today:
all1 = np.array(_all)
p = np.percentile(all1, 95)
for cons, tm in today:
if cons > p:
return cons, tm
return 0, 0
def has_burst(household):
"""
We won't be using this algorithm any more
:param household:
:return:
"""
name = household.user.username
if not name.startswith('GR'):
return 0, 0
timeseries = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
series = TSeries(id=timeseries.id)
series.read_from_db(db.connection)
timestamps = sorted(series.keys())
today = [] # all today's values
daily_maxes = {}
for i in range(1, len(timestamps)):
ts = timestamps[i]
prev_ts = timestamps[i-1]
date = ts.date()
# if previous value is NaN we don't take this value into consideration
# Because it might have all consumption of all the previous NaN times
val = series[ts]
prev_val = series[prev_ts]
if isnan(prev_val):
continue
if i < len(timestamps) - 100:
if not isnan(val) and not val == 0:
daily_max = daily_maxes.get(date, 0)
if val > daily_max:
daily_maxes[date] = val
else:
tm = "%s-%s-%s %s:%s" % (ts.year, ts.month, ts.day,
ts.time().hour, ts.time().minute)
if not isnan(val) and not val == 0:
today.append((val, tm))
if daily_maxes and today:
maxes = np.array(daily_maxes.values())
p = np.percentile(maxes, 90)
for cons, tm in today:
if cons > p:
return cons, tm
return 0, 0
def setUp(self):
get_server_from_env(self.__dict__)
self.ref_ts = Timeseries(0)
if not self.base_url:
return
self.cookies = enhydris_api.login(self.base_url, self.user,
self.password)
self.timeseries_id = create_timeseries(self.cookies, self.__dict__)
self.ts = Timeseries(self.timeseries_id)
def h_integrate(mask, stations_layer, date, output_filename_prefix, date_fmt,
funct, kwargs):
date_fmt_for_filename = date.strftime(date_fmt).replace(' ', '-').replace(
':', '-')
output_filename = '{}-{}.tif'.format(output_filename_prefix,
date.strftime(date_fmt_for_filename))
if not _needs_calculation(output_filename, date, stations_layer):
return
# Read the time series values and add the 'value' attribute to
# stations_layer
stations_layer.CreateField(ogr.FieldDefn('value', ogr.OFTReal))
input_files = []
stations_layer.ResetReading()
for station in stations_layer:
filename = station.GetField('filename')
t = Timeseries()
with open(filename) as f:
t.read_file(f)
value = t.get(date, float('NaN'))
station.SetField('value', value)
if not isnan(value):
input_files.append(filename)
stations_layer.SetFeature(station)
if not input_files:
return
# Create destination data source
output = gdal.GetDriverByName('GTiff').Create(
output_filename, mask.RasterXSize, mask.RasterYSize, 1,
gdal.GDT_Float32)
output.SetMetadataItem('TIMESTAMP', date.strftime(date_fmt))
output.SetMetadataItem('INPUT_FILES', '\n'.join(input_files))
try:
# Set geotransform and projection in the output data source
output.SetGeoTransform(mask.GetGeoTransform())
output.SetProjection(mask.GetProjection())
# Do the integration
integrate(mask, stations_layer, output.GetRasterBand(1), funct, kwargs)
finally:
# Close the dataset
output = None
def create_objects(dma, household_identifier, series, force=False):
"""
When a household is fully parsed then this command is called to create
database objects thus: user (household owner), household, database time
series placeholders (for raw data and for processed data), to write actual
time series data in database and finally to estimate the household
occupancy.
"""
print "Processing household %s, user username will be %s as well"%(
household_identifier, household_identifier)
# Create user (household owner), household, database series placeholders
user = create_user(household_identifier)
household=create_household(household_identifier, user,
zone=dma.id)
db_series = create_raw_timeseries(household)
create_processed_timeseries(household)
timeseries_data = {}
# Now we will create timeseries.Timeseries() and we will add
# parsed values
for variable in db_series:
if variable not in ('WaterCold', 'Electricity'):
continue
s, e = timeseries_bounding_dates_from_db(db.connection,
db_series[variable].id)
if not force and (s or e):
print 'Raw timeseries id=%s has already data, skipping...'%(
db_series[variable].id,)
continue
timeseries = TSeries()
timeseries.id = db_series[variable].id
total = 0.0
for timestamp, value in series[variable]:
if not math.isnan(value):
total += value
timeseries[timestamp] = total
else:
timeseries[timestamp] = float('NaN')
timeseries_data[variable] = timeseries
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
if 'WaterCold' in timeseries_data:
calc_occupancy(timeseries_data['WaterCold'], household)
def _needs_calculation(output_filename, date, stations_layer):
"""
Used by h_integrate to check whether the output file needs to be calculated
or not. It does not need to be calculated if it already exists and has been
calculated from all available data.
"""
# Return immediately if output file does not exist
if not os.path.exists(output_filename):
return True
# Get list of files which were used to calculate the output file
fp = gdal.Open(output_filename)
try:
actual_input_files = fp.GetMetadataItem('INPUT_FILES')
if actual_input_files is None:
raise IOError('{} does not contain the metadata item INPUT_FILES'
.format(output_filename))
finally:
fp = None # Close file
actual_input_files = set(actual_input_files.split('\n'))
# Get list of files available for calculating the output file
stations_layer.ResetReading()
available_input_files = set(
[station.GetField('filename') for station in stations_layer
if os.path.exists(station.GetField('filename'))])
# Which of these files have not been used?
unused_files = available_input_files - actual_input_files
# For each one of these files, check whether it has newly available data.
# Upon finding one that does, the verdict is made: return True
for filename in unused_files:
t = Timeseries()
with open(filename) as f:
t.read_file(f)
value = t.get(date, float('NaN'))
if not isnan(value):
return True
# We were unable to find data that had not already been used
return False
def parse_and_save_timeseries(device_id, timeseries_id):
"""
Reads a RAW timeseries from REST API and saves in our local
database using the timeseries_id.
``device_id`` will be the ``identifier`` used in other functions,
usualy is the customerID==deviceID
"""
s, e = timeseries_bounding_dates_from_db(db.connection,
timeseries_id)
if s or e:
print 'Raw timeseries id=%s has already data, skipping...'%(
timeseries_id,)
return
timeseries = TSeries()
timeseries.id = timeseries_id
for timestamp, value in ibm_restapi.get_raw_timeseries(device_id):
timeseries[timestamp] = value
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
def parse_and_save_timeseries(filename, timeseries_id):
first_line = True
timeseries = TSeries()
timeseries.id = timeseries_id
with open(filename) as fp:
for line in fp.readlines():
if first_line:
first_line = False
continue
components = line.split(',')
date_str = components[1].strip('"')
value_str = components[2].strip('"')
value = float(value_str)
if value<MIN_VALUE or value>=MAX_VALUE:
value = float('nan')
tstamp = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S')
tstamp = tstamp.replace(second=0)
timeseries[tstamp] = value
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
def get_values_after(household, dt, variable):
timeseries = None
if variable == "WaterCold":
timeseries = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
elif variable == "Electricity":
timeseries = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_ENERGY_PERIOD)
data = []
if timeseries:
series = TSeries(id=timeseries.id)
series.read_from_db(db.connection)
timestamps = sorted(series.keys())
for ts in timestamps:
val = series[ts]
if ts <= dt:
continue
data.append((ts, val))
return data
def test_execute(self):
application = AggregateApp()
application.read_command_line()
application.read_configuration()
# Verify the output files don't exist yet
self.assertFalse(os.path.exists(self.output_filenames[0]))
self.assertFalse(os.path.exists(self.output_filenames[1]))
# Execute
application.run()
# Check that it has created two files
self.assertTrue(os.path.exists(self.output_filenames[0]))
self.assertTrue(os.path.exists(self.output_filenames[1]))
# Check that the created time series are correct
t = Timeseries()
with open(self.output_filenames[0]) as f:
t.read_file(f)
self.assertEqual(t.timezone, 'EET (UTC+0200)')
self.assertEqual(len(t), 1)
self.assertAlmostEqual(t['2014-06-16 16:00'], 114.9, places=5)
t = Timeseries()
with open(self.output_filenames[1]) as f:
t.read_file(f)
self.assertEqual(t.timezone, '')
self.assertEqual(len(t), 1)
self.assertAlmostEqual(t['2014-06-17 16:00'], 50.8167, places=5)
def setUp(self):
parms = settings.BITIA_TEST_ENHYDRIS_INSTALLATION
self.cookies = enhydris_api.login(parms['base_url'],
parms['user'],
parms['password'])
# Create two time series
j = {
'gentity': parms['station_id'],
'variable': parms['variable_id'],
'unit_of_measurement': parms['unit_of_measurement_id'],
'time_zone': parms['time_zone_id'],
}
self.ts1_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Timeseries', j)
self.ts2_id = enhydris_api.post_model(
parms['base_url'], self.cookies, 'Timeseries', j)
assert self.ts1_id != self.ts2_id
# Add some data (all but the last record) to the database
ts = Timeseries(self.ts1_id)
ts.read(StringIO(self.timeseries1_top))
enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)
ts = Timeseries(self.ts2_id)
ts.read(StringIO(self.timeseries2_top))
enhydris_api.post_tsdata(parms['base_url'], self.cookies, ts)
# Temporary directory for cache files
self.tempdir = tempfile.mkdtemp()
def handle(self, *args, **options):
try:
username = args[0]
except IndexError:
print "I need a username!"
return -1
try:
if username not in ["GR", "GB", "PT", "GBA"]:
users = User.objects.filter(username=username)
else:
users = User.objects.filter(username__startswith=username)
for user in users:
out = []
print "output for {x}".format(x=username)
household = Household.objects.get(user=user)
# ts_raw = household.timeseries.filter(time_step__isnull=True,
# variable__id=VAR_CUMULATIVE)[0]
# series = TSeries(id=ts_raw.id)
timeseries = household \
.timeseries.get(variable__id=VAR_CUMULATIVE)
series = TSeries(id=timeseries.id)
series.read_from_db(db.connection)
timestamps = sorted(series.keys())
values = np.array([])
for ts in timestamps:
val = series[ts]
if isnan(val) or val == 0:
continue
values = np.append(values, val)
#perc = np.percentile(values, 90)
out.append([ts, val])
_outfile = "timeseries_cumulative_%s.csv" % user.username
_path = "data/"
with open(path.join(_path, _outfile), 'w') as of:
a = csv.writer(of, delimiter=',',
quotechar='"',
quoting=csv.QUOTE_ALL)
a.writerows(out)
except Exception as e:
print "failed with %s" % repr(e)
def setUp(self):
self.parms = json.loads(os.getenv('PTHELMA_TEST_ENHYDRIS_API'))
self.cookies = enhydris_api.login(self.parms['base_url'],
self.parms['user'],
self.parms['password'])
# Create two time series
j = {
'gentity': self.parms['station_id'],
'variable': self.parms['variable_id'],
'unit_of_measurement': self.parms['unit_of_measurement_id'],
'time_zone': self.parms['time_zone_id'],
'time_step': 3,
'timestamp_offset_minutes': 0,
'timestamp_offset_months': 0,
'remarks': 'Très importante',
}
self.ts1_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Timeseries', j)
self.ts2_id = enhydris_api.post_model(
self.parms['base_url'], self.cookies, 'Timeseries', j)
assert self.ts1_id != self.ts2_id
# Add some data (all but the last record) to the database
ts = Timeseries(self.ts1_id)
ts.read(StringIO(self.timeseries1_top))
enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
ts = Timeseries(self.ts2_id)
ts.read(StringIO(self.timeseries2_top))
enhydris_api.post_tsdata(self.parms['base_url'], self.cookies, ts)
# Temporary directory for cache files
self.tempdir = tempfile.mkdtemp()
self.savedcwd = os.getcwd()
os.chdir(self.tempdir)
def create_ogr_layer_from_timeseries(filenames, epsg, data_source):
# Prepare the co-ordinate transformation from WGS84 to epsg
source_sr = osr.SpatialReference()
source_sr.ImportFromEPSG(4326)
target_sr = osr.SpatialReference()
target_sr.ImportFromEPSG(epsg)
transform = osr.CoordinateTransformation(source_sr, target_sr)
layer = data_source.CreateLayer('stations', target_sr)
layer.CreateField(ogr.FieldDefn('filename', ogr.OFTString))
for filename in filenames:
with open(filename) as f:
ts = Timeseries()
ts.read_meta(f)
point = ogr.Geometry(ogr.wkbPoint)
point.AddPoint(ts.location['abscissa'], ts.location['ordinate'])
point.Transform(transform)
f = ogr.Feature(layer.GetLayerDefn())
f.SetGeometry(point)
f.SetField('filename', filename)
layer.CreateFeature(f)
return layer
def process_dma(dma, bounds):
"""Process DMA timeseries by aggregating all the contained
households in the DMA"""
print "Process DMA %s" % (dma,)
for dma_series in dma.timeseries.all():
print "Process series %s" % (dma_series,)
per_capita = dma_series.name.find('capita') > -1
variable = dma_series.variable.id
if dma_series.time_step.id == TSTEP_FIFTEEN_MINUTES:
start = bounds[variable]['fifteen_start']
end = bounds[variable]['fifteen_end']
# Fifteen minutes process is DEACTIVATED!
# We don't process fifteen minutes, it takes too long,
# maybe we reactivate later after we optimize the
# algorithm to process only new records
continue
elif dma_series.time_step.id == TSTEP_HOURLY:
start = bounds[variable]['hourly_start']
end = bounds[variable]['hourly_end']
elif dma_series.time_step.id == TSTEP_DAILY:
start = bounds[variable]['daily_start']
end = bounds[variable]['daily_end']
elif dma_series.time_step.id == TSTEP_MONTHLY:
start = bounds[variable]['monthly_start']
end = bounds[variable]['monthly_end']
time_step = ReadTimeStep(dma_series.id, dma_series)
tseries = TSeries(time_step = time_step, id=dma_series.id)
nhseries = TSeries(time_step = time_step)
pointer = start
while pointer<=end:
tseries[pointer] = 0
nhseries[pointer] = 0
pointer = tseries.time_step.next(pointer)
for household in dma.households.all():
for h_series_db in household.timeseries.filter(
time_step__id=dma_series.time_step.id,
variable__id=variable):
hseries = TSeries(id=h_series_db.id)
hseries.read_from_db(db.connection)
pointer = start
while pointer<=end:
try:
v = hseries[pointer]
if math.isnan(v):
pointer = tseries.time_step.next(pointer)
continue
if per_capita:
v = v/float(household.num_of_occupants)
tseries[pointer] += v
nhseries[pointer] += 1
except KeyError:
v = 0
pointer = tseries.time_step.next(pointer)
pointer = start
while pointer<=end:
if per_capita and nhseries[pointer]>0:
tseries[pointer] = tseries[pointer] / nhseries[pointer]
pointer = tseries.time_step.next(pointer)
tseries.write_to_db(db.connection, commit=True)#False)
def testUploadTsData(self):
self.assert_(self.client.login(username='******', password='******'))
response = self.client.put(
"/api/tsdata/1/",
encode_multipart(BOUNDARY,
{'timeseries_records': '2012-11-06 18:17,20,\n'}),
content_type=MULTIPART_CONTENT)
t = Timeseries(1)
t.read_from_db(connection)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, '1')
self.assertEqual(len(t), 1)
self.assertEqual(t.items(0)[0], datetime(2012, 11, 06, 18, 17, 0))
self.assertEqual(t.items(0)[1], 20)
self.client.logout()
# Append two more records
self.assert_(self.client.login(username='******', password='******'))
response = self.client.put(
"/api/tsdata/1/",
encode_multipart(BOUNDARY,
{'timeseries_records':
'2012-11-06 18:18,21,\n2012-11-07 18:18,23,\n'}),
content_type=MULTIPART_CONTENT)
t = Timeseries(1)
t.read_from_db(connection)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content, '2')
self.assertEqual(len(t), 3)
self.assertEqual(t.items(0)[0], datetime(2012, 11, 06, 18, 17, 0))
self.assertEqual(t.items(0)[1], 20)
self.assertEqual(t.items(1)[0], datetime(2012, 11, 06, 18, 18, 0))
self.assertEqual(t.items(1)[1], 21)
self.assertEqual(t.items(2)[0], datetime(2012, 11, 07, 18, 18, 0))
self.assertEqual(t.items(2)[1], 23)
self.client.logout()
# Try to append an earlier record; should fail
self.assert_(self.client.login(username='******', password='******'))
response = self.client.put(
"/api/tsdata/1/",
encode_multipart(BOUNDARY,
{'timeseries_records': '2012-11-05 18:18,21,\n'}),
content_type=MULTIPART_CONTENT)
self.client.logout()
t = Timeseries(1)
t.read_from_db(connection)
self.assertEqual(response.status_code, 409)
self.assertEqual(len(t), 3)
self.client.logout()
def regularize_raw_series(raw_series_db, proc_series_db, rs, re, ps, pe):
"""
This function regularize raw_series_db object from database and
writes a processed proc_series_db in database.
Raw series is a continuously increasing values time series,
aggregating the water consumption. Resulting processed timeseries
contains water consumption for each of its interval. I.e. if the
timeseries is of 15 minutes time step, then each record contains
the water consumption for each record period.
"""
raw_series = TSeries(id=raw_series_db.id)
raw_series.read_from_db(db.connection)
# We keep the last value for x-checking reasons, see last print
# command
test_value = raw_series[raw_series.bounding_dates()[1]]
time_step = ReadTimeStep(proc_series_db.id, proc_series_db)
proc_series = TSeries(id=proc_series_db.id, time_step=time_step)
# The following code can be used in real conditions to append only
# new records to db, in a next version
#if not pe:
# start = proc_series.time_step.down(rs)
#else:
# start = proc_series.time_step.up(pe)
# Instead of the above we use now:
start = proc_series.time_step.down(rs)
end = proc_series.time_step.up(re)
pointer = start
# Pass 1: Initialize proc_series
while pointer <= end:
proc_series[pointer] = float('nan')
pointer = proc_series.time_step.next(pointer)
# Pass 2: Transfer cummulative raw series to differences series:
prev_s = 0
for i in xrange(len(raw_series)):
dat, value = raw_series.items(pos=i)
if not math.isnan(value):
raw_series[dat] = value - prev_s
prev_s = value
# Pass 3: Regularize step: loop over raw series records and distribute
# floating point values to processed series
for i in xrange(len(raw_series)):
dat, value = raw_series.items(pos=i)
if not math.isnan(value):
# find previous, next timestamp of the proc time series
d1 = proc_series.time_step.down(dat)
d2 = proc_series.time_step.up(dat)
if math.isnan(proc_series[d1]): proc_series[d1] = 0
if math.isnan(proc_series[d2]): proc_series[d2] = 0
if d1 == d2: # if dat on proc step then d1=d2
proc_series[d1] += value
continue
dif1 = _dif_in_secs(d1, dat)
dif2 = _dif_in_secs(dat, d2)
dif = dif1 + dif2
# Distribute value to d1, d2
proc_series[d1] += (dif2 / dif) * value
proc_series[d2] += (dif1 / dif) * value
# Uncomment the following line in order to show debug information.
# Usually the three following sums are consistent by equality. If
# not equality is satisfied then there is a likelyhood of algorith
# error
print raw_series.sum(), proc_series.sum(), test_value
proc_series.write_to_db(db=db.connection, commit=True) #False)
#return the full timeseries
return proc_series
def readseries(self,timeseries):
time_step = ReadTimeStep(timeseries.id, timeseries)
timeseries = Timeseries(time_step=time_step,id=timeseries.id)
timeseries.read_from_db(connection)
return timeseries.items()
