def MultiTimeseriesProcessDb(method, timeseries_arg, out_timeseries_id,
db, read_tstep_func, transaction=None,
commit=True, options={}):
out_timeseries = Timeseries(id = out_timeseries_id)
opts = copy.deepcopy(options)
if 'append_only' in opts and opts['append_only']:
bounds = timeseries_bounding_dates_from_db(db,
id = out_timeseries_id)
opts['start_date'] = bounds[1] if bounds else None;
opts['interval_exclusive'] = True
tseries_arg={}
for key in timeseries_arg:
ts = Timeseries(id=timeseries_arg[key])
if ('append_only' in opts and opts['append_only']) \
and opts['start_date'] is not None:
ts.read_from_db(db, bottom_only=True)
if ts.bounding_dates()[0]>opts['start_date']:
ts.read_from_db(db)
else:
ts.read_from_db(db)
ts.time_step = read_tstep_func(ts.id)
tseries_arg[key] = ts
MultiTimeseriesProcess(method, tseries_arg, out_timeseries, opts)
if 'append_only' in opts and opts['append_only']:
out_timeseries.append_to_db(db=db, transaction=transaction,
commit=commit)
else:
out_timeseries.write_to_db(db=db, transaction=transaction,
commit=commit)
def put(self, request, pk, format=None):
try:
ts = Timeseries(id=int(pk))
self.check_object_permissions(request, ts)
result_if_error = status.HTTP_400_BAD_REQUEST
ts.read(StringIO(request.DATA['timeseries_records']))
result_if_error = status.HTTP_409_CONFLICT
ts.append_to_db(connection, commit=False)
return HttpResponse(str(len(ts)), content_type="text/plain")
except ValueError as e:
return HttpResponse(status=result_if_error,
content=str(e),
content_type="text/plain")
def put(self, request, pk, format=None):
try:
ts = Timeseries(id=int(pk))
self.check_object_permissions(request, ts)
result_if_error = status.HTTP_400_BAD_REQUEST
ts.read(StringIO(request.DATA['timeseries_records']))
result_if_error = status.HTTP_409_CONFLICT
ts.append_to_db(connection, commit=False)
return HttpResponse(str(len(ts)), content_type="text/plain")
except ValueError as e:
return HttpResponse(status=result_if_error,
content=str(e),
content_type="text/plain")
def create_objects(data, usernames, force, z_names, z_dict):
"""
:param data: meter_id -> consumption_type -> [timestamp, volume]
:param force: True to overwrite
:return: True for success
"""
households = []
# Create user (household owner), household, database series placeholders
hh_ids = sorted(data.keys())
found = False
for hh_id in hh_ids:
username = usernames[hh_id]
if username == "PT94993":
pass
try:
zone_name = z_dict[username]
except KeyError:
zone_name = z_names[0]
zone = DMA.objects.get(name=zone_name)
user, created = create_user(username, hh_id)
household, found = create_household(hh_id, user, zone.id)
households.append(household)
db_series = create_raw_timeseries(household)
create_processed_timeseries(household)
timeseries_data = {}
# Now we will create timeseries.Timeseries() and we will add
# parsed values
for variable in db_series:
if variable not in ('WaterCold', 'Electricity'):
continue
exists = False
s, e = timeseries_bounding_dates_from_db(db.connection,
db_series[variable].id)
latest_ts = e
ts_id = db_series[variable].id
# checking to see if timeseries records already exist in order
# to append
# d = read_timeseries_tail_from_db(db.connection, ts_id)
total = 0.0
# if s or e:
# exists = True
# timeseries = TSeries(ts_id)
# timeseries.read_from_db(db.connection)
# else:
# timeseries = TSeries()
# timeseries.id = ts_id
_dict = data[hh_id]
arr = _dict[variable]
series = arr
if not series:
continue
earlier = []
if (not latest_ts) or (latest_ts < series[0][0]): # append
timeseries = TSeries()
timeseries.id = ts_id
try:
tail = read_timeseries_tail_from_db(db.connection, ts_id)
total = float(tail[1]) # keep up from last value
except Exception as e:
log.debug(repr(e))
total = 0
for timestamp, value in series:
if (not latest_ts) or (timestamp > latest_ts):
if not isnan(value):
total += value
timeseries[timestamp] = total
else:
timeseries[timestamp] = float('NaN')
elif timestamp < latest_ts:
earlier.append((timestamp, value))
timeseries.append_to_db(db=db.connection,
transaction=transaction,
commit=True)
elif latest_ts >= series[0][0]:
if not force: # ignore
continue
else: # insert
for timestamp, value in series:
if timestamp < latest_ts:
earlier.append((timestamp, value))
if earlier and ("GR" in username
or "GBA" in username): # insert (only for athens)
# print "appending %s items for %s" % (len(earlier), username)
if variable == "WaterCold":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
series15 = TSeries(id=ts15.id)
elif variable == "Electricity":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_ENERGY_PERIOD)
series15 = TSeries(id=ts15.id)
series15.read_from_db(db.connection)
for ts, value in earlier:
series15[ts] = value
series15.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
raw_ts = TSeries(ts_id) # read existing ts raw data
raw_ts.read_from_db(db.connection)
total = get_consumption_totals(household, earlier[0][0],
variable)
init = total
for timestamp, value in earlier:
if not isnan(value):
total += value
raw_ts[timestamp] = total
else:
raw_ts[timestamp] = float('NaN')
# correct later values, too
diff = total - init
all_ts = sorted(raw_ts.keys())
for ts in all_ts:
if ts <= timestamp:
continue
curr = raw_ts[ts]
raw_ts[ts] = curr + diff
raw_ts.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
if 'WaterCold' in timeseries_data and not found: # only for new HH
calc_occupancy(timeseries_data['WaterCold'], household)
return households
def create_objects(data, usernames, force, z_names, z_dict):
"""
:param data: meter_id -> consumption_type -> [timestamp, volume]
:param force: True to overwrite
:return: True for success
"""
households = []
# Create user (household owner), household, database series placeholders
hh_ids = sorted(data.keys())
found = False
for hh_id in hh_ids:
username = usernames[hh_id]
if username == "PT94993":
pass
try:
zone_name = z_dict[username]
except KeyError:
zone_name = z_names[0]
zone = DMA.objects.get(name=zone_name)
user, created = create_user(username, hh_id)
household, found = create_household(hh_id, user, zone.id)
households.append(household)
db_series = create_raw_timeseries(household)
create_processed_timeseries(household)
timeseries_data = {}
# Now we will create timeseries.Timeseries() and we will add
# parsed values
for variable in db_series:
if variable not in ('WaterCold', 'Electricity'):
continue
exists = False
s, e = timeseries_bounding_dates_from_db(db.connection,
db_series[variable].id)
latest_ts = e
ts_id = db_series[variable].id
# checking to see if timeseries records already exist in order
# to append
# d = read_timeseries_tail_from_db(db.connection, ts_id)
total = 0.0
# if s or e:
# exists = True
# timeseries = TSeries(ts_id)
# timeseries.read_from_db(db.connection)
# else:
# timeseries = TSeries()
# timeseries.id = ts_id
_dict = data[hh_id]
arr = _dict[variable]
series = arr
if not series:
continue
earlier = []
if (not latest_ts) or (latest_ts < series[0][0]): # append
timeseries = TSeries()
timeseries.id = ts_id
try:
tail = read_timeseries_tail_from_db(db.connection, ts_id)
total = float(tail[1]) # keep up from last value
except Exception as e:
log.debug(repr(e))
total = 0
for timestamp, value in series:
if (not latest_ts) or (timestamp > latest_ts):
if not isnan(value):
total += value
timeseries[timestamp] = total
else:
timeseries[timestamp] = float('NaN')
elif timestamp < latest_ts:
earlier.append((timestamp, value))
timeseries.append_to_db(db=db.connection,
transaction=transaction,
commit=True)
elif latest_ts >= series[0][0]:
if not force: # ignore
continue
else: # insert
for timestamp, value in series:
if timestamp < latest_ts:
earlier.append((timestamp, value))
if earlier and ("GR" in username or "GBA" in username): # insert (only for athens)
# print "appending %s items for %s" % (len(earlier), username)
if variable == "WaterCold":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
series15 = TSeries(id=ts15.id)
elif variable == "Electricity":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_ENERGY_PERIOD)
series15 = TSeries(id=ts15.id)
series15.read_from_db(db.connection)
for ts, value in earlier:
series15[ts] = value
series15.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
raw_ts = TSeries(ts_id) # read existing ts raw data
raw_ts.read_from_db(db.connection)
total = get_consumption_totals(household, earlier[0][0],
variable)
init = total
for timestamp, value in earlier:
if not isnan(value):
total += value
raw_ts[timestamp] = total
else:
raw_ts[timestamp] = float('NaN')
# correct later values, too
diff = total - init
all_ts = sorted(raw_ts.keys())
for ts in all_ts:
if ts <= timestamp:
continue
curr = raw_ts[ts]
raw_ts[ts] = curr + diff
raw_ts.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
if 'WaterCold' in timeseries_data and not found: # only for new HH
calc_occupancy(timeseries_data['WaterCold'], household)
return households
