def parse_and_save_timeseries(device_id, timeseries_id):
"""
Reads a RAW timeseries from REST API and saves in our local
database using the timeseries_id.
``device_id`` will be the ``identifier`` used in other functions,
usualy is the customerID==deviceID
"""
s, e = timeseries_bounding_dates_from_db(db.connection, timeseries_id)
if s or e:
print 'Raw timeseries id=%s has already data, skipping...' % (
timeseries_id, )
return
timeseries = TSeries()
timeseries.id = timeseries_id
for timestamp, value in ibm_restapi.get_raw_timeseries(device_id):
timeseries[timestamp] = value
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
def create_objects(dma, household_identifier, series, force=False):
"""
When a household is fully parsed then this command is called to create
database objects thus: user (household owner), household, database time
series placeholders (for raw data and for processed data), to write actual
time series data in database and finally to estimate the household
occupancy.
"""
print "Processing household %s, user username will be %s as well"%(
household_identifier, household_identifier)
# Create user (household owner), household, database series placeholders
user = create_user(household_identifier)
household=create_household(household_identifier, user,
zone=dma.id)
db_series = create_raw_timeseries(household)
create_processed_timeseries(household)
timeseries_data = {}
# Now we will create timeseries.Timeseries() and we will add
# parsed values
for variable in db_series:
if variable not in ('WaterCold', 'Electricity'):
continue
s, e = timeseries_bounding_dates_from_db(db.connection,
db_series[variable].id)
if not force and (s or e):
print 'Raw timeseries id=%s has already data, skipping...'%(
db_series[variable].id,)
continue
timeseries = TSeries()
timeseries.id = db_series[variable].id
total = 0.0
for timestamp, value in series[variable]:
if not math.isnan(value):
total += value
timeseries[timestamp] = total
else:
timeseries[timestamp] = float('NaN')
timeseries_data[variable] = timeseries
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
if 'WaterCold' in timeseries_data:
calc_occupancy(timeseries_data['WaterCold'], household)
def parse_and_save_timeseries(device_id, timeseries_id):
"""
Reads a RAW timeseries from REST API and saves in our local
database using the timeseries_id.
``device_id`` will be the ``identifier`` used in other functions,
usualy is the customerID==deviceID
"""
s, e = timeseries_bounding_dates_from_db(db.connection,
timeseries_id)
if s or e:
print 'Raw timeseries id=%s has already data, skipping...'%(
timeseries_id,)
return
timeseries = TSeries()
timeseries.id = timeseries_id
for timestamp, value in ibm_restapi.get_raw_timeseries(device_id):
timeseries[timestamp] = value
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
def parse_and_save_timeseries(filename, timeseries_id):
first_line = True
timeseries = TSeries()
timeseries.id = timeseries_id
with open(filename) as fp:
for line in fp.readlines():
if first_line:
first_line = False
continue
components = line.split(',')
date_str = components[1].strip('"')
value_str = components[2].strip('"')
value = float(value_str)
if value < MIN_VALUE or value >= MAX_VALUE:
value = float('nan')
tstamp = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S')
tstamp = tstamp.replace(second=0)
timeseries[tstamp] = value
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
def parse_and_save_timeseries(filename, timeseries_id):
first_line = True
timeseries = TSeries()
timeseries.id = timeseries_id
with open(filename) as fp:
for line in fp.readlines():
if first_line:
first_line = False
continue
components = line.split(',')
date_str = components[1].strip('"')
value_str = components[2].strip('"')
value = float(value_str)
if value<MIN_VALUE or value>=MAX_VALUE:
value = float('nan')
tstamp = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S')
tstamp = tstamp.replace(second=0)
timeseries[tstamp] = value
timeseries.write_to_db(db=db.connection,
transaction=transaction,
commit=False)
def create_objects(data, usernames, force, z_names, z_dict):
"""
:param data: meter_id -> consumption_type -> [timestamp, volume]
:param force: True to overwrite
:return: True for success
"""
households = []
# Create user (household owner), household, database series placeholders
hh_ids = sorted(data.keys())
found = False
for hh_id in hh_ids:
username = usernames[hh_id]
if username == "PT94993":
pass
try:
zone_name = z_dict[username]
except KeyError:
zone_name = z_names[0]
zone = DMA.objects.get(name=zone_name)
user, created = create_user(username, hh_id)
household, found = create_household(hh_id, user, zone.id)
households.append(household)
db_series = create_raw_timeseries(household)
create_processed_timeseries(household)
timeseries_data = {}
# Now we will create timeseries.Timeseries() and we will add
# parsed values
for variable in db_series:
if variable not in ('WaterCold', 'Electricity'):
continue
exists = False
s, e = timeseries_bounding_dates_from_db(db.connection,
db_series[variable].id)
latest_ts = e
ts_id = db_series[variable].id
# checking to see if timeseries records already exist in order
# to append
# d = read_timeseries_tail_from_db(db.connection, ts_id)
total = 0.0
# if s or e:
# exists = True
# timeseries = TSeries(ts_id)
# timeseries.read_from_db(db.connection)
# else:
# timeseries = TSeries()
# timeseries.id = ts_id
_dict = data[hh_id]
arr = _dict[variable]
series = arr
if not series:
continue
earlier = []
if (not latest_ts) or (latest_ts < series[0][0]): # append
timeseries = TSeries()
timeseries.id = ts_id
try:
tail = read_timeseries_tail_from_db(db.connection, ts_id)
total = float(tail[1]) # keep up from last value
except Exception as e:
log.debug(repr(e))
total = 0
for timestamp, value in series:
if (not latest_ts) or (timestamp > latest_ts):
if not isnan(value):
total += value
timeseries[timestamp] = total
else:
timeseries[timestamp] = float('NaN')
elif timestamp < latest_ts:
earlier.append((timestamp, value))
timeseries.append_to_db(db=db.connection,
transaction=transaction,
commit=True)
elif latest_ts >= series[0][0]:
if not force: # ignore
continue
else: # insert
for timestamp, value in series:
if timestamp < latest_ts:
earlier.append((timestamp, value))
if earlier and ("GR" in username
or "GBA" in username): # insert (only for athens)
# print "appending %s items for %s" % (len(earlier), username)
if variable == "WaterCold":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
series15 = TSeries(id=ts15.id)
elif variable == "Electricity":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_ENERGY_PERIOD)
series15 = TSeries(id=ts15.id)
series15.read_from_db(db.connection)
for ts, value in earlier:
series15[ts] = value
series15.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
raw_ts = TSeries(ts_id) # read existing ts raw data
raw_ts.read_from_db(db.connection)
total = get_consumption_totals(household, earlier[0][0],
variable)
init = total
for timestamp, value in earlier:
if not isnan(value):
total += value
raw_ts[timestamp] = total
else:
raw_ts[timestamp] = float('NaN')
# correct later values, too
diff = total - init
all_ts = sorted(raw_ts.keys())
for ts in all_ts:
if ts <= timestamp:
continue
curr = raw_ts[ts]
raw_ts[ts] = curr + diff
raw_ts.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
if 'WaterCold' in timeseries_data and not found: # only for new HH
calc_occupancy(timeseries_data['WaterCold'], household)
return households
def create_objects(data, usernames, force, z_names, z_dict):
"""
:param data: meter_id -> consumption_type -> [timestamp, volume]
:param force: True to overwrite
:return: True for success
"""
households = []
# Create user (household owner), household, database series placeholders
hh_ids = sorted(data.keys())
found = False
for hh_id in hh_ids:
username = usernames[hh_id]
if username == "PT94993":
pass
try:
zone_name = z_dict[username]
except KeyError:
zone_name = z_names[0]
zone = DMA.objects.get(name=zone_name)
user, created = create_user(username, hh_id)
household, found = create_household(hh_id, user, zone.id)
households.append(household)
db_series = create_raw_timeseries(household)
create_processed_timeseries(household)
timeseries_data = {}
# Now we will create timeseries.Timeseries() and we will add
# parsed values
for variable in db_series:
if variable not in ('WaterCold', 'Electricity'):
continue
exists = False
s, e = timeseries_bounding_dates_from_db(db.connection,
db_series[variable].id)
latest_ts = e
ts_id = db_series[variable].id
# checking to see if timeseries records already exist in order
# to append
# d = read_timeseries_tail_from_db(db.connection, ts_id)
total = 0.0
# if s or e:
# exists = True
# timeseries = TSeries(ts_id)
# timeseries.read_from_db(db.connection)
# else:
# timeseries = TSeries()
# timeseries.id = ts_id
_dict = data[hh_id]
arr = _dict[variable]
series = arr
if not series:
continue
earlier = []
if (not latest_ts) or (latest_ts < series[0][0]): # append
timeseries = TSeries()
timeseries.id = ts_id
try:
tail = read_timeseries_tail_from_db(db.connection, ts_id)
total = float(tail[1]) # keep up from last value
except Exception as e:
log.debug(repr(e))
total = 0
for timestamp, value in series:
if (not latest_ts) or (timestamp > latest_ts):
if not isnan(value):
total += value
timeseries[timestamp] = total
else:
timeseries[timestamp] = float('NaN')
elif timestamp < latest_ts:
earlier.append((timestamp, value))
timeseries.append_to_db(db=db.connection,
transaction=transaction,
commit=True)
elif latest_ts >= series[0][0]:
if not force: # ignore
continue
else: # insert
for timestamp, value in series:
if timestamp < latest_ts:
earlier.append((timestamp, value))
if earlier and ("GR" in username or "GBA" in username): # insert (only for athens)
# print "appending %s items for %s" % (len(earlier), username)
if variable == "WaterCold":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_PERIOD)
series15 = TSeries(id=ts15.id)
elif variable == "Electricity":
ts15 = household \
.timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
variable__id=VAR_ENERGY_PERIOD)
series15 = TSeries(id=ts15.id)
series15.read_from_db(db.connection)
for ts, value in earlier:
series15[ts] = value
series15.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
raw_ts = TSeries(ts_id) # read existing ts raw data
raw_ts.read_from_db(db.connection)
total = get_consumption_totals(household, earlier[0][0],
variable)
init = total
for timestamp, value in earlier:
if not isnan(value):
total += value
raw_ts[timestamp] = total
else:
raw_ts[timestamp] = float('NaN')
# correct later values, too
diff = total - init
all_ts = sorted(raw_ts.keys())
for ts in all_ts:
if ts <= timestamp:
continue
curr = raw_ts[ts]
raw_ts[ts] = curr + diff
raw_ts.write_to_db(db=db.connection,
transaction=transaction,
commit=True)
if 'WaterCold' in timeseries_data and not found: # only for new HH
calc_occupancy(timeseries_data['WaterCold'], household)
return households
