def __init__(self, start=None, end=None, timezone='US/Pacific'):
""" params:
start: (string, "%Y-%m-%d %H:%M:%S %Z") When should the data start.
end: (string, "%Y-%m-%d %H:%M:%S %Z") when should the data end.
timezone: (string) As used by pytz. Used for both start and end
"""
# data clients
self.client = mdal.MDALClient("xbos/mdal")
self.hod = HodClient("xbos/hod")
self.SITE = "ciee"
tz = pytz.timezone('US/Pacific')
temp_end = dati.datetime.today() if end is None else datetime.strptime(
end, "%Y-%m-%d %H:%M:%S %Z")
temp_start = (
temp_end -
dati.timedelta(10)) if start is None else datetime.strptime(
start, "%Y-%m-%d %H:%M:%S %Z")
self.end = tz.localize(temp_end)
self.start = tz.localize(temp_start)
self.zone_sensor_df = self.get_occupancy()
self.building_df = self.get_building_occupancy()
self.zone_df = self.get_zone_occupancy()
def preprocess_occ_mdal(self):
"""
Returns the required dataframe for the occupancy predictions
-------
Pandas DataFrame
"""
hod = HodClient(
"xbos/hod", self.c
) # TODO MAKE THIS WORK WITH FROM AND xbos/hod, FOR SOME REASON IT DOES NOT
occ_query = """SELECT ?sensor ?uuid ?zone FROM %s WHERE {
?sensor rdf:type brick:Occupancy_Sensor .
?sensor bf:isPointOf/bf:isPartOf ?zone .
?sensor bf:uuid ?uuid .
?zone rdf:type brick:HVAC_Zone
};
""" % self.controller_cfg[
"Building"] # get all the occupancy sensors uuids
results = hod.do_query(occ_query) # run the query
uuids = [[x['?zone'], x['?uuid']] for x in results['Rows']] # unpack
# only choose the sensors for the zone specified in cfg
query_list = []
for i in uuids:
if i[0] == self.zone:
query_list.append(i[1])
# get the sensor data
c = mdal.MDALClient("xbos/mdal", client=self.c)
dfs = c.do_query({
'Composition': query_list,
'Selectors': [mdal.MAX] * len(query_list),
'Time': {
'T0':
(self.now - timedelta(days=25)).strftime('%Y-%m-%d %H:%M:%S') +
' UTC',
'T1':
self.now.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize':
str(self.interval) + 'min',
'Aligned':
True
}
})
dfs = pd.concat([dframe for uid, dframe in dfs.items()], axis=1)
df = dfs[[query_list[0]]]
df.columns.values[0] = 'occ'
df.is_copy = False
df.columns = ['occ']
# perform OR on the data, if one sensor is activated, the whole zone is considered occupied
for i in range(1, len(query_list)):
df.loc[:, 'occ'] += dfs[query_list[i]]
df.loc[:, 'occ'] = 1 * (df['occ'] > 0)
return df.tz_localize(None)
def __init__(self):
self.bw_client = Client()
self.bw_client.setEntityFromEnviron()
self.bw_client.overrideAutoChainTo(True)
self.hod_client = HodClient("xbos/hod", self.bw_client)
self.priority = {
"fan": 1,
"kettle": 2,
"student_office_tstat": 3,
"microwave": 4,
"sra_office_tstat": 5,
"space_heater": 5,
"fridge": 6,
"evse": 7,
"michaels_office_tstat": 8
}
meter_q = """
SELECT ?uri FROM rfs WHERE {
?m rdf:type brick:Building_Electric_Meter .
?m bf:uri ?uri .
};
"""
self.building_meter = Meter(
self.bw_client,
self.hod_client.do_query(meter_q)['Rows'][0]['?uri'])
self.tstats = {
"student_office_tstat":
Thermostat(
self.bw_client,
"rfs/devices/s.pelican/Student_Office/i.xbos.thermostat"),
"sra_office_tstat":
Thermostat(self.bw_client,
"rfs/devices/s.pelican/SRA_Office/i.xbos.thermostat"),
"michaels_office_tstat":
Thermostat(
self.bw_client,
"rfs/devices/s.pelican/Michaels_Office/i.xbos.thermostat")
}
self.plugloads = {
"fan":
Plug(self.bw_client, "rfs/devices/fan/s.tplink.v0/0/i.xbos.plug"),
"fridge":
Plug(self.bw_client,
"rfs/devices/refrigerator/s.tplink.v0/0/i.xbos.plug"),
"space_heater":
Plug(self.bw_client,
"rfs/devices/heater/s.tplink.v0/0/i.xbos.plug"),
"kettle":
Plug(self.bw_client,
"rfs/devices/boiler/s.tplink.v0/0/i.xbos.plug"),
"microwave":
Plug(self.bw_client,
"rfs/devices/microwave/s.tplink.v0/0/i.xbos.plug")
}
self.evse = EVSE(self.bw_client,
"rfs/devices/s.aerovironment/ParkingLot/i.xbos.evse")
self.evse.set_current_limit(_MAX_EVSE_CURRENT)
def getTstats(self):
hod_client = HodClient("xbos/hod", self.client)
thermostat_query_data = hod_client.do_query(self.thermostat_query %
self.building)["Rows"]
return {
tstat["?zone"]: Thermostat(client, tstat["?uri"])
for tstat in thermostat_query_data
}
def __init__(self):
# get a bosswave client
c = get_client(entity="/Users/Daniel/CIEE/SetUp/ciee_readonly.ent", agent="127.0.0.1:28589")
# get a HodDB client
self.hod = HodClient("ciee/hod", c)
# get an archiver client
self.archiver = DataClient(c, archivers=["ucberkeley"])
self.zone_sensor_df = self.get_occupancy()
self.building_df = self.get_building_occupany()
self.zone_df = self.get_zone_occupancy()
def __init__(self, building_cfg, client, interval=5):
"""
:param building_cfg: A dictionary which should include data for keys "Building" and "Interval_Length"
:param client: An xbos client.
:param interval: the interval in which to split thermal data actions.
"""
self.building_cfg = building_cfg
self.building = building_cfg["Building"]
self.interval = interval # the interval in which to split thermal data actions.
self.client = client
self.window_size = 1 # minutes. TODO should come from config.
self.hod_client = HodClient("xbos/hod", self.client) # TODO Potentially could incorporate this into MDAL query.
def preprocess_occ(self):
#this only works for ciee, check how it should be writen properly:
hod = HodClient("ciee/hod", self.c)
occ_query = """SELECT ?sensor ?uuid ?zone WHERE {
?sensor rdf:type brick:Occupancy_Sensor .
?sensor bf:isLocatedIn/bf:isPartOf ?zone .
?sensor bf:uuid ?uuid .
?zone rdf:type brick:HVAC_Zone
};
"""
results = hod.do_query(occ_query)
uuids = [[x['?zone'], x['?uuid']] for x in results['Rows']]
query_list = []
for i in uuids:
if i[0] == self.zone:
query_list.append(i[1])
c = mdal.MDALClient("xbos/mdal")
dfs = c.do_query({
'Composition': query_list,
'Selectors': [mdal.MAX] * len(query_list),
'Time': {
'T0':
(self.now - timedelta(days=30)).strftime('%Y-%m-%d %H:%M:%S') +
' UTC',
'T1':
self.now.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize':
str(self.interval) + 'min',
'Aligned':
True
}
})
dfs = pd.concat([dframe for uid, dframe in dfs.items()], axis=1)
df = dfs[[query_list[0]]]
df.columns.values[0] = 'occ'
df.is_copy = False
df.columns = ['occ']
for i in range(1, len(query_list)):
df.loc[:, 'occ'] += dfs[query_list[i]]
df.loc[:, 'occ'] = 1 * (df['occ'] > 0)
return df.tz_localize(None)
def __init__(self):
# get a bosswave client
client = get_client() # defaults to $BW2_AGENT, $BW2_DEFAULT_ENTITY
# Get hod client.
hod_client = HodClient("xbos/hod", client)
self.building_tstats = {}
hod_xsg_match = True
try:
for bldg in XSG_ALL_BUILDINGS:
# Getting the tstats for the building.
self.building_tstats[bldg] = get_all_thermostats(
client, hod_client, bldg)
if not set(XSG_ALL_ZONES[bldg]).issubset(
set(self.building_tstats[bldg])):
missing_zones = []
for zone in XSG_ALL_ZONES[bldg]:
if zone not in self.building_tstats[bldg]:
missing_zones.append(zone)
logging.critical(
"zone mismatch between hod and xbos_services_getter for bldg: %s \nhod_zones:%s\nmissing zones:%s\n",
bldg, self.building_tstats[bldg].keys(), missing_zones)
hod_xsg_match = False
if not hod_xsg_match:
sys.exit(0)
except Exception:
tb = traceback.format_exc()
logging.critical("failed to get thermostats\n%s", tb)
sys.exit(0)
def __init__(
self,
controller_cfg,
client,
now=datetime.datetime.utcnow().replace(tzinfo=pytz.timezone("UTC"))):
self.controller_cfg = controller_cfg
self.pytz_timezone = pytz.timezone(controller_cfg["Pytz_Timezone"])
self.interval = controller_cfg["Interval_Length"]
self.now = now.astimezone(self.pytz_timezone)
self.client = client
self.window_size = 1 # minutes. TODO should come from config. Has to be a multiple of 15 for weather getting.
self.building = controller_cfg["Building"]
self.hod_client = HodClient(
"xbos/hod", self.client
) # TODO hopefully i could incorporate this into the query.
import time
import pytz
from datetime import datetime, timedelta
# this is required for sending emails
import boto3
c = boto3.client('sns', region_name='us-east-2')
# XBOS services: Hod for getting UUIDs from query, MDAL for getting data
from xbos.services.hod import HodClient
hod = HodClient("xbos/hod")
import xbos.services.mdal as mdal
data = mdal.MDALClient("xbos/mdal")
# how many UUIDs to investigate at a time
CHUNK_UUID = 10
oldmsg = ""
# Here, we get the point names and corresponding UUIDs for all Brick models. We provide "values_only" to be False so that
# we can get the full URIs of the results, which can tell us which namespace the result is in.
# If we want to only do the scan for one namespace, we can change the "*" to that namespace
resp = hod.do_query("SELECT ?point ?uuid FROM * WHERE { ?point bf:uuid ?uuid };", values_only=False)
# extract UUIDs
uuids = [x['?uuid']['Value'] for x in resp['Rows']]
# extract point names as full URIs
points = [x['?point']['Namespace']+'#'+x['?point']['Value'] for x in resp['Rows']]
# Every 30 minutes, we initiate a scan of all the streams (identified by UUIDs) that we want to monitor.
# We pull out 15-min aggregates of each stream from the last 30 minutes and see if there is any data in that range
while True:
start = time.time()
def __init__(self,
cfg,
now=datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC")).astimezone(
tz=pytz.timezone("America/Los_Angeles"))):
# query the server to get all the available occupancy sensors
zone_name = cfg['zone']
if cfg['Server']:
c = get_client(agent='172.17.0.1:28589', entity="thanos.ent")
else:
c = get_client()
archiver = DataClient(c)
hod = HodClient("ciee/hod", c)
occ_query = """SELECT ?sensor ?uuid ?zone WHERE {
?sensor rdf:type brick:Occupancy_Sensor .
?sensor bf:isLocatedIn/bf:isPartOf ?zone .
?sensor bf:uuid ?uuid .
?zone rdf:type brick:HVAC_Zone
};
"""
results = hod.do_query(occ_query)
uuids = [[x['?zone'], x['?uuid']] for x in results['Rows']]
temp_now = now
# select the sensors that are contained in the zone we are optimizing for
query_list = []
for i in uuids:
if i[0] == zone_name:
query_list.append(i[1])
start = '"' + (temp_now).strftime('%Y-%m-%d %H:%M:%S') + ' PST"'
end = '"' + (temp_now - timedelta(days=30)
).strftime('%Y-%m-%d %H:%M:%S') + ' PST"'
dfs = make_dataframe(
archiver.window_uuids(query_list, end, start, '15min',
timeout=120))
for uid, df in dfs.items():
if 'mean' in df.columns:
df = df[['mean']]
df.columns = ['occ']
dfs[uid] = df.resample('15min').mean()
df = dfs.values()[0]
if len(dfs) > 1:
for newdf in dfs.values()[1:]:
df['occ'] += newdf['occ']
df['occ'] = 1 * (df['occ'] > 0)
df.index = df.index.tz_localize(pytz.timezone("America/Los_Angeles"))
observation_length_addition = 4 * 60
k = 5
prediction_time = 4 * 60
resample_time = 15
#now = df.index[-prediction_time/resample_time]
now = df.index[-1]
observation_length = mins_in_day(now) + observation_length_addition
similar_moments = find_similar_days(df, now, observation_length, k)
self.predictions = predict(df, now, similar_moments, prediction_time,
resample_time)
class Occupancy():
def __init__(self):
# get a bosswave client
c = get_client(entity="/Users/Daniel/CIEE/SetUp/ciee_readonly.ent", agent="127.0.0.1:28589")
# get a HodDB client
self.hod = HodClient("ciee/hod", c)
# get an archiver client
self.archiver = DataClient(c, archivers=["ucberkeley"])
self.zone_sensor_df = self.get_occupancy()
self.building_df = self.get_building_occupany()
self.zone_df = self.get_zone_occupancy()
def get_occupancy(self, start='"2017-09-01 00:00:00 MST"', end='"2017-09-12 00:00:00 MST"', time_steps="15T"):
"""Returns a dictionary {Zone: occupancy_data} where the occupancy data is a pandas
dataframe, where each column represents a sensor. It is indexed as timeseries data with time_steps as given.
It has all the data in the range of times start to end as given.
Note, the data is given in MST time as that is how it stores. We will convert it in the function to PST.
Parameters:
start: When should the data start. Has to be a string of a string '"Time"'.
end: when should the data end. Has to be a string of a string '"Time"'.
time_steps: in what intervals (start, start+time+steps) do we want to look at data.
If a sensor regiester someone in those 15minutes,
then we will treat the whole 15 minute as being occupied. (Is this a valid assumption?)
Returns:
A dataframe. Only returns the data for whole days. Will truncate edge days if needed.
IMPROVEMENT TO ME: maybe make it such that we don't have to store the data in a different cell. such that
we only ever have to make a call to this function, and it will somehow store the already pulled function,
such that it doesn't have to pull it over and over again.
"""
# define a Brick query to get the occupancy information
q = """SELECT ?zone ?sensor_uuid WHERE {
?zone rdf:type brick:HVAC_Zone .
?zone bf:hasPart ?room .
?sensor bf:isLocatedIn ?room .
?sensor rdf:type/rdfs:subClassOf* brick:Occupancy_Sensor .
?sensor bf:uuid ?sensor_uuid .
};
"""
result = self.hod.do_query(q)['Rows']
occupancy_sensors = defaultdict(list)
for sensor in result:
occupancy_sensors[sensor['?zone']].append(sensor['?sensor_uuid'])
zone_sensor_df = defaultdict()
for zone, sensor_uuid in occupancy_sensors.items():
occupancy_data = make_dataframe(self.archiver.data_uuids(sensor_uuid, start, end, timeout=300))
occupancy_df = merge_dfs(occupancy_data, resample=time_steps, do_max=True)
idx = occupancy_df.index
idx = idx.tz_localize(pytz.utc).tz_convert(tz)
occupancy_df.index = idx
#The following will make sure that we only have whole days in the returned DF. Only because of timezones.
# Need to truncate ends since we don't have more data.
if occupancy_df.index[0].time() > pd.Timestamp("00:00:00").time():
temp = occupancy_df.index[0]
d_temp = pd.Timestamp(temp + pd.Timedelta("1 days"))
d_temp = d_temp.replace(hour=0,minute=0,second=0)
occupancy_df = occupancy_df.loc[d_temp:]
if occupancy_df.index[-1].time() < pd.Timestamp("23:45:00").time():
temp = occupancy_df.index[-1]
d_temp = pd.Timestamp(temp + pd.Timedelta("-1 days"))
d_temp = d_temp.replace(hour=23,minute=45,second=0)
occupancy_df = occupancy_df.loc[:d_temp]
zone_sensor_df[zone] = 1*(occupancy_df > 0)
return zone_sensor_df
def compute_envelope(self, start="08:00:00", end="18:00:00", time_steps="15T"):
"""Computes an envelope for a day. Will be filled with 1's from start to end,
and the time series will have a period of time_steps.
Parameters:
start: When the envelope should start. Provide pandas time string. should be less than end.
end: When the envelope should end. Provdie pandas time string. should be greater than start.
time_steps: the period of the indexing. Taken to be that if at time 08:15:00 we have a one, then
in the period between 08:15:00 and 08:30:00 someone was by the occupancy sensor."""
t = pd.DatetimeIndex(freq='15T', start='2017-01-01 00:00:00', end='2017-01-01 23:59:59')
envelope = pd.Series(data=0, index=t)
idx_t = envelope.index
selector = (t.hour >= 8) & (t.hour < 18)
envelope.loc[selector] = 1
envelope.index = idx_t.time # is this necessary? I would like to have it independent of day though.
return envelope
def hamming_distance(self, arr, compare_to):
"""Returns hamming distance of two numpy arrays.
>>> a = np.array([1,0,1])
>>> b = np.array([0,1,0])
>>> hamming_distance(a, b)
3
>>> c = np.array([1])
>>> hamming_distance(a, c)
1
The last output is a result from appending an array of zeros to c to make a and c equal length."""
return np.count_nonzero(arr != compare_to)
def get_zone_occupancy(self, cond=(lambda x: True)):
"""Returns the logical or of all sensors in a zone.
Parameters:
cond: A function on a timestamp by which to filter the data.
return:
A dictionary of key=zone and value=pd.df(occupancy)"""
result = defaultdict()
for zone, df in self.zone_sensor_df.items():
temp = df[df.index.map(cond)]
result[zone] = temp.apply(lambda x: max(x), axis = 1)
return result
def get_building_occupany(self, cond=(lambda x: True)):
"""Returns the logical or of all sensors.
Parameters:
cond: A function on a timestamp by which to filter the data.
Returns:
pd.DataFrame"""
result = None
# putting all sensors in one pd.df.
for key, value in self.zone_sensor_df.items():
if result is None:
result = value
else:
result = pd.concat([result, value], axis=1)
idx = list(result.index.map(cond))
result = result[idx] # index the rows to filter by times.
result = result.apply(lambda x: max(x), axis=1) # applying a function to occupancy value of all zones at a given time
return pd.DataFrame(data=result.values, columns=["Occupied"], index=result.index)
# Improvement: Right now only works for a data df with one column or pandas series
def compute_histogram(self, data, cond_weight, equivalence_classes):
"""Creates a weighted histogram of the data.
The histograms will be computed for all equivalent dates for the given equivalence.
For example, if we give the equivalence relation (lambda date, equivalent_day: date.dayofweek == equivalent_day)
and the list of equivalences we want to find [0,1,2,3,4,5,6],
then we will get a dictionary {e: histogram for equivalent days}. So, for Monday I would get
{0: histogram for all mondays}. Furthermore, every equivalence is given a weight which depends on the equivalence
class and the input day. So, we could weight days which are further in the past less.
Frequency of histrogram will be the same as the data given.
Parameters:
data: The data for which to compute the histogram.
cond_weight: The condition by which to establish equivalence or the equialence relation and returns the weight
of the specific equivalence and day. Has to have the input format
cond_weight(day, equivalence). (Returns: int: weight). A weight of 0 means no equivalence and a weight of one means
full equivalence.
equivalence_classes: The equivalences we want to find.
Returns:
A dictionary of the format {e: [histogram for this element (a day) as a pandas Series, number of equivalent
days]} """
histograms = defaultdict()
for e in equivalence_classes:
data["weight"] = np.array(data.index.map(lambda d: cond_weight(d, e))) # since it should also affect the column of the weights, so I add it to the dataframe.
nonzero_weights = data[data["weight"]>0] # we only care about the days where the weights are nonzero.
weighted_occupancy = nonzero_weights["Occupied"] * nonzero_weights["weight"] # to scale by the importance of each day.
g_obj = weighted_occupancy.groupby(by=lambda x: x.time()) # we want to group all same times for same equivalance days.
sum_obj = g_obj.sum()
histograms[e] = [sum_obj, int(g_obj.count()[0])] # Since every entry has the same number of days, we can just get the first.
return histograms
# Improve that it could take more than one column.
def plot_histogram(self, his, e_mapping, e, plot_with_envelope=False):
"""Plots the histograms for the data (assumes only one column).
Parameters:
his: The histogram to plot. Should be a pandas Series.
e_mapping: The mapping from the numerical representation of a equivalence class to something more representative.
e.g. 4 would map to Friday.
e: the equivalence classes to consider.
plot_with_envelope: Plots the given envelope we have. Fromats it such that it fits to the index of
the output of compute_histogram. Also, will be scaled by the max of each histogram to
make it more visible."""
if plot_with_envelope:
envelope = self.compute_envelope().groupby(lambda x: x.time()).sum()
# Plots a histogram for equivalence class.
for i in e:
plt.figure(figsize=(12,8))
plt.plot(his[i][0])
if plot_with_envelope:
plt.plot(envelope*max(his[i][0]))
plt.title("Histogram for: "+ e_mapping[i] +".")
plt.xlabel("Time of day.")
plt.ylabel("Number of days the region was occupied.")
plt.show()
# In[20]:
# IMPROVEMENT. Implemented it on a room basis for now. It will plot every day of the data given on one plot.
def plot_distribution_overlay(self, data, env, kind="room"):
"""Plots the occupancy distributions of rooms for several days on graphs. The distributions are represented by
lines. If the line is visible at a point, then the room was occupied. If there is a gap between lines, then in that
time frame the room was unoccupied. The x axis will be the hours of a day, and a graph will have several lines,
each representing a different room for the day given.
Parameters:
data: The data for which to find the distributions. Index should be time series, and columns should be boolean
arrays. Will plot for every column, and the number of lines will correspond to number of rooms/columns
in the data given.
env: We also plot the envelope for reference.
kind: The kind of data we are plotting. On a room basis for now. """
weekday_mapping = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
dates = data.index.normalize().unique().date # get the unique days in the data given.
for date in dates:
idx = list(data.index.map(lambda x: x.date() == date))
data_use = data[idx] # filter by the dates to be used
env_use = pd.DataFrame(data=env.values, columns=["Envelope"], index=data_use.index)
def f(x, height, c):
"""Finds wether or not a room was occupied at the time. Does not return a value
but modiefies the attributes of the function. Improvement: Don't use functions as classes."""
t = x["time"]
curr_occ = x[c] == 1
if curr_occ and f.occupied:
pass
elif curr_occ and not f.occupied:
f.temp = [t, "23:59:00", height] # setting the temporary end to a minute before midnight, so we stay in the same day.
f.occupied = True
elif not curr_occ and f.occupied:
f.temp[1] = t # update where it ends
f.lines.append(f.temp)
f.occupied = False
col = data_use.columns
h = 0 # a counter variable for the height/y_coordinate on which to plot a room.
f.lines = [] # the lines which f will be setting once we get the right line.
f.temp = None # the intermediate lines we are constructing. Will be added to f.lines once the room is not occupied for a period of time.
f.occupied = False # Is the room currently occupied. Helps f.temp to decide when to add a line to f.lines.
for c in col:
data_use[c].reset_index().apply(lambda x: f(x, h, c), axis=1)
f.temp = None # have to reset our class variables for a new room
f.occupied = False # have to reset our class variables for a new room
h += 1
points = []
fig, ax = plt.subplots(figsize=(12,8))
# need to append lines as points in a specific way. We add the starting and end point of each line.
for l in f.lines:
points.append([(l[0].time(), l[2]), (l[1].time(), l[2])])
# Here we pass it to the figure as (x1, x2) and (y1, y2)
for p in points:
(x, y) = zip(*p) # *p to zip elements in p, instead of expanding it
ax.add_line(plt_line.Line2D(x, y))
# Compute lines and points for envelope.
f.lines = []
f.temp = None
f.occupied = False
env_use.reset_index().apply(lambda x: f(x, h+1, "Envelope"), axis=1)
points = []
for l in f.lines:
points.append([(l[0].time(), l[2]), (l[1].time(), l[2])])
for p in points:
(x, y) = zip(*p)
ax.add_line(plt_line.Line2D(x, y, color='red'))
# Set up the plot.
plt.title("Distribution for day: " + str(date)+ ", " + str(weekday_mapping[date.weekday()]))
plt.xlabel("Time of day.")
ax.get_xaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: round(x/(4*60*15), 1)))
ax.set_xlim(0,4*60*15*24)
plt.plot()
plt.show
# IMPROVE: on a room basis we can make this a helper function and jjust loop for each loop.
# Maybe just return confusion matrix. Should be looked into.
def plot_confusion_matrix(self, data, env, time_step="15T", sensor=None):
"""Computes the confusion matrix for each day of a given week.
Parameters:
data: Should be a week of data.
envelope: The prediction for each day.
time_step: The time step to set all data to."""
def assign_confusion(x):
"""If predicted and actualy YES then return 3.
If predicted Yes and actual False return 2.
If predicted False and actual True return 1
If predicted False and actual False return 0"""
if x["Envelope"] == 0:
if x["Occupied"] == 0:
return 0
else:
return 1
if x["Occupied"] !=0:
return 3
return 2
dates = data.index.normalize().unique().date
confusion_matricies=defaultdict()
# For each day, get the value from assign confusion to determine which confusion class it belongs to.
for date in dates:
idx = list(data.index.map(lambda x: x.date() == date))
data_use = data[idx]
env_use = pd.DataFrame(data=env.values, columns=["Envelope"], index=data_use.index)
temp = pd.concat([data_use, env_use], axis=1)
temp["Confusion"] = temp.apply(assign_confusion, axis=1)
confusion_matricies[date] = temp["Confusion"].value_counts()
# display the confusion matrix for each. Needed to Hard code which number belongs to which confusion class.
for day, confusion in confusion_matricies.items():
zero = confusion[0] if 0 in confusion.keys() else 0
one = confusion[1] if 1 in confusion.keys() else 0
two = confusion[2] if 2 in confusion.keys() else 0
three = confusion[3] if 3 in confusion.keys() else 0
data = [[three, one], [two, zero]]
row = ["Actual True", "Actual False"]
column = ["Predicted True", "Predicted False"]
fig = plt.figure(figsize=(3,1))
ax = fig.add_subplot(111)
ax.axis('off')
the_table = ax.table(cellText=data, rowLabels=row, colLabels=column, loc='upper center')
if sensor is not None:
plt.title("Confusion Matrix for day: " + str(day) + "and sensor: "+ sensor)
else:
plt.title("Confusion Matrix for day: " + str(day))
plt.show()
# # Adaptive Schedule
# Implement a scheduler which takes in an unlimated number of constraints which are all or'ed to get valid dates by which to find a schedule. All the valid days will be taken and put into a histogram. The Schedule will result of an On state if more than half of the days had occupancy at the time and Off when less than half had occupancy. Should be done for each HVAC zone.
#
# # Improvement
# Right now, we might want to find a schedule for a Friday, but if we didn't record data for the whole week, the adaptive schedule will work with only the data before that but with the same weights and constraints. That means, if we set #of same classes = 4 #of same days=1 then we won't get a schedule, since we don't ahve the data to make it work.
def weekday_weekend(self, main_d, compare_d, num_c):
"""Checks wether main_d and compare_d are either a weekday or a weekend day. Also checks that
between the two, there aren't more than num_c days which are also weekdays/weekends (same class) depending on main_d.
We will use a linear model for now to determine how similar a day is. Meaning, that if compare_d
is num_c away from main_d, we will return 0.5.
Parameters:
main_d: The date I want to compare everything with.
compare_d: The day for which I want to see if it is in the same class as main_d.
num_c: The max number of same class days between main_d and compare_d.
Returns:
An float. Representing how similar the day is to the given day. For now we will use a linear model."""
num_c = float(num_c)
temp_diff = (abs((main_d.date() - compare_d.date()).days) - main_d.weekday()) # Sets up how many days away compare_d is from the beginning of the week of main_d.
num_weeks = temp_diff // 7 # number of weeks from beginning of the week
if main_d.weekday() <= 4 and compare_d.weekday() <=4: # for weekdays
remain = (temp_diff - 2) % 7 # adjusts it so i don't have to worry about weekends in the remainder
if num_weeks < 0: # if negative, then I know that my compare_d is not more than a week away. So all days between are valid days.
diff = num_c - abs((main_d.date() - compare_d.date()).days)
return diff/num_c if diff > 0 else 0
diff = num_c - (main_d.weekday() + num_weeks * 5 + remain) # check if the number of valid days in the weeks between,
#minus the days i subtracted for temp_diff, and the remainder days are not more than num_class days.
return diff/num_c if diff > 0 else 0
elif main_d.weekday() > 4 and compare_d.weekday() > 4: # for weekends
remain = 2 if (temp_diff%7) > 2 and num_weeks >= 0 else (temp_diff%7)
if num_weeks < 0:
diff = num_c - (abs((main_d.date() - compare_d.date()).days))
return diff/num_c if diff > 0 else 0
diff = num_c - ((main_d.weekday() - 5) + num_weeks * 2 + remain)
return diff/num_c if diff > 0 else 0
return 0
def cond_adaptive(self, class_weight, num_same_class, num_same_day):
"""A condition function. Its purpose is to return a function which will be used as
a equivalence condition for a histogram. Assumes that any day has the format of a pandas Timestamp.
To give weights, we will use a linear model, given what type of equivalence we achieve (day or class), we
will give it a percentage weight of how far away it is from the possible distance (num_same_class or num_same_day)
Parameters:
class_weight: Function(main_day, compare_day, num_same_class) which decides if the two dates are in the same class (e.g. workdays or weekend days)
num_same_class: The number of days that should be in the same class as the date. For current purposes
the same class means that they are weekdays/weekends.
num_same_day: The number of days which are the same weekdays as my given date.
Retruns:
Function which has parameters (date_check, equivalence class). The equivalence class parameter will be set
with the date for the adaptive
schedule. date_check is the date we pass in to see if it satisifies the equivalence relation."""
def final_cond(d, e):
"""Returns a float weighting for the day we compare to the equivalence which is our main day. According
to what we expect from the cond_adaptive function."""
day_same = 4*int(d.weekday() == e.weekday())*(abs((d.date() - e.date()).days) / float(7*num_same_day))
class_same = (class_weight(e, d, num_same_class))
# want to give it the largest weight we get.
# IS THIS DESIRABLE? SHOULD WE JUST RETURN THE SUM??
if day_same > class_same:
return day_same * int(d.date() != e.date())
else:
return class_same * int(d.date() != e.date())
return final_cond
# IMPROVEMENT: Give it sort of learning rate. The further back the days are, the less they count.
def adaptive_schedule(self, data, day, num_classes, num_same_days):
"""Will return an adaptive schedule for the given data, looking at the past data given. Produces an adaptive
schedule depending on which days should be considered. For this purpose we will use same weekday and same classes
(weekday or weekend day).
Parameters:
data: The data over which we find the schedule, should start the day before the given day. Is a pandas
Dataframe with only one column or Pandas Series.
day: A pd Timestamp day. We will generate the schedule for this day.
num_same_class: The number of days that should be in the same class as the date. For current purposes
the same class means that they are weekdays/weekends.
num_same_date: The number of days which are the same weekdays as my given date.
Returns:
A Pandas Series with True for having the schedule on and False for having it off."""
his = self.compute_histogram(data=data, cond_weight=self.cond_adaptive(self.weekday_weekend, 10, 10), equivalence_classes=[day])
temp = his[day]
schedule = 1*((temp[0] / float(max(temp[0]))) >= 0.50) # TODO need to reconsider what we are doing here. What should be the cutoff with variable weights?
return schedule
def plot_adaptive_schedule(self):
# building = get_building_occupany()
building = self.zone_df["EastZone"] # if there is one person then we want the person to be comfortable.
building = pd.DataFrame(data=building, columns=["Occupied"]) # this is awful. fix it
day = building.index[-400]
# UNCOMMENT TO SEE ORIGINAL HISTOGRAM
his = self.compute_histogram(data=building, cond_weight=self.cond_adaptive(self.weekday_weekend, 10, 10), equivalence_classes=[day])
self.plot_histogram(his=his, e_mapping={day:str(day)}, e=[day], plot_with_envelope=True)
schedule = self.adaptive_schedule(building, day, 10, 10)
self.plot_histogram(his={day: (schedule, 1)}, e_mapping={day:str(day)}, e=[day], plot_with_envelope=True)
class Controller:
def __init__(self):
self.bw_client = Client()
self.bw_client.setEntityFromEnviron()
self.bw_client.overrideAutoChainTo(True)
self.hod_client = HodClient("xbos/hod", self.bw_client)
self.priority = {
"fan": 1,
"kettle": 2,
"student_office_tstat": 3,
"microwave": 4,
"sra_office_tstat": 5,
"space_heater": 5,
"fridge": 6,
"evse": 7,
"michaels_office_tstat": 8
}
meter_q = """
SELECT ?uri FROM rfs WHERE {
?m rdf:type brick:Building_Electric_Meter .
?m bf:uri ?uri .
};
"""
self.building_meter = Meter(
self.bw_client,
self.hod_client.do_query(meter_q)['Rows'][0]['?uri'])
self.tstats = {
"student_office_tstat":
Thermostat(
self.bw_client,
"rfs/devices/s.pelican/Student_Office/i.xbos.thermostat"),
"sra_office_tstat":
Thermostat(self.bw_client,
"rfs/devices/s.pelican/SRA_Office/i.xbos.thermostat"),
"michaels_office_tstat":
Thermostat(
self.bw_client,
"rfs/devices/s.pelican/Michaels_Office/i.xbos.thermostat")
}
self.plugloads = {
"fan":
Plug(self.bw_client, "rfs/devices/fan/s.tplink.v0/0/i.xbos.plug"),
"fridge":
Plug(self.bw_client,
"rfs/devices/refrigerator/s.tplink.v0/0/i.xbos.plug"),
"space_heater":
Plug(self.bw_client,
"rfs/devices/heater/s.tplink.v0/0/i.xbos.plug"),
"kettle":
Plug(self.bw_client,
"rfs/devices/boiler/s.tplink.v0/0/i.xbos.plug"),
"microwave":
Plug(self.bw_client,
"rfs/devices/microwave/s.tplink.v0/0/i.xbos.plug")
}
self.evse = EVSE(self.bw_client,
"rfs/devices/s.aerovironment/ParkingLot/i.xbos.evse")
self.evse.set_current_limit(_MAX_EVSE_CURRENT)
def control(self, threshold):
if self.evse.state:
car_demand = self.evse.current * self.evse.voltage
allowed_car_threshold = threshold - self.building_meter.power * 1000
if car_demand < allowed_car_threshold:
if self.evse.current_limit < _MAX_EVSE_CURRENT:
print(
"current total consumption is less than threshold, increasing car charging limit"
)
if allowed_car_threshold < _MAX_EVSE_CONSUMPTION:
print("setting limit to:",
(allowed_car_threshold) // _MAX_EVSE_VOLTAGE)
self.evse.set_current_limit(
(allowed_car_threshold) // _MAX_EVSE_VOLTAGE)
else:
print("setting limit to:", _MAX_EVSE_CURRENT)
self.evse.set_current_limit(_MAX_EVSE_CURRENT)
else:
print(
"current total consumption is less than threshold, evse limit is already at max nothing to do"
)
return
else:
if self.building_meter.power <= threshold:
print(
"current building consumption is less than threshold and evse is off, nothing to do"
)
return
# current consumption is higher than threshold
# get total power consumption of controllable loads without evse (plugloads and baseboard heaters)
controllable_loads = 0
for _, tstat in self.tstats.iteritems():
controllable_loads += (tstat.state * _BASEBOARD_HEATER_CONSUMPTION)
for _, plug in self.plugloads.iteritems():
controllable_loads += plug.power
print("Building electric meter:", self.building_meter.power * 1000)
print("Total controllable_loads power:", controllable_loads)
process_load = self.building_meter.power * 1000 - controllable_loads
print("Total process (uncontrollable) loads power:", process_load)
# if process_load is greater than threshold
if process_load >= threshold:
if controllable_loads == 0:
print(
"current consumption is greater than threshold, but all controllable_loads are off, nothing to do"
)
return
else:
print(
"current consumption is greater than threshold, turning off all controllable loads. nothing else to do"
)
for _, tstat in self.tstats.iteritems():
tstat.set_mode(0)
for _, plug in self.plugloads.iteritems():
plug.set_state(0.0)
self.evse.set_state(False)
return
# subtract uncontrollable loads from threshold
ctrl_threshold = threshold - process_load
print("Controllable threshold:", ctrl_threshold)
# get and sort controllable_loads
controllable_loads = [{
'name': 'fan',
'priority': self.priority.get('fan'),
'capacity': self.plugloads.get('fan').power,
'state': int(self.plugloads.get('fan').state)
}, {
'name': 'kettle',
'priority': self.priority.get('kettle'),
'capacity': self.plugloads.get('kettle').power,
'state': int(self.plugloads.get('kettle').state)
}, {
'name':
'student_office_tstat',
'priority':
self.priority.get('student_office_tstat'),
'capacity':
_BASEBOARD_HEATER_CONSUMPTION,
'state':
self.tstats.get('student_office_tstat').state
}, {
'name':
'microwave',
'priority':
self.priority.get('microwave'),
'capacity':
self.plugloads.get('microwave').power,
'state':
int(self.plugloads.get('microwave').state)
}, {
'name':
'sra_office_tstat',
'priority':
self.priority.get('sra_office_tstat'),
'capacity':
_BASEBOARD_HEATER_CONSUMPTION,
'state':
self.tstats.get('sra_office_tstat').state
}, {
'name':
'space_heater',
'priority':
self.priority.get('space_heater'),
'capacity':
self.plugloads.get('space_heater').power,
'state':
int(self.plugloads.get('space_heater').state)
}, {
'name':
'michaels_office_tstat',
'priority':
self.priority.get('michaels_office_tstat'),
'capacity':
_BASEBOARD_HEATER_CONSUMPTION,
'state':
self.tstats.get('michaels_office_tstat').state
}, {
'name': 'fridge',
'priority': self.priority.get('fridge'),
'capacity': self.plugloads.get('fridge').power,
'state': int(self.plugloads.get('fridge').state)
}, {
'name': 'evse',
'priority': self.priority.get('evse'),
'capacity': _MIN_EVSE_CONSUMPTION,
'state': int(self.evse.state)
}]
controllable_loads = sorted(controllable_loads,
key=operator.itemgetter(
'priority', 'capacity'),
reverse=True)
loads_to_keep = []
# get loads to keep on
for load in controllable_loads:
if load['state'] == 1 and load['capacity'] <= ctrl_threshold:
print load
loads_to_keep.append(load['name'])
ctrl_threshold -= load['capacity']
print("Updated controllable threshold:", ctrl_threshold)
# turn off other loads
for load in controllable_loads:
if load['state'] == 1 and load['name'] not in loads_to_keep:
if load['name'] == 'evse':
print "turning off evse"
self.evse.set_state(False)
elif load['name'].endswith('_tstat'):
print "turning off tstat: " + load['name']
self.tstats.get(load['name']).set_mode(0)
else:
print "turning off plug: " + load['name']
self.plugloads.get(load['name']).set_state(0.0)
# set current_limit for evse to match threshold
if ctrl_threshold > 0 and 'evse' in loads_to_keep:
if _MIN_EVSE_CONSUMPTION + ctrl_threshold < _MAX_EVSE_CONSUMPTION:
print("setting limit to:",
(_MIN_EVSE_CONSUMPTION + ctrl_threshold) //
_MAX_EVSE_VOLTAGE)
self.evse.set_current_limit(
(_MIN_EVSE_CONSUMPTION + ctrl_threshold) //
_MAX_EVSE_VOLTAGE)
else:
print("setting limit to:", _MAX_EVSE_CURRENT)
self.evse.set_current_limit(_MAX_EVSE_CURRENT)
def report_state(self):
print("EVSE state: ", self.evse.state)
print("EVSE current: ", self.evse.current)
print("EVSE voltage: ", self.evse.voltage)
print("Building_Electric_Meter: ", self.building_meter.power * 1000)
for _, tstat in self.tstats.iteritems():
print("Tstat ", tstat._uri, " State: ", tstat.state)
print("Tstat ", tstat._uri, " Power: ",
tstat.state * _BASEBOARD_HEATER_CONSUMPTION)
for _, plug in self.plugloads.iteritems():
print("Plugload ", plug._uri, " State: ", plug.state)
print("Plugload ", plug._uri, " Power: ", plug.power)
from threading import Thread
import msgpack
from xbos import get_client
from xbos.services.hod import HodClient
from xbos.devices.thermostat import Thermostat
from xbos.devices.light import Light
with open("params.json") as f:
try:
params = json.loads(f.read())
except ValueError:
print "Invalid parameter file"
sys.exit(1)
c = get_client()
hod = HodClient(params["HOD_URI"], c)
lighting_query = """SELECT ?equipment ?uri WHERE {
?equipment rdf:type/rdfs:subClassOf* brick:Lighting_System .
?equipment bf:uri ?uri .
}"""
tstat_query = """SELECT ?equipment ?uri WHERE {
?equipment rdf:type/rdfs:subClassOf* brick:Thermostat .
?equipment bf:uri ?uri .
}"""
results = hod.do_query(lighting_query)
lights = []
if results["Count"] > 0:
for row in results["Rows"]:
class Occupancy():
def __init__(self, start=None, end=None, timezone='US/Pacific'):
""" params:
start: (string, "%Y-%m-%d %H:%M:%S %Z") When should the data start.
end: (string, "%Y-%m-%d %H:%M:%S %Z") when should the data end.
timezone: (string) As used by pytz. Used for both start and end
"""
# data clients
self.client = mdal.MDALClient("xbos/mdal")
self.hod = HodClient("xbos/hod")
self.SITE = "ciee"
tz = pytz.timezone('US/Pacific')
temp_end = dati.datetime.today() if end is None else datetime.strptime(
end, "%Y-%m-%d %H:%M:%S %Z")
temp_start = (
temp_end -
dati.timedelta(10)) if start is None else datetime.strptime(
start, "%Y-%m-%d %H:%M:%S %Z")
self.end = tz.localize(temp_end)
self.start = tz.localize(temp_start)
self.zone_sensor_df = self.get_occupancy()
self.building_df = self.get_building_occupancy()
self.zone_df = self.get_zone_occupancy()
def get_occupancy(self, start=None, end=None, time_steps="15m"):
"""Get occupancy for dates specified.
param:
start: (datetime) When should the data start.
end: (datetime) when should the data end.
time_steps: in what intervals (start, start+time+steps) do we want to look at data.
Returns:
(pandas DF) Index: Timerseries, Column: Boolean. If sensor located someone, we will set the interval to be occupied and True.
"""
start = self.start.strftime(
"%Y-%m-%d %H:%M:%S %Z") if start is None else start.strftime(
"%Y-%m-%d %H:%M:%S %Z")
end = self.end.strftime(
"%Y-%m-%d %H:%M:%S %Z") if end is None else end.strftime(
"%Y-%m-%d %H:%M:%S %Z")
# Brick queries
zone_query = """SELECT ?zone FROM %s WHERE {
?zone rdf:type brick:HVAC_Zone .
};"""
# define a Brick query to get the occupancy information
occupancy_query = """SELECT ?sensor_uuid FROM %s WHERE {
?sensor bf:isPointOf/bf:isPartOf <%s> .
?sensor bf:isLocatedIn ?room .
?sensor rdf:type/rdfs:subClassOf* brick:Occupancy_Sensor .
?sensor bf:uuid ?sensor_uuid .
};
"""
zones = [
x['?zone']['Namespace'] + '#' + x['?zone']['Value']
for x in self.hod.do_query(zone_query % self.SITE,
values_only=False)['Rows']
]
ret = {}
for zone in zones:
tstat_data_query = {
"Composition": ["occupancy_sensor"],
"Selectors": [MEAN, MAX, MEAN],
"Variables": [
{
"Name": "occupancy_sensor",
"Definition": occupancy_query % (self.SITE, zone),
"Units": "C",
},
],
"Time": {
"T0": start,
"T1": end,
"WindowSize": time_steps,
"Aligned": True,
}
}
resp = self.client.do_query(tstat_data_query, timeout=120)
if resp.get('error'):
continue
print("zone: ", zone)
print(resp)
df = resp['df']
df = df.dropna()
ret[zone] = df
return ret
def compute_envelope(self,
start="08:00:00",
end="18:00:00",
time_steps="15T"):
"""Computes an envelope for a day. Will be filled with 1's from start to end,
and the time series will have a period of time_steps.
Parameters:
start: When the envelope should start. Provide pandas time string. should be less than end.
end: When the envelope should end. Provdie pandas time string. should be greater than start.
time_steps: the period of the indexing. Taken to be that if at time 08:15:00 we have a one, then
in the period between 08:15:00 and 08:30:00 someone was by the occupancy sensor."""
t = pd.DatetimeIndex(freq='15T',
start='2017-01-01 00:00:00',
end='2017-01-01 23:59:59')
envelope = pd.Series(data=0, index=t)
idx_t = envelope.index
selector = (t.hour >= 8) & (t.hour < 18)
envelope.loc[selector] = 1
envelope.index = idx_t.time # is this necessary? I would like to have it independent of day though.
return envelope
def hamming_distance(self, arr, compare_to):
"""Returns hamming distance of two numpy arrays.
>>> a = np.array([1,0,1])
>>> b = np.array([0,1,0])
>>> hamming_distance(a, b)
3
>>> c = np.array([1])
>>> hamming_distance(a, c)
1
The last output is a result from appending an array of zeros to c to make a and c equal length."""
return np.count_nonzero(arr != compare_to)
def get_zone_occupancy(self, cond=(lambda x: True)):
"""Returns the logical or of all sensors in a zone.
Parameters:
cond: A function on a timestamp by which to filter the data.
return:
A dictionary of key=zone and value=pd.df(occupancy)"""
result = defaultdict()
for zone, df in self.zone_sensor_df.items():
temp = df[df.index.map(cond)]
result[zone] = temp.apply(lambda x: max(x), axis=1)
return result
def get_building_occupancy(self, cond=(lambda x: True)):
"""Returns the logical or of all sensors.
Parameters:
cond: A function on a timestamp by which to filter the data.
Returns:
pd.DataFrame"""
result = None
# putting all sensors in one pd.df.
for key, value in self.zone_sensor_df.items():
if result is None:
result = value
else:
result = pd.concat([result, value], axis=1)
idx = list(result.index.map(cond))
result = result[idx] # index the rows to filter by times.
result = result.apply(
lambda x: max(x), axis=1
) # applying a function to occupancy value of all zones at a given time
return pd.DataFrame(data=result.values,
columns=["Occupied"],
index=result.index)
# Improvement: Right now only works for a data df with one column or pandas series
def compute_histogram(self, data, cond_weight, equivalence_classes):
"""Creates a weighted histogram of the data.
The histograms will be computed for all equivalent dates for the given equivalence.
For example, if we give the equivalence relation (lambda date, equivalent_day: date.dayofweek == equivalent_day)
and the list of equivalences we want to find [0,1,2,3,4,5,6],
then we will get a dictionary {e: histogram for equivalent days}. So, for Monday I would get
{0: histogram for all mondays}. Furthermore, every equivalence is given a weight which depends on the equivalence
class and the input day. So, we could weight days which are further in the past less.
Frequency of histrogram will be the same as the data given.
Parameters:
data: The data for which to compute the histogram.
cond_weight: The condition by which to establish equivalence or the equialence relation and returns the weight
of the specific equivalence and day. Has to have the input format
cond_weight(day, equivalence). (Returns: int: weight). A weight of 0 means no equivalence and a weight of one means
full equivalence.
equivalence_classes: The equivalences we want to find.
Returns:
A dictionary of the format {e: [histogram for this element (a day) as a pandas Series, number of equivalent
days]} """
histograms = defaultdict()
for e in equivalence_classes:
data["weight"] = np.array(
data.index.map(lambda d: cond_weight(d, e))
) # since it should also affect the column of the weights, so I add it to the dataframe.
nonzero_weights = data[
data["weight"] >
0] # we only care about the days where the weights are nonzero.
weighted_occupancy = nonzero_weights["Occupied"] * nonzero_weights[
"weight"] # to scale by the importance of each day.
g_obj = weighted_occupancy.groupby(by=lambda x: x.time(
)) # we want to group all same times for same equivalance days.
sum_obj = g_obj.sum()
histograms[e] = [
sum_obj, int(g_obj.count()[0])
] # Since every entry has the same number of days, we can just get the first.
return histograms
# Improve that it could take more than one column.
def plot_histogram(self, his, e_mapping, e, plot_with_envelope=False):
"""Plots the histograms for the data (assumes only one column).
Parameters:
his: The histogram to plot. Should be a pandas Series.
e_mapping: The mapping from the numerical representation of a equivalence class to something more representative.
e.g. 4 would map to Friday.
e: the equivalence classes to consider.
plot_with_envelope: Plots the given envelope we have. Fromats it such that it fits to the index of
the output of compute_histogram. Also, will be scaled by the max of each histogram to
make it more visible."""
if plot_with_envelope:
envelope = self.compute_envelope().groupby(
lambda x: x.time()).sum()
# Plots a histogram for equivalence class.
for i in e:
plt.figure(figsize=(12, 8))
plt.plot(his[i][0])
if plot_with_envelope:
plt.plot(envelope * max(his[i][0]))
plt.title("Histogram for: " + e_mapping[i] + ".")
plt.xlabel("Time of day.")
plt.ylabel("Number of days the region was occupied.")
plt.show()
# In[20]:
# IMPROVEMENT. Implemented it on a room basis for now. It will plot every day of the data given on one plot.
def plot_distribution_overlay(self, data, env, kind="room"):
"""Plots the occupancy distributions of rooms for several days on graphs. The distributions are represented by
lines. If the line is visible at a point, then the room was occupied. If there is a gap between lines, then in that
time frame the room was unoccupied. The x axis will be the hours of a day, and a graph will have several lines,
each representing a different room for the day given.
Parameters:
data: The data for which to find the distributions. Index should be time series, and columns should be boolean
arrays. Will plot for every column, and the number of lines will correspond to number of rooms/columns
in the data given.
env: We also plot the envelope for reference.
kind: The kind of data we are plotting. On a room basis for now. """
weekday_mapping = [
"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday",
"Sunday"
]
dates = data.index.normalize().unique(
).date # get the unique days in the data given.
for date in dates:
idx = list(data.index.map(lambda x: x.date() == date))
data_use = data[idx] # filter by the dates to be used
env_use = pd.DataFrame(data=env.values,
columns=["Envelope"],
index=data_use.index)
def f(x, height, c):
"""Finds wether or not a room was occupied at the time. Does not return a value
but modiefies the attributes of the function. Improvement: Don't use functions as classes."""
t = x["time"]
curr_occ = x[c] == 1
if curr_occ and f.occupied:
pass
elif curr_occ and not f.occupied:
f.temp = [
t, "23:59:00", height
] # setting the temporary end to a minute before midnight, so we stay in the same day.
f.occupied = True
elif not curr_occ and f.occupied:
f.temp[1] = t # update where it ends
f.lines.append(f.temp)
f.occupied = False
col = data_use.columns
h = 0 # a counter variable for the height/y_coordinate on which to plot a room.
f.lines = [
] # the lines which f will be setting once we get the right line.
f.temp = None # the intermediate lines we are constructing. Will be added to f.lines once the room is not occupied for a period of time.
f.occupied = False # Is the room currently occupied. Helps f.temp to decide when to add a line to f.lines.
for c in col:
data_use[c].reset_index().apply(lambda x: f(x, h, c), axis=1)
f.temp = None # have to reset our class variables for a new room
f.occupied = False # have to reset our class variables for a new room
h += 1
points = []
fig, ax = plt.subplots(figsize=(12, 8))
# need to append lines as points in a specific way. We add the starting and end point of each line.
for l in f.lines:
points.append([(l[0].time(), l[2]), (l[1].time(), l[2])])
# Here we pass it to the figure as (x1, x2) and (y1, y2)
for p in points:
(x, y) = zip(
*p) # *p to zip elements in p, instead of expanding it
ax.add_line(plt_line.Line2D(x, y))
# Compute lines and points for envelope.
f.lines = []
f.temp = None
f.occupied = False
env_use.reset_index().apply(lambda x: f(x, h + 1, "Envelope"),
axis=1)
points = []
for l in f.lines:
points.append([(l[0].time(), l[2]), (l[1].time(), l[2])])
for p in points:
(x, y) = zip(*p)
ax.add_line(plt_line.Line2D(x, y, color='red'))
# Set up the plot.
plt.title("Distribution for day: " + str(date) + ", " +
str(weekday_mapping[date.weekday()]))
plt.xlabel("Time of day.")
ax.get_xaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(
lambda x, p: round(x / (4 * 60 * 15), 1)))
ax.set_xlim(0, 4 * 60 * 15 * 24)
plt.plot()
plt.show
# IMPROVE: on a room basis we can make this a helper function and jjust loop for each loop.
# Maybe just return confusion matrix. Should be looked into.
def plot_confusion_matrix(self, data, env, time_step="15T", sensor=None):
"""Computes the confusion matrix for each day of a given week.
Parameters:
data: Should be a week of data.
envelope: The prediction for each day.
time_step: The time step to set all data to."""
def assign_confusion(x):
"""If predicted and actualy YES then return 3.
If predicted Yes and actual False return 2.
If predicted False and actual True return 1
If predicted False and actual False return 0"""
if x["Envelope"] == 0:
if x["Occupied"] == 0:
return 0
else:
return 1
if x["Occupied"] != 0:
return 3
return 2
dates = data.index.normalize().unique().date
confusion_matricies = defaultdict()
# For each day, get the value from assign confusion to determine which confusion class it belongs to.
for date in dates:
idx = list(data.index.map(lambda x: x.date() == date))
data_use = data[idx]
env_use = pd.DataFrame(data=env.values,
columns=["Envelope"],
index=data_use.index)
temp = pd.concat([data_use, env_use], axis=1)
temp["Confusion"] = temp.apply(assign_confusion, axis=1)
confusion_matricies[date] = temp["Confusion"].value_counts()
# display the confusion matrix for each. Needed to Hard code which number belongs to which confusion class.
for day, confusion in confusion_matricies.items():
zero = confusion[0] if 0 in confusion.keys() else 0
one = confusion[1] if 1 in confusion.keys() else 0
two = confusion[2] if 2 in confusion.keys() else 0
three = confusion[3] if 3 in confusion.keys() else 0
data = [[three, one], [two, zero]]
row = ["Actual True", "Actual False"]
column = ["Predicted True", "Predicted False"]
fig = plt.figure(figsize=(3, 1))
ax = fig.add_subplot(111)
ax.axis('off')
the_table = ax.table(cellText=data,
rowLabels=row,
colLabels=column,
loc='upper center')
if sensor is not None:
plt.title("Confusion Matrix for day: " + str(day) +
"and sensor: " + sensor)
else:
plt.title("Confusion Matrix for day: " + str(day))
plt.show()
# # Adaptive Schedule
# Implement a scheduler which takes in an unlimated number of constraints which are all or'ed to get valid dates by which to find a schedule. All the valid days will be taken and put into a histogram. The Schedule will result of an On state if more than half of the days had occupancy at the time and Off when less than half had occupancy. Should be done for each HVAC zone.
#
# # Improvement
# Right now, we might want to find a schedule for a Friday, but if we didn't record data for the whole week, the adaptive schedule will work with only the data before that but with the same weights and constraints. That means, if we set #of same classes = 4 #of same days=1 then we won't get a schedule, since we don't ahve the data to make it work.
def weekday_weekend(self, main_d, compare_d, num_c):
"""Checks wether main_d and compare_d are either a weekday or a weekend day. Also checks that
between the two, there aren't more than num_c days which are also weekdays/weekends (same class) depending on main_d.
We will use a linear model for now to determine how similar a day is. Meaning, that if compare_d
is num_c away from main_d, we will return 0.5.
Parameters:
main_d: (datetime) The date I want to compare everything with.
compare_d: (datetime) The day for which I want to see if it is in the same class as main_d.
num_c: (int) The max number of same class days between main_d and compare_d.
Returns:
(float) Representing how similar the day is to the given day. For now we will use a linear model (i.e. (#same class days between
main_d and compare_d)/num_c). #same class days betweeen main_d and compare_d is inclusive of compare_d but exclusive of main_d"""
dt = main_d.date() - compare_d.date() # number days between them
wkd_dt = main_d.weekday() - compare_d.weekday(
) # the weekday difference
num_weeks = (dt.days - wkd_dt) / float(
7
) # implicitly finding the same weekday in the compare_d week. Num #num_days is a multiple of 7.
if main_d.weekday() <= 4 and compare_d.weekday(
) <= 4 and dt.days > 0 and (5 * num_weeks +
wkd_dt) < num_c: # for weekdays
return 1 - (5 * num_weeks + wkd_dt) / float(
num_c
) # 5 * num_weeks between the weeks they are in + the weekday difference
elif main_d.weekday() > 4 and compare_d.weekday(
) > 4 and dt.days > 0 and (2 * num_weeks +
wkd_dt) < num_c: # for weekends
return 1 - (2 * num_weeks + wkd_dt) / float(
num_c
) # 2 * num_weeks between the weeks they are in + the weekday difference
else: # if both day are not the same class or if compare_d is not in the past of main_d or more than num_c class days.
return 0
def cond_adaptive(self, class_weight, num_same_class, num_same_day):
"""A condition function. Its purpose is to return a function which will be used as
a equivalence condition for a histogram. Assumes that any day has the format of a pandas Timestamp.
To give weights, we will use a linear model, given what type of equivalence we achieve (day or class), we
will give it a percentage weight of how far away it is from the possible distance (num_same_class or num_same_day)
Parameters:
class_weight: Function(main_day, compare_day, num_same_class) which decides if the two dates are in the same class (e.g. workdays or weekend days)
num_same_class: The number of days that should be in the same class as the date. For current purposes
the same class means that they are weekdays/weekends.
num_same_day: The number of days which are the same weekdays as my given date.
Retruns:
Function which has parameters (date_check, equivalence class). The equivalence class parameter will be set
with the date for the adaptive
schedule. date_check is the date we pass in to see if it satisifies the equivalence relation."""
def final_cond(d, e):
"""Returns a float weighting for the day we compare to the equivalence which is our main day. According
to what we expect from the cond_adaptive function."""
dt = (e.date() - d.date()).days
if d.weekday() == e.weekday() and dt > 0:
day_same = (1 - dt / float(7 * num_same_day))
else:
day_same = 0
class_same = (class_weight(e, d, num_same_class))
# want to give it the largest weight we get.
# IS THIS DESIRABLE? SHOULD WE JUST RETURN THE average??
if day_same > class_same:
return day_same * int(d.date() != e.date())
else:
return class_same * int(d.date() != e.date())
return final_cond
# IMPROVEMENT: Give it sort of learning rate. The further back the days are, the less they count.
def adaptive_schedule(self, zone, day, num_classes, num_same_days, cutoff):
"""Will return an adaptive schedule for the given data, looking at the past data given. Produces an adaptive
schedule depending on which days should be considered. For this purpose we will use same weekday and same classes
(weekday or weekend day).
Parameters:
zone: (string) Zones from building for which i have the occupancy
day: (datetime) We will generate the schedule for this day.
num_same_class: The number of days that should be in the same class as the date. For current purposes
the same class means that they are weekdays/weekends.
num_same_date: The number of days which are the same weekdays as my given date.
Returns:
A Pandas Series with True for having the schedule on and False for having it off."""
if day.weekday() <= 4:
temp_days = max(
7 * (num_classes // 5 + 1), num_same_days * 7
) # taking the max of possible days to account for num_classes or num_same_days. num_classes//# + 1 used to roughly round up.
else:
temp_days = max(7 * (num_classes // 2 + 1), num_same_days * 7)
temp_start = day - dati.timedelta(temp_days)
# this is super ugly. Should hide this in getting the occupancy
if self.start > temp_start or (self.end < day
and self.end.date != dati.date.today()):
self.start = temp_start
if day > dati.datetime.today():
self.end = dati.datetime.today()
else:
self.end = day
self.zone_sensor_df = self.get_occupancy()
self.building = self.get_building_occupany()
self.zone_df = self.get_zone_occupancy()
if "Building" in zone:
data = self.building
else:
data = self.zone_df[zone]
his = self.compute_histogram(data=data,
cond_weight=self.cond_adaptive(
self.weekday_weekend, 10, 10),
equivalence_classes=[day])
temp = his[day]
schedule = 1 * (
(temp[0] / float(max(temp[0]))) >= cutoff
) # TODO need to reconsider what we are doing here. What should be the cutoff with variable weights?
return schedule
def plot_adaptive_schedule(self):
# building = get_building_occupany()
building = self.zone_df[
"EastZone"] # if there is one person then we want the person to be comfortable.
building = pd.DataFrame(data=building,
columns=["Occupied"]) # this is awful. fix it
day = building.index[-400]
# UNCOMMENT TO SEE ORIGINAL HISTOGRAM
his = self.compute_histogram(data=building,
cond_weight=self.cond_adaptive(
self.weekday_weekend, 10, 10),
equivalence_classes=[day])
self.plot_histogram(his=his,
e_mapping={day: str(day)},
e=[day],
plot_with_envelope=True)
schedule = self.adaptive_schedule(building, day, 10, 10)
self.plot_histogram(his={day: (schedule, 1)},
e_mapping={day: str(day)},
e=[day],
plot_with_envelope=True)
# TODO INTERVAL SHOULD NOT BE IN config_file.yml, THERE SHOULD BE A DIFFERENT INTERVAL FOR EACH ZONE
zone_thermal_models = {
zone: MPCThermalModel(zone,
zone_thermal_data,
interval_length=cfg["Interval_Length"],
thermal_precision=cfg["Thermal_Precision"])
for zone, zone_thermal_data in thermal_data.items()
}
print("Trained Thermal Model")
# --------------------------------------
with open(yaml_filename, 'r') as ymlfile:
cfg = yaml.load(ymlfile)
hc = HodClient("xbos/hod", client)
q = """SELECT ?uri ?zone FROM %s WHERE {
?tstat rdf:type/rdfs:subClassOf* brick:Thermostat .
?tstat bf:uri ?uri .
?tstat bf:controls/bf:feeds ?zone .
};""" % cfg["Building"]
# Start of FIX for missing Brick query
thermostat_query = """SELECT ?zone ?uri FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:uri ?uri.
import sys
import time
import schedule
with open("params.json") as f:
try:
params = json.loads(f.read())
except ValueError:
print "Invalid parameter file"
sys.exit(1)
# setup clients
client = get_client()
dataclient = DataClient(client, archivers=[params["ARCHIVER_URI"]])
hodclient = HodClient(params["HOD_URI"], client)
slack_token = params["SLACK_API_TOKEN"]
sc = SlackClient(slack_token)
def notify(msg):
sc.api_call("chat.postMessage",channel="#xbos_alarms",text=msg)
# get all thermostat states
query = """SELECT ?dev ?uri WHERE {
?dev rdf:type/rdfs:subClassOf* brick:Thermostat .
?dev bf:uri ?uri .
};"""
res = hodclient.do_query(query)
if res["Count"] == 0:
print "no thermostats"
else:
from xbos.services.hod import HodClient
# from matplotlib.pyplot import step, xlim, ylim, show
import matplotlib.pyplot as plt
import datetime, pytz
now = datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC")).astimezone(
tz=pytz.timezone("America/Los_Angeles"))
print now
start = now.strftime('%Y-%m-%d %H:%M:%S %Z')
end = '2017-09-21 00:00:00 PST'
WINDOW = '10min'
# data clients
mdal = MDALClient("xbos/mdal")
hod = HodClient("xbos/hod")
# temporal parameters
SITE = "ciee"
# Brick queries
building_meters_query = """SELECT ?meter ?meter_uuid FROM %s WHERE {
?meter rdf:type brick:Building_Electric_Meter .
?meter bf:uuid ?meter_uuid .
};"""
thermostat_state_query = """SELECT ?tstat ?status_uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat_Status .
?tstat bf:uuid ?status_uuid .
};"""
lighting_state_query = """SELECT ?lighting ?state_uuid FROM %s WHERE {
?light rdf:type brick:Lighting_State .
if __name__ == '__main__':
import yaml
with open("config_file.yml", 'r') as ymlfile:
cfg = yaml.load(ymlfile)
from xbos import get_client
from xbos.services.hod import HodClient
if cfg["Server"]:
client = get_client(agent=cfg["Agent_IP"], entity=cfg["Entity_File"])
else:
client = get_client()
hc = HodClient("xbos/hod", client)
q = """SELECT ?uri ?zone FROM %s WHERE {
?tstat rdf:type/rdfs:subClassOf* brick:Thermostat .
?tstat bf:uri ?uri .
?tstat bf:controls/bf:feeds ?zone .
};""" % cfg["Building"]
from xbos.devices.thermostat import Thermostat
for tstat in hc.do_query(q)['Rows']:
print tstat
with open(
"Buildings/" + cfg["Building"] + "/ZoneConfigs/" +
tstat["?zone"] + ".yml", 'r') as ymlfile:
advise_cfg = yaml.load(ymlfile)
class ControllerDataManager:
"""
# Class that handles all the data fetching and some of the preprocess for data that is relevant to controller
and which does not have to be fetched every 15 min but only once.
"""
def __init__(
self,
controller_cfg,
client,
now=datetime.datetime.utcnow().replace(tzinfo=pytz.timezone("UTC"))):
self.controller_cfg = controller_cfg
self.pytz_timezone = pytz.timezone(controller_cfg["Pytz_Timezone"])
self.interval = controller_cfg["Interval_Length"]
self.now = now.astimezone(self.pytz_timezone)
self.client = client
self.window_size = 1 # minutes. TODO should come from config. Has to be a multiple of 15 for weather getting.
self.building = controller_cfg["Building"]
self.hod_client = HodClient(
"xbos/hod", self.client
) # TODO hopefully i could incorporate this into the query.
def _get_outside_data(self, start=None, end=None):
# TODO update docstring
"""Get outside temperature for thermal model.
:param start: (datetime) time to start. relative to datamanager instance timezone.
:param end: (datetime) time to end. relative to datamanager instance timezone.
:return outside temperature has freq of 15 min and
pd.df columns["tin", "a"] has freq of self.window_size. """
if end is None:
end = self.now
if start is None:
start = end - timedelta(hours=10)
# Converting start and end from datamanger timezone to UTC timezone.
start = start.astimezone(pytz.timezone("UTC"))
end = end.astimezone(pytz.timezone("UTC"))
outside_temperature_query = """SELECT ?weather_station ?uuid FROM %s WHERE {
?weather_station rdf:type brick:Weather_Temperature_Sensor.
?weather_station bf:uuid ?uuid.
};"""
# get outside temperature data
# TODO UPDATE OUTSIDE TEMPERATURE STUFF
# TODO for now taking all weather stations and preprocessing it. Should be determined based on metadata.
outside_temperature_query_data = self.hod_client.do_query(
outside_temperature_query % self.building)["Rows"][0]
# Get data from MDAL
mdal_client = mdal.MDALClient("xbos/mdal", client=self.client)
outside_temperature_data = mdal_client.do_query({
'Composition': [outside_temperature_query_data["?uuid"]],
# uuid from Mr.Plotter. should use outside_temperature_query_data["?uuid"],
'Selectors': [mdal.MEAN],
'Time': {
'T0': start.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'T1': end.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize': str(15) + 'min',
# TODO document that we are getting 15 min intervals because then we get less nan values.
'Aligned': True
}
})
outside_temperature_data = outside_temperature_data["df"]
outside_temperature_data.columns = ["t_out"]
return outside_temperature_data
def _get_inside_data(self, start, end):
"""Get thermostat status and temperature and outside temperature for thermal model.
:param start: (datetime) time to start. relative to datamanager instance timezone.
:param end: (datetime) time to end. relative to datamanager instance timezone.
:return outside temperature has freq of 15 min and
pd.df columns["tin", "a"] has freq of self.window_size. """
# Converting start and end from datamanger timezone to UTC timezone.
start = start.astimezone(pytz.timezone("UTC"))
end = end.astimezone(pytz.timezone("UTC"))
# following queries are for the whole building.
thermostat_status_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:hasLocation/bf:isPartOf ?location_zone .
?location_zone rdf:type brick:HVAC_Zone .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?status_point .
?status_point rdf:type brick:Thermostat_Status .
?status_point bf:uuid ?uuid.
};"""
# Start of FIX for missing Brick query
thermostat_status_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?status_point .
?status_point rdf:type brick:Thermostat_Status .
?status_point bf:uuid ?uuid.
};"""
# End of FIX - delete when Brick is fixed
thermostat_temperature_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:hasLocation/bf:isPartOf ?location_zone .
?location_zone rdf:type brick:HVAC_Zone .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?thermostat_point .
?thermostat_point rdf:type brick:Temperature_Sensor .
?thermostat_point bf:uuid ?uuid.
};"""
# Start of FIX for missing Brick query
thermostat_temperature_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?thermostat_point .
?thermostat_point rdf:type brick:Temperature_Sensor .
?thermostat_point bf:uuid ?uuid.
};"""
# End of FIX - delete when Brick is fixed
temp_thermostat_query_data = {
"tstat_temperature":
self.hod_client.do_query(thermostat_temperature_query %
self.building)["Rows"],
"tstat_action":
self.hod_client.do_query(thermostat_status_query %
self.building)["Rows"],
}
# give the thermostat query data better structure for later loop. Can index by zone and then get uuids for each
# thermostat attribute.
thermostat_query_data = {}
for tstat_attr, attr_dicts in temp_thermostat_query_data.items():
for dict in attr_dicts:
if dict["?zone"] not in thermostat_query_data:
thermostat_query_data[dict["?zone"]] = {}
thermostat_query_data[
dict["?zone"]][tstat_attr] = dict["?uuid"]
# get the data for the thermostats for each zone.
mdal_client = mdal.MDALClient("xbos/mdal", client=self.client)
zone_thermal_data = {}
for zone, dict in thermostat_query_data.items():
# get the thermostat data
dfs = mdal_client.do_query({
'Composition':
[dict["tstat_temperature"], dict["tstat_action"]],
'Selectors': [mdal.MEAN, mdal.MAX],
'Time': {
'T0': start.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'T1': end.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize': str(self.window_size) + 'min',
'Aligned': True
}
})
df = pd.concat([dframe for uid, dframe in dfs.items()], axis=1)
zone_thermal_data[zone] = df.rename(columns={
dict["tstat_temperature"]: 't_in',
dict["tstat_action"]: 'a'
})
# TODO Note: The timezone for the data relies to be converted by MDAL to the local timezone.
return zone_thermal_data
def _preprocess_thermal_data(self, zone_data, outside_data):
"""Preprocesses the data for the thermal model.
:param zone_data: dict{zone: pd.df columns["tin", "a"]}
:param outside_data: pd.df columns["tout"].
NOTE: outside_data freq has to be a multiple of zone_data frequency and has to have a higher freq.
:returns {zone: pd.df columns: t_in', 't_next', 'dt','t_out', 'action', 'a1', 'a2', [other mean zone temperatures]}
where t_out and zone temperatures are the mean values over the intervals.
a1 is whether heating and a2 whether cooling."""
# thermal data preprocess starts here
# Heating
def f1(row):
"""
helper function to format the thermal model dataframe
"""
if row['action'] == 1.:
val = 1
else:
val = 0
return val
# if state is 2 we are doing cooling
def f2(row):
"""
helper function to format the thermal model dataframe
"""
if row['action'] == 2.:
val = 1
else:
val = 0
return val
def f3(row):
"""
helper function to format the thermal model dataframe
"""
if 0 < row['a'] <= 1:
return 1
elif 1 < row['a'] <= 2:
return 2
elif np.isnan(row['a']):
return row['a']
else:
return 0
all_temperatures = pd.concat(
[tstat_df["t_in"] for tstat_df in zone_data.values()], axis=1)
all_temperatures.columns = [
"zone_temperature_" + zone for zone in zone_data.keys()
]
zone_thermal_model_data = {}
for zone in zone_data.keys():
# Putting together outside and zone data.
actions = zone_data[zone]["a"]
thermal_model_data = pd.concat(
[all_temperatures, actions, outside_data],
axis=1) # should be copied data according to documentation
thermal_model_data = thermal_model_data.rename(
columns={"zone_temperature_" + zone: "t_in"})
thermal_model_data['a'] = thermal_model_data.apply(f3, axis=1)
# Assumption:
# Outside temperature will have nan values because it does not have same frequency as zone data.
# Hence, we fill with last known value to assume a constant temperature throughout intervals.
thermal_model_data["t_out"] = thermal_model_data["t_out"].fillna(
method="pad")
# prepares final data type.
thermal_model_data['change_of_action'] = (
thermal_model_data['a'].diff(1) != 0
).astype('int').cumsum(
) # given a row it's the number of times we have had an action change up till then. e.g. from nothing to heating.
# This is accomplished by taking the difference of two consecutive rows and checking if their difference is 0 meaning that they had the same action.
# following adds the fields "time", "dt" etc such that we accumulate all values where we have consecutively the same action.
# maximally we group terms of total self.interval
data_list = []
for j in thermal_model_data.change_of_action.unique():
for i in range(
0, thermal_model_data[
thermal_model_data['change_of_action'] ==
j].shape[0], self.interval):
for dfs in [
thermal_model_data[
thermal_model_data['change_of_action'] == j]
[i:i + self.interval]
]:
# we only look at intervals where the last and first value for T_in are not Nan.
dfs.dropna(subset=["t_in"])
zone_col_filter = [
"zone_temperature_" in col for col in dfs.columns
]
temp_data_dict = {
'time':
dfs.index[0],
't_in':
dfs['t_in'][0],
't_next':
dfs['t_in'][-1],
# need to add windowsize for last timestep.
'dt': (dfs.index[-1] - dfs.index[0]).seconds / 60 +
self.window_size,
't_out':
dfs['t_out'].mean(
), # mean does not count Nan values
'action':
dfs['a'][0]
}
for temperature_zone in dfs.columns[zone_col_filter]:
# mean does not count Nan values
temp_data_dict[temperature_zone] = dfs[
temperature_zone].mean()
data_list.append(temp_data_dict)
thermal_model_data = pd.DataFrame(data_list).set_index('time')
thermal_model_data['a1'] = thermal_model_data.apply(f1, axis=1)
thermal_model_data['a2'] = thermal_model_data.apply(f2, axis=1)
thermal_model_data = thermal_model_data.dropna(
) # final drop. Mostly if the whole interval for the zones or t_out were nan.
zone_thermal_model_data[zone] = thermal_model_data
print('one zone preproccessed')
return zone_thermal_model_data
def thermal_data(self, start=None, end=None, days_back=60):
"""
:param start: In timezone of datamanger
:param end: in timezone of datamanger
:param if start is None, then we set start to end - timedelta(days=days_back).
:return: pd.df {zone: pd.df columns: t_in', 't_next', 'dt','t_out', 'action', 'a1', 'a2', [other mean zone temperatures]}
where t_out and zone temperatures are the mean values over the intervals.
a1 is whether heating and a2 whether cooling.
"""
if end is None:
end = self.now
if start is None:
start = end - timedelta(days=days_back)
z, o = self._get_inside_data(start,
end), self._get_outside_data(start, end)
return self._preprocess_thermal_data(z, o)
#print data[data.index == now],prediction[0]
prediction[0] = data[data.index == now]
time_index = pd.date_range(now,
now + timedelta(minutes=prediction_time),
freq='15T')
return pd.DataFrame(data=prediction, index=time_index)
zones = ['SouthZone', 'EastZone', 'CentralZone', 'NorthZone']
temp_now = datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC")).astimezone(
tz=pytz.timezone("America/Los_Angeles"))
c = get_client()
archiver = DataClient(c)
hod = HodClient("ciee/hod", c)
occ_query = """SELECT ?sensor ?uuid ?zone WHERE {
?sensor rdf:type brick:Occupancy_Sensor .
?sensor bf:isLocatedIn/bf:isPartOf ?zone .
?sensor bf:uuid ?uuid .
?zone rdf:type brick:HVAC_Zone
};
"""
results = hod.do_query(occ_query)
uuids = [[x['?zone'], x['?uuid']] for x in results['Rows']]
per_zone_occ_list = []
for zone_name in zones:
# buildings = ["south-berkeley-senior-center",
# "north-berkeley-senior-center",
# "avenal-veterans-hall",
# "ciee", "orinda-community-center",
# "word-of-faith-cc",
# "jesse-turner-center",
# "orinda-community-center",
# "avenal-recreation-center",
# "avenal-animal-shelter", "avenal-movie-theatre", "avenal-public-works-yard",
# "avenal-recreation-center", "berkeley-corporate-yard"]
buildings = ["orinda-community-center"]
# Getting clients
client = get_client()
hc = HodClient("xbos/hod", client)
for BUILDING in buildings:
print("================================================")
print("")
print("Working on building", BUILDING)
print("")
query_data = hc.do_query(thermostat_query % BUILDING)["Rows"]
query_data = [
x for x in query_data if x["?zone"] != "HVAC_Zone_Please_Delete_Me"
] #TODO CHANGE THE PLEASE DELETE ME ZONE CHECK WHEN FIXED
try:
tstats = {
d["?zone"]: Thermostat(client, d["?uri"])
class ThermalDataManager:
"""
# Class that handles all the data fetching and some of the preprocess for data that is relevant to controller
and which does not have to be fetched every 15 min but only once.
Time is always in UTC
"""
def __init__(self, building_cfg, client, interval=5):
"""
:param building_cfg: A dictionary which should include data for keys "Building" and "Interval_Length"
:param client: An xbos client.
:param interval: the interval in which to split thermal data actions.
"""
self.building_cfg = building_cfg
self.building = building_cfg["Building"]
self.interval = interval # the interval in which to split thermal data actions.
self.client = client
self.window_size = 1 # minutes. TODO should come from config.
self.hod_client = HodClient("xbos/hod", self.client) # TODO Potentially could incorporate this into MDAL query.
def _preprocess_outside_data(self, outside_data):
"""
:param outside_data: (list) of pd.
with col ["t_out"] for each weather station.
:return: pd.df with col ["t_out"]
"""
# since the archiver had a bug while storing weather.gov data, nan values were stored as 32. We will
# replace any 32 values with Nan. Since we usually have more than one weather station, we can then take the
# average across those to compensate. If no other weather stations, then issue a warning that
# we might be getting rid of real data and we won't be able to recover through the mean.
if len(outside_data) == 1:
print("WARNING: Only one weather station for selected region. We need to replace 32 values with Nan due to "
"past inconsistencies, but not enough data to compensate for the lost data by taking mean.")
for i in range(len(outside_data)):
temp_data = outside_data[i]["t_out"].apply(lambda t: np.nan if t == 32 else t) # TODO this only works for fahrenheit now.
outside_data[i]["t_out"] = temp_data
# Note: Assuming same index for all weather station data returned by mdal
final_outside_data = pd.concat(outside_data, axis=1).mean(axis=1)
final_outside_data = pd.DataFrame(final_outside_data, columns=["t_out"])
# outside temperature may contain nan values.
# Hence, we interpolate values linearly,
# since outside temperature does not need to be as accurate as inside data.
final_outside_data = final_outside_data.interpolate()
return final_outside_data
def _get_outside_data(self, start=None, end=None, inclusive=False):
# TODO update docstring
"""Get outside temperature for thermal model.
:param start: (datetime) time to start. in UTC time.
:param end: (datetime) time to end. in UTC time.
:param inclusive: (bool) whether the end time should be inclusive.
which means that we get the time for the end, plus 15 min.
:return ({uuid: (pd.df) (col: "t_out) outside_data}) outside temperature has freq of 15 min and
pd.df columns["tin", "action"] has freq of self.window_size. """
# add an interval, to make the end inclusive, which means that we get the time for the end, plus 15 min.
if inclusive:
end += datetime.timedelta(minutes=15)
outside_temperature_query = """SELECT ?weather_station ?uuid FROM %s WHERE {
?weather_station rdf:type brick:Weather_Temperature_Sensor.
?weather_station bf:uuid ?uuid.
};"""
# get outside temperature data
# TODO UPDATE OUTSIDE TEMPERATURE STUFF
# TODO for now taking all weather stations and preprocessing it. Should be determined based on metadata.
outside_temperature_query_data = self.hod_client.do_query(outside_temperature_query % self.building)["Rows"]
outside_temperature_data = {}
for weather_station in outside_temperature_query_data:
# Get data from MDAL
mdal_client = mdal.MDALClient("xbos/mdal", client=self.client)
mdal_query = {
'Composition': [weather_station["?uuid"]],
# uuid from Mr.Plotter. should use outside_temperature_query_data["?uuid"],
'Selectors': [mdal.MEAN]
, 'Time': {'T0': start.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'T1': end.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize': str(15) + 'min',
# TODO document that we are getting 15 min intervals because then we get fewer nan values.
'Aligned': True}}
mdal_outside_data = utils.get_mdal_data(mdal_client, mdal_query)
mdal_outside_data.columns = ["t_out"]
outside_temperature_data[weather_station["?uuid"]] = mdal_outside_data
return outside_temperature_data
def _get_inside_data(self, start, end):
"""Get thermostat status and temperature and outside temperature for thermal model.
:param start: (datetime) time to start. in UTC time.
:param end: (datetime) time to end. in UTC time.
:return outside temperature has freq of 15 min and
pd.df columns["tin", "action"] has freq of self.window_size. """
# following queries are for the whole building.
thermostat_status_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:hasLocation/bf:isPartOf ?location_zone .
?location_zone rdf:type brick:HVAC_Zone .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?status_point .
?status_point rdf:type brick:Thermostat_Status .
?status_point bf:uuid ?uuid.
};"""
# Start of FIX for missing Brick query
thermostat_status_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?status_point .
?status_point rdf:type brick:Thermostat_Status .
?status_point bf:uuid ?uuid.
};"""
# End of FIX - delete when Brick is fixed
thermostat_temperature_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:hasLocation/bf:isPartOf ?location_zone .
?location_zone rdf:type brick:HVAC_Zone .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?thermostat_point .
?thermostat_point rdf:type brick:Temperature_Sensor .
?thermostat_point bf:uuid ?uuid.
};"""
# Start of FIX for missing Brick query
thermostat_temperature_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?thermostat_point .
?thermostat_point rdf:type brick:Temperature_Sensor .
?thermostat_point bf:uuid ?uuid.
};"""
# End of FIX - delete when Brick is fixed
# get query data
temp_thermostat_query_data = {
"tstat_temperature": self.hod_client.do_query(thermostat_temperature_query % self.building)["Rows"],
"tstat_action": self.hod_client.do_query(thermostat_status_query % self.building)["Rows"],
}
# give the thermostat query data better structure for later loop. Can index by zone and then get uuids for each
# thermostat attribute.
thermostat_query_data = {}
for tstat_attr, attr_dicts in temp_thermostat_query_data.items():
for dict in attr_dicts:
if dict["?zone"] not in thermostat_query_data:
thermostat_query_data[dict["?zone"]] = {}
thermostat_query_data[dict["?zone"]][tstat_attr] = dict["?uuid"]
# get the data for the thermostats for each zone.
mdal_client = mdal.MDALClient("xbos/mdal", client=self.client)
zone_thermal_data = {}
for zone, dict in thermostat_query_data.items():
mdal_query = {'Composition': [dict["tstat_temperature"], dict["tstat_action"]],
'Selectors': [mdal.MEAN, mdal.MEAN]
, 'Time': {'T0': start.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'T1': end.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize': str(self.window_size) + 'min',
'Aligned': True}}
# get the thermostat data
df = utils.get_mdal_data(mdal_client, mdal_query)
zone_thermal_data[zone] = df.rename(columns={dict["tstat_temperature"]: 't_in', dict["tstat_action"]: 'action'})
return zone_thermal_data
def _preprocess_zone_thermal_data(self, thermal_model_data):
"""
Preprocess data for one zone.
:param thermal_model_data: pd.df columns: t_in','t_out', 'action', [other zone temperatures]
:return: pd.df columns: t_in', 't_next', 'dt','t_out', 'action', 'a1', 'a2', [other mean zone temperatures]
"""
# Finding all points where action changed.
thermal_model_data['change_of_action'] = (thermal_model_data['action'].diff(1) != 0).astype(
'int').cumsum() # given a row it's the number of times we have had an action change up till then. e.g. from nothing to heating.
# This is accomplished by taking the difference of two consecutive rows and checking if their difference is 0 meaning that they had the same action.
# following adds the fields "time", "dt" etc such that we accumulate all values where we have consecutively the same action.
# maximally we group terms of total self.interval. To get self.interval of data, we will actually look at
# self.interval + 1 data-points. | -- 1 min -- | -- 1 min -- | -- 1 min -- | will only give us a span of two
# minutes because we can only look at the Tin corresponding to the start of the 1 min windows. So, at the
# start of the third window, only 2 min have passed. (In fact, MDAL gives us the mean of the windows, but we
# can just assume that it is the starting temperature, the logic stays the same.)
# We need to accept that we will loose a datapoint by doing so. Ideally, we should be able to just look at the
# window that follows right after a contigious action block and assume that that is the t_next. However, since
# we have many nan values, that t_next could be hours in the future.
# NOTE: Only dropping nan value during gathering stage and after that. Before then not doing it because
# we want to keep the times contigious.
data_list = []
for j in thermal_model_data.change_of_action.unique():
for i in range(0, thermal_model_data[thermal_model_data['change_of_action'] == j].shape[0],
self.interval + 1):
for dfs in [thermal_model_data[thermal_model_data['change_of_action'] == j][i:i + self.interval + 1]]:
# we only look at intervals where the last and first value for T_in are not Nan.
dfs.dropna(subset=["t_in"])
zone_col_filter = ["zone_temperature_" in col for col in dfs.columns]
temp_data_dict = {'time': dfs.index[0],
't_in': dfs['t_in'][0],
't_next': dfs['t_in'][-1],
'dt': (dfs.index[-1] - dfs.index[0]).seconds / 60,
't_out': dfs['t_out'].mean(), # mean does not count Nan values
'action': dfs['action'][
0]} # TODO document why we expect no nan in a block. There actually is a ton of nan
for temperature_zone in dfs.columns[zone_col_filter]:
# mean does not count Nan values
temp_data_dict[temperature_zone] = dfs[temperature_zone].mean()
data_list.append(temp_data_dict)
thermal_model_data = pd.DataFrame(data_list).set_index('time')
thermal_model_data = thermal_model_data.dropna() # final drop. Mostly if the whole interval for the zones or t_out were nan.
return thermal_model_data
def _evaluation_preprocess_zone_thermal_data(self, thermal_model_data):
"""
Preprocess data for one zone. Add two field t_min and t_max which tells you how much the data varied. Add t_mean and
t_std as well.
:param thermal_model_data: pd.df columns: t_in','t_out', 'action', [other zone temperatures]
:return: pd.df columns: 't_in', 't_next', 't_min', 't_max', 'dt','t_out', 'action', 'a1', 'a2', [other mean zone temperatures]
"""
# Finding all points where action changed.
thermal_model_data['change_of_action'] = (thermal_model_data['action'].diff(1) != 0).astype(
'int').cumsum() # given a row it's the number of times we have had an action change up till then. e.g. from nothing to heating.
# This is accomplished by taking the difference of two consecutive rows and checking if their difference is 0 meaning that they had the same action.
# following adds the fields "time", "dt" etc such that we accumulate all values where we have consecutively the same action.
# maximally we group terms of total self.interval. To get self.interval of data, we will actually look at
# self.interval + 1 data-points. | -- 1 min -- | -- 1 min -- | -- 1 min -- | will only give us a span of two
# minutes because we can only look at the Tin corresponding to the start of the 1 min windows. So, at the
# start of the third window, only 2 min have passed. (In fact, MDAL gives us the mean of the windows, but we
# can just assume that it is the starting temperature, the logic stays the same.)
# We need to accept that we will loose a datapoint by doing so. Ideally, we should be able to just look at the
# window that follows right after a contigious action block and assume that that is the t_next. However, since
# we have many nan values, that t_next could be hours in the future.
# NOTE: Only dropping nan value during gathering stage and after that. Before then not doing it because
# we want to keep the times contigious.
data_list = []
for j in thermal_model_data.change_of_action.unique():
for i in range(0, thermal_model_data[thermal_model_data['change_of_action'] == j].shape[0],
self.interval + 1):
for dfs in [thermal_model_data[thermal_model_data['change_of_action'] == j][i:i + self.interval + 1]]:
# we only look at intervals where the last and first value for T_in are not Nan.
dfs.dropna(subset=["t_in"])
zone_col_filter = ["zone_temperature_" in col for col in dfs.columns]
temp_data_dict = {'time': dfs.index[0],
't_in': dfs['t_in'][0],
't_next': dfs['t_in'][-1],
't_mean': np.mean(dfs['t_in']),
't_std': np.std(dfs['t_in']),
't_min': np.min(dfs['t_in']),
't_max': np.max(dfs['t_in']),
'dt': (dfs.index[-1] - dfs.index[0]).seconds / 60,
't_out': dfs['t_out'].mean(), # mean does not count Nan values
'action': dfs['action'][0]} # TODO document why we expect no nan in a block. There actually is a ton of nan
for temperature_zone in dfs.columns[zone_col_filter]:
# mean does not count Nan values
temp_data_dict[temperature_zone] = dfs[temperature_zone].mean()
data_list.append(temp_data_dict)
thermal_model_data = pd.DataFrame(data_list).set_index('time')
print("Before drop", thermal_model_data.shape) # TODO turns out we drop a lot of data. check why.
thermal_model_data = thermal_model_data.dropna() # final drop. Mostly if the whole interval for the zones or t_out were nan.
print("After drop", thermal_model_data.shape)
return thermal_model_data
def _preprocess_thermal_data(self, zone_data, outside_data, evaluate_preprocess=False):
"""Preprocesses the data for the thermal model.
:param zone_data: dict{zone: pd.df columns["tin", "action"]}
:param outside_data: pd.df columns["tout"].
NOTE: outside_data freq has to be a multiple of zone_data frequency and has to have a higher freq.
:returns {zone: pd.df columns: t_in', 't_next', 'dt','t_out', 'action', 'a1', 'a2', [other mean zone temperatures]}
where t_out and zone temperatures are the mean values over the intervals.
a1 is whether heating and a2 whether cooling."""
# thermal data preprocess starts here
all_temperatures = pd.concat([tstat_df["t_in"] for tstat_df in zone_data.values()], axis=1)
all_temperatures.columns = ["zone_temperature_" + zone for zone in zone_data.keys()]
zone_thermal_model_data = {}
for zone in zone_data.keys():
# Putting together outside and zone data.
actions = zone_data[zone]["action"]
thermal_model_data = pd.concat([all_temperatures, actions, outside_data],
axis=1) # should be copied data according to documentation
thermal_model_data = thermal_model_data.rename(columns={"zone_temperature_" + zone: "t_in"})
# thermal_model_data['action'] = thermal_model_data.apply(utils.f3, axis=1) # TODO if we get 0.5 the heating happend for half the time.
# NOTE:
# The dataframe will have nan values because outside_temperature does not have same frequency as zone_data.
# Hence, we interpolate values linearly,
# since outside temperature does not need to be as accurate as inside data.
# TODO interpolate the temperatures also for forecasting reports if needed.
thermal_model_data["t_out"] = thermal_model_data["t_out"].interpolate()
# From here on preprocessing data. Concatinating all time contigious datapoints which have the same action.
if evaluate_preprocess:
zone_thermal_model_data[zone] = self._evaluation_preprocess_zone_thermal_data(thermal_model_data)
else:
zone_thermal_model_data[zone] = self._preprocess_zone_thermal_data(thermal_model_data)
print('one zone preproccessed')
return zone_thermal_model_data
def thermal_data(self, start=None, end=None, days_back=60, evaluate_preprocess=False):
"""
:param start: In UTC time.
:param end: In UTC time.
:param days_back: if start is None, then we set start to end - timedelta(days=days_back).
:param evaluate_preprocess: Whether to add the fields t_max, t_min, t_mean, t_std to the preprocessed data.
:return: pd.df {zone: pd.df columns: t_in', 't_next', 'dt','t_out', 'action', 'a1', 'a2', [other mean zone temperatures]}
where t_out and zone temperatures are the mean values over the intervals.
a1 is whether heating and a2 whether cooling.
"""
if end is None:
end = utils.get_utc_now()
if start is None:
start = end - timedelta(days=days_back)
z, o = self._get_inside_data(start, end), self._get_outside_data(start, end)
# Take mean of all weather stations.
o = self._preprocess_outside_data(o.values())
print("Received Thermal Data from MDAL.")
return self._preprocess_thermal_data(z, o, evaluate_preprocess)
def thermostat_setpoints(self, start, end, window_size=1):
"""
Gets the thermostat setpoints from archiver from start to end. Does not preprocess the data.
:param start: datetime in utc time.
:param end: datetime in utc time.
:param window_size: The frequency with which to get the data.
:return: pd.df columns="t_high", "t_low" with timeseries in utc time with freq=window_size.
"""
cooling_setpoint_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?setpoint .
?setpoint rdf:type brick:Supply_Air_Temperature_Cooling_Setpoint .
?setpoint bf:uuid ?uuid.
};"""
heating_setpoint_query = """SELECT ?zone ?uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat .
?tstat bf:controls ?RTU .
?RTU rdf:type brick:RTU .
?RTU bf:feeds ?zone.
?zone rdf:type brick:HVAC_Zone .
?tstat bf:hasPoint ?setpoint .
?setpoint rdf:type brick:Supply_Air_Temperature_Heating_Setpoint .
?setpoint bf:uuid ?uuid.
};"""
hod_client = HodClient("xbos/hod", self.c)
# get query data
temp_thermostat_query_data = {
"cooling_setpoint":
hod_client.do_query(cooling_setpoint_query %
self.controller_cfg["Building"])["Rows"],
"heating_setpoint":
hod_client.do_query(heating_setpoint_query %
self.controller_cfg["Building"])["Rows"],
}
# give the thermostat query data better structure for later loop. Can index by zone and then get uuids for each
# thermostat attribute.
thermostat_query_data = {}
for tstat_attr, attr_dicts in temp_thermostat_query_data.items():
for dict in attr_dicts:
if dict["?zone"] not in thermostat_query_data:
thermostat_query_data[dict["?zone"]] = {}
thermostat_query_data[
dict["?zone"]][tstat_attr] = dict["?uuid"]
# get the data for the thermostats for each zone.
mdal_client = mdal.MDALClient("xbos/mdal", client=self.c)
zone_setpoint_data = {}
for zone, dict in thermostat_query_data.items():
mdal_query = {
'Composition':
[dict["heating_setpoint"], dict["cooling_setpoint"]],
'Selectors': [mdal.MEAN, mdal.MEAN],
'Time': {
'T0': start.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'T1': end.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize': str(window_size) + 'min',
'Aligned': True
}
}
# get the thermostat data
df = utils.get_mdal_data(mdal_client, mdal_query)
zone_setpoint_data[zone] = df.rename(
columns={
dict["heating_setpoint"]: 't_low',
dict["cooling_setpoint"]: 't_high'
})
return zone_setpoint_data
# TODO check for comfortband height and whether correctly implemented
building = sys.argv[1]
# read from config file
try:
yaml_filename = "Buildings/%s/%s.yml" % (sys.argv[1], sys.argv[1])
except:
sys.exit(
"Please specify the configuration file as: python2 controller.py config_file.yaml"
)
cfg = utils.get_config(building)
client = utils.choose_client() # TODO add config
hc = HodClient("xbos/hod", client)
tstats = utils.get_thermostats(client, hc, cfg["Building"])
# --- Thermal Model Init ------------
# initialize and fit thermal model
# only single stage cooling buildings get to retrive data. otherwise takes too long.
# if building in ["north-berkeley-senior-center", "ciee", "avenal-veterans-hall", "orinda-community-center",
# "avenal-recreation-center", "word-of-faith-cc", "jesse-turner-center", "berkeley-corporate-yard"]:
if building != "south-berkeley-senior-center":
thermal_data = utils.get_data(cfg=cfg,
client=client,
days_back=150,
force_reload=False)
from xbos.services.pundat import DataClient, timestamp, make_dataframe, merge_dfs
# for performing Brick queries
from xbos.services.hod import HodClient
# for interacting with the thermostat control state
from xbos.devices.thermostat import Thermostat
# for deserializing messages
import msgpack
import time
import pandas as pd
# get a bosswave client
c = get_client() # defaults to $BW2_AGENT, $BW2_DEFAULT_ENTITY
# get a HodDB client
hod = HodClient("ciee/hod", c)
# get an archiver client
archiver = DataClient(c, archivers=["ucberkeley"])
# mode
OFF = 0
HEAT = 1
COOL = 2
AUTO = 3
# store zone name to thermostat
zone2tstat = {}
# store zone name to meter for the RTU for that zone
zone2meter = {}
# query for the the thermostat APIs. Once we have the BOSSWAVE URI, we can instantiate
# a Thermostat object in order to control it.
# Easiest way to get it is by using get_client() which you import from xbos. Other ways include entity files.
# https://github.com/SoftwareDefinedBuildings/XBOS.
# To use xbos make sure to get an entity file from Thanos and to get a executable file which
# connects you to the system. Also, make sure to set the entity file in your bash_profile with
# export BW2_DEFAULT_ENTITY=path to .ent file
# The MDAL client gets the data from our database. The query to get the data is illustrated by,
# buidling_meteres_query_mdal and lighting_meter_query_mdal.
# Documentation: https://docs.xbos.io/mdal.html#using and https://github.com/gtfierro/mdal <- better
mdal = MDALClient("xbos/mdal", client=get_client())
# HODClient gets the uuid for data. This uses brick which is a language built on SPARQL.
# Can be trick to use.
# To try your own queries go to: corbusier.cs.berkeley.edu:47808. And try the queries we set up below.
# Documentation: for brick: brickschema.org/structure/
# If you need queries, it's best to ask either Thanos or Daniel.
hod = HodClient("xbos/hod") #IDs of thermostat
# temporal parameters
SITE = "ciee" #building name
# Brick queries -- graphical rep of building, queries of where to get data
building_meters_query = """SELECT ?meter ?meter_uuid FROM %s WHERE {
?meter rdf:type brick:Building_Electric_Meter .
?meter bf:uuid ?meter_uuid .
};"""
thermostat_state_query = """SELECT ?tstat ?status_uuid FROM %s WHERE {
?tstat rdf:type brick:Thermostat_Status .
?tstat bf:uuid ?status_uuid .
};"""
lighting_state_query = """SELECT ?lighting ?state_uuid FROM %s WHERE {
?light rdf:type brick:Lighting_State .
def lights(building, client, actuate=False):
ACTUATE = actuate
SITE = building # Cahnge this according to the site
# print RoomsType.RoomsType[df['column_name'] == some_value]
c = client
hod = HodClient("xbos/hod")
skipped = ""
####### occ query
occ_query = """SELECT * FROM %s WHERE {
?l rdf:type brick:Lighting_System .
?l bf:feeds ?room .
?room rdf:type brick:Room .
?l bf:uri ?luri .
?l bf:hasPoint ?p .
?p rdf:type brick:Occupancy_Sensor .
?p bf:uri ?puri .
?p bf:uuid ?puuid
};"""
res2 = hod.do_query(occ_query % SITE)
byrooms_o = defaultdict(list)
for item in res2['Rows']:
byrooms_o[item['?room']].append(item['?puuid'])
print "Occupancy uuids loaded!"
####### light query
light_query = """
SELECT ?light ?room ?light_uri FROM %s WHERE {
?light rdf:type brick:Lighting_System .
?light bf:feeds ?room .
?room rdf:type brick:Room .
?light bf:uri ?light_uri
};
"""
res = hod.do_query(light_query % SITE)
byrooms = defaultdict(list)
all_rooms = set()
for item in res['Rows']:
all_rooms.add(item['?room'])
try:
l = Light(c, item['?light_uri'])
byrooms[item['?room']].append(l)
except:
skipped += str(item['?light']) + "in room" + str(
item['?room']) + " skipped! \n"
pass
print "Lighting systems loaded!"
####### room type query
type_query = """SELECT * FROM %s WHERE {
?room rdf:type brick:Room.
?room rdf:label ?label
};
"""
res3 = hod.do_query(type_query % SITE)
byrooms_type = defaultdict(list)
for item in res3['Rows']:
byrooms_type[item['?room']].append(item['?label'])
print byrooms_type
print "Room types loaded!"
Actuated = ""
################################################ Controls
c = mdal.MDALClient("xbos/mdal")
for room in all_rooms:
Type = byrooms_type[room][0]
if Type == "Hallway":
for light in byrooms[room]:
if ACTUATE:
brightness = 20
Actuated += "Lights in room" + str(
room) + " was set to" + str(brightness) + "\n"
light.set_brightness(min(light.brightness, brightness))
elif Type == "Toilet":
for light in byrooms[room]:
print "c" # print light.brightness
if ACTUATE:
brightness = 10
Actuated += "Lights in room" + str(
room) + " was set to" + str(brightness) + "\n"
light.set_brightness(min(light.brightness, brightness))
else:
query_list = []
for i in byrooms_o[room]:
query_list.append(i)
# get the sensor data
now = datetime.datetime.utcnow().replace(
tzinfo=pytz.timezone("UTC"))
dfs = c.do_query({
'Composition': query_list,
'Selectors': [mdal.MAX] * len(query_list),
'Time': {
'T0':
(now - timedelta(hours=0.5)).strftime('%Y-%m-%d %H:%M:%S')
+ ' UTC',
'T1':
now.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize':
str(30) + 'min',
'Aligned':
True
}
})
dfs = pd.concat([dframe for uid, dframe in dfs.items()], axis=1)
df = dfs[[query_list[0]]]
df.columns.values[0] = 'occ'
df.is_copy = False
df.columns = ['occ']
# perform OR on the data, if one sensor is activated, the whole room is considered occupied
for i in range(1, len(query_list)):
df.loc[:, 'occ'] += dfs[query_list[i]]
df.loc[:, 'occ'] = 1 * (df['occ'] > 0)
if df["occ"][0] != 1:
for light in byrooms[room]:
if ACTUATE:
brightness = 0
Actuated += "Lights in room" + str(
room) + " was set to " + str(brightness) + "\n"
light.set_brightness(min(light.brightness, brightness))
else:
for light in byrooms[room]:
print light.brightness
if ACTUATE:
brightness = 10
Actuated += "Lights in room" + str(
room) + " was set to " + str(brightness) + "\n"
light.set_brightness(min(light.brightness, brightness))
print "Done!"
print "================================"
print "We skipped the following lights:"
print skipped
print "We actuated the following lights:"
print Actuated
from xbos.services.hod import HodClient
from xbos.devices.light import Light
from xbos.devices.occupancy_sensor import Occupancy_Sensor
import pandas as pd
from xbos.services import mdal
import datetime, pytz
from datetime import timedelta
############################################### Initializing our datasets
ACTUATE = False
SITE = "ciee" #Cahnge this according to the site
#print RoomsType.RoomsType[df['column_name'] == some_value]
c = get_client()
hod = HodClient("xbos/hod")
skipped = ""
####### occ query
occ_query = """SELECT * FROM %s WHERE {
?l rdf:type brick:Lighting_System .
?l bf:feeds ?room .
?room rdf:type brick:Room .
?l bf:uri ?luri .
?l bf:hasPoint ?p .
?p rdf:type brick:Occupancy_Sensor .
?p bf:uri ?puri .
?p bf:uuid ?puuid
};"""
def data_fetch(cfg, cli, zones):
# state, tin, tout, setp_high, setp_low
UUIDS = {
"SouthZone": [
"dfb2b403-fd08-3e9b-bf3f-18c699ce40d6",
"03099008-5224-3b61-b07e-eee445e64620",
"1c467b79-b314-3c1e-83e6-ea5e7048c37b",
"dbbf4a91-107a-3b15-b2c0-a49b54116daa",
"eeadc8ed-6255-320d-b845-84f44748fe95"
],
"NorthZone": [
"5e55e5b1-007b-39fa-98b6-ae01baa6dccd",
"c7e33fa6-f683-36e9-b97a-7f096e4b57d4",
"1c467b79-b314-3c1e-83e6-ea5e7048c37b",
"9fa56ac1-0f8a-3ad2-86e8-72e816b875ad",
"e4e0db0b-1c15-330e-a864-011e558f542e"
],
"CentralZone": [
"187ed9b8-ee9b-3042-875e-088a08da37ae",
"c05385e5-a947-37a3-902e-f6ea45a43fe8",
"1c467b79-b314-3c1e-83e6-ea5e7048c37b",
"0d037818-02c2-3e5b-87e9-94570d43b418",
"6cbee2ae-06e7-3fc3-a2fc-698fa3deadee"
],
"EastZone": [
"7e543d07-16d1-32bb-94af-95a01f4675f9",
"b47ba370-bceb-39cf-9552-d1225d910039",
"1c467b79-b314-3c1e-83e6-ea5e7048c37b",
"d38446d4-32cc-34bd-b293-0a3871a6759b",
"e4d39723-5907-35bd-a9b2-fc57b58b3779"
]
}
uspac = pytz.timezone(cfg["Pytz_Timezone"])
startime = uspac.localize(
datetime.datetime.strptime(
cfg["Start_Date"], '%Y-%m-%d %H:%M:%S')).astimezone(tz=pytz.utc)
endtime = startime + timedelta(hours=24)
hod = HodClient(cfg["Building"] + "/hod", cli)
occ_query = """SELECT ?sensor ?uuid ?zone WHERE {
?sensor rdf:type brick:Occupancy_Sensor .
?sensor bf:isLocatedIn/bf:isPartOf ?zone .
?sensor bf:uuid ?uuid .
?zone rdf:type brick:HVAC_Zone
};
""" # get all the occupancy sensors uuids
results = hod.do_query(occ_query) # run the query
occ_uuids = [[x['?zone'], x['?uuid']] for x in results['Rows']] # unpack
dataframes = {}
for zone in zones:
query_list = []
for i in occ_uuids:
if i[0] == zone:
query_list.append(i[1])
c = mdal.MDALClient("xbos/mdal", client=cli)
dfs = c.do_query({
'Composition':
UUIDS[zone],
'Selectors':
[mdal.MAX, mdal.MEAN, mdal.MEAN, mdal.MEAN, mdal.MEAN],
'Time': {
'T0': startime.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'T1': endtime.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize': '1min',
'Aligned': True
}
})
df = pd.concat([dframe for uid, dframe in dfs.items()], axis=1)
df_th = df.rename(
columns={
UUIDS[zone][0]: 'State',
UUIDS[zone][1]: 'Tin',
UUIDS[zone][2]: 'Tout',
UUIDS[zone][3]: "STPH",
UUIDS[zone][4]: "STPL"
})
df_th['change_of_action'] = (df_th['State'].diff(1) !=
0).astype('int').cumsum()
c = mdal.MDALClient("xbos/mdal", client=cli)
dfs = c.do_query({
'Composition': query_list,
'Selectors': [mdal.MAX] * len(query_list),
'Time': {
'T0': startime.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'T1': endtime.strftime('%Y-%m-%d %H:%M:%S') + ' UTC',
'WindowSize': '1min',
'Aligned': True
}
})
dfs = pd.concat([dframe for uid, dframe in dfs.items()], axis=1)
df = dfs[[query_list[0]]]
df.columns.values[0] = 'Occ'
df.is_copy = False
df.columns = ['Occ']
# perform OR on the data, if one sensor is activated, the whole zone is considered occupied
for i in range(1, len(query_list)):
df.loc[:, 'Occ'] += dfs[query_list[i]]
df.loc[:, 'Occ'] = 1 * (df['Occ'] > 0)
df_occ = df
dataframes[zone] = pd.concat([df_th, df_occ], axis=1)
return dataframes