def check_dl_db(self, month=None, forcedl=False):
if not os.path.exists(self._dl_db_path):
os.mkdir(self._dl_db_path)
if month is None:
month = self._now - relativedelta(months=1)
prev_db_path_xls = (self._dl_db_path + F'WEL_log_{month.year}'
F'_{month.month:02d}.xls')
this_month = self._now.date().month == month.month
if (not os.path.exists(prev_db_path_xls)) or forcedl or this_month:
prev_url = ('http://www.welserver.com/WEL1060/'
F'WEL_log_{month.year}_{month.month:02d}')
prev_db_path_zip = (self._dl_db_path + F'WEL_log_{month.year}'
F'_{month.month:02d}.zip')
try:
message(F"Downloading {month.year}-{month.month}:\n",
mssgType='ADMIN')
download(prev_url + '.zip', prev_db_path_zip)
os.system(F'unzip {prev_db_path_zip} -d {self._dl_db_path}'
F';rm {prev_db_path_zip}')
except HTTPError:
try:
download(prev_url + '.xls', prev_db_path_xls)
except Exception:
message(F"Error while downloading log: {HTTPError}",
mssgType='ERROR')
def _plots(self):
tic = time.time()
# if self._sensor_groups is None:
# self._sensor_groups = [self.in_default]
with st.spinner('Generating Plots'):
plot = alt.hconcat(
self.plotNonTime(
'T_diff', 'T_diff_eff').properties(width=self.def_width),
self.plotNonTime('solar_w',
'geo_tot_w').properties(width=self.def_width))
plot = plot.configure_axis(
labelFontSize=self.label_font_size,
titleFontSize=self.title_font_size,
titlePadding=41,
domain=False).configure_legend(
labelFontSize=self.label_font_size,
titleFontSize=self.title_font_size).configure_view(
cornerRadius=2)
message([F"{'Altair plot gen:': <20}", F"{time.time() - tic:.2f} s"],
tbl=self.mssg_tbl,
mssgType='TIMING')
return [plot, plot]
def _getDataSubset(self, vars, id_vars='dateandtime', decimate_factor=1):
if decimate_factor == 1:
source = self.dat_resample
else:
source = self.dat_resample.resample(self.resample_T *
decimate_factor).mean()
source = source.reset_index()
try:
source = source.melt(id_vars=id_vars,
value_vars=vars,
var_name='label')
except KeyError:
goodKeys = []
badKeys = []
for key in vars:
if key in source:
goodKeys.append(key)
else:
badKeys.append(key)
message(["Key(s) not found in db:", F"{badKeys}"],
tbl=self.mssg_tbl,
mssgType='WARNING')
if not goodKeys:
message("No valid keys selected, returning empty dataframe",
mssgType='ERROR')
return pd.DataFrame()
source = source.melt(id_vars=id_vars,
value_vars=goodKeys,
var_name='label')
return source
def _calced_cols(self, frame):
out_frame = pd.DataFrame()
heat_mask = frame.heat_1_b % 2
heat_mask[heat_mask == 0] = np.nan
# Additional calculated columns
frame['power_tot'] = frame.TAH_W + frame.HP_W
try:
out_frame['geo_tot_w'] = frame.TAH_W + frame.TES_sense_w
except AttributeError:
out_frame['geo_tot_w'] = frame.power_tot
try:
out_frame['base_load_w'] = np.abs(frame.house_w -
out_frame['geo_tot_w'] -
frame.dehumidifier_w)
except AttributeError:
pass
try:
out_frame['T_diff'] = np.abs(
np.nanmean([
frame.fireplace_T, frame.D_room_T, frame.V_room_T,
frame.T_room_T
]) - frame.outside_T)
except AttributeError:
out_frame['T_diff'] = np.abs(frame.living_T - frame.outside_T)
out_frame['T_diff_eff'] = (frame.power_tot / out_frame.T_diff)
# COP calculation
air_density = 1.15
surface_area = 0.34
heat_capacity = 1.01
COP = (((air_density * surface_area * heat_capacity * frame.TAH_fpm) *
(np.abs(frame.TAH_out_T - frame.TAH_in_T))) /
(frame.power_tot / 1000))
COP[COP > 5] = np.nan
COP = COP * heat_mask
out_frame['COP'] = COP
# WEL COP calculation
well_gpm = 13.6
gpm_to_lpm = 0.0630902
out_frame['well_W'] = ((well_gpm * gpm_to_lpm) * 4.186 *
(np.abs(frame.loop_out_T - frame.loop_in_T)))
well_COP = out_frame.well_W / (frame.power_tot / 1000)
well_COP[well_COP > 10] = np.nan
well_COP = well_COP * heat_mask
out_frame['well_COP'] = well_COP
# Reset rain accumulation every 24 hrs
try:
rain_offset = (frame.groupby(
frame.index.date)['weather_station_R'].transform(
lambda x: np.mean(x.iloc[-10:-1])))
out_frame['rain_accum_R'] = (frame['weather_station_R'] -
rain_offset)
except KeyError:
message("Weather station rain data not present in selection",
mssgType='WARNING')
return out_frame
async def send_post(post):
utc_time = post['dateandtime'].strftime('%Y-%m-%d %H:%M:%S')
try:
post_id = connects.db.insert_one(post).inserted_id
message(F"Successful post @ UTC time: {utc_time}"
F" | post_id: {post_id}", mssgType='SUCCESS')
except DuplicateKeyError:
message("Tried to insert duplicate key "
F"{post['dateandtime'].strftime('%Y-%m-%d %H:%M:%S')}",
mssgType='WARNING')
async def getRtlData():
tic = time.time()
post = connects.mc.get('rtl')
if post is None:
message("RTL data not found in memCache, "
"excluding RTL from post.",
mssgType='WARNING')
return {}
else:
message([F"{'Getting RTL:': <20}", F"{time.time() - tic:.3f} s"],
mssgType='TIMING')
return post
def makeWEL(self, date_range, force_refresh=False):
tic = time.time()
if not force_refresh:
dat = _cachedWELData(date_range)
else:
dat = WELData(timerange=date_range,
mongo_connection=_cachedMongoConnect())
self.resample_T = (dat.timerange[1] -
dat.timerange[0]) / self.resample_N
self.dat_resample = dat.data.resample(self.resample_T).mean()
message([F"{'WEL Data init:': <20}", F"{time.time() - tic:.2f} s"],
tbl=self.mssg_tbl,
mssgType='TIMING')
def _plots(self):
tic = time.time()
if self._sensor_groups is None:
self._sensor_groups = [self.in_default]
with st.spinner('Generating Plots'):
plot = alt.vconcat(
self.plotStatus().properties(width=self.def_width,
height=self.def_height *
self.stat_height_mod),
self.plotMainMonitor(self._sensor_groups[0]).properties(
width=self.def_width,
height=self.def_height * self.pwr_height_mod),
self.plotPowerStack([
'solar_w', 'base_load_w', 'dehumidifier_w', 'geo_tot_w'
], ).properties(width=self.def_width,
height=self.def_height * self.pwr_height_mod),
# self.plotMainMonitor(self._sensor_groups[1]).properties(
# width=self.def_width,
# height=self.def_height * self.stat_height_mod
# ),
self.plotRollMean(
['COP', 'well_COP'], axis_label="COP").properties(
width=self.def_width,
height=self.def_height * self.stat_height_mod),
self.plotRollMean(['T_diff_eff'],
axis_label="Defficiency / W/°C",
disp_raw=False,
bottomPlot=True,
height_mod=self.stat_height_mod).properties(
width=self.def_width,
height=self.def_height *
self.stat_height_mod,
),
spacing=self.def_spacing).resolve_scale(y='independent',
color='independent')
plot = plot.configure_axis(
labelFontSize=self.label_font_size,
titleFontSize=self.title_font_size,
titlePadding=41,
domain=False).configure_legend(
labelFontSize=self.label_font_size,
titleFontSize=self.title_font_size).configure_view(
cornerRadius=2)
message([F"{'Altair plot gen:': <20}", F"{time.time() - tic:.2f} s"],
tbl=self.mssg_tbl,
mssgType='TIMING')
return [plot]
def accumulate(p):
signals = pd.DataFrame()
tic = time.time()
for line in p.stdout:
packet = processLine(line)
signals = signals.append(packet, ignore_index=True)
if time.time() - tic >= 29:
break
message("Found Signals:", mssgType='HEADER')
[
print(F"{22 * ' '}{idx: <25}{value}", flush=True)
for idx, value in signals.count().items()
]
signals.drop_duplicates(inplace=True)
return signals.mean().to_dict()
def processLine(line):
line = json.loads(line)
packet = {}
try:
id = F"{line['id']}_{line['message_type']}"
except KeyError:
id = str(line['id'])
try:
for quantity in id_to_name[id]['sensors']:
sensor_name = (F"{id_to_name[id]['name']}_"
F"{quantity_short[quantity]}")
packet[sensor_name] = float(line[quantity])
return packet
except KeyError:
message([F"Unknown Sensor ID: {id}", F"\n{line}"], mssgType='WARNING')
def connectSense():
sn = Senseable()
if platform.system() == 'Linux':
path = "/home/ubuntu/WEL/WELPi/sense_info.txt"
elif platform.system() == 'Darwin':
path = "./sense_info.txt"
sense_info = open(path).read().strip().split()
try:
sn.authenticate(*sense_info)
except Exception as e:
message("Error in authenticating with Sense, "
F"excluding Sense from post. \n Error: {e}", mssgType='ERROR')
sn.rate_limit = 10
message("Sense Connected", mssgType='ADMIN')
return sn
def __init__(self,
data_source='Pi',
timerange=None,
WEL_download=False,
dl_db_path='../log_db/',
mongo_connection=None,
calc_cols=True):
self._calc_cols = calc_cols
self._data_source = data_source
self._dl_db_path = dl_db_path
self._now = dt.datetime.now().astimezone(self._to_tzone)
if timerange is None:
self.timerange = self.time_from_args()
elif type(timerange[0]) is str:
self.timerange = self.time_from_args(timerange)
else:
self.timerange = self.timeCondition(timerange)
self.timerange = [
time.replace(
tzinfo=self._to_tzone) if time.tzinfo is None else time
for time in self.timerange
]
if self._data_source == 'WEL':
self.refresh_db()
if WEL_download:
dat_url = ("http://www.welserver.com/WEL1060/" +
F"WEL_log_{self._now.year}" +
F"_{self._now.month:02d}.xls")
downfilepath = (self._dl_db_path +
F"WEL_log_{self._now.year}" +
F"_{self._now.month:02d}.xls")
downfile = download(dat_url, downfilepath)
if os.path.exists(downfilepath):
move(downfile, downfilepath)
self._stitch()
elif self._data_source == 'Pi':
if mongo_connection is None:
self._mongo_db = mongoConnect()
else:
self._mongo_db = mongo_connection
self._stitch()
else:
message("Valid data sources are 'Pi' or 'WEL'", mssgType='WARNING')
quit()
def calc_stats(stp):
last_rev_valve = np.round(stp.dat_resample['rev_valve_b'][-1] % 2)
rev_valve_stat = {1: "Cooling", 0: "Heating"}
N = len(stp.dat_resample)
heat_2_count = (stp.dat_resample['heat_2_b'] % 2).sum()
heat_1_count = (stp.dat_resample['heat_1_b'] % 2).sum() - heat_2_count
# Heat 1 is ~80% of full power
duty = 100 * ((0.8 * heat_1_count + heat_2_count) / N)
try:
house_w_avg = stp.dat_resample['house_w'].mean() / 1000
geo_w_avg = stp.dat_resample['geo_tot_w'].mean() / 1000
except KeyError:
message("House power data not available",
mssgType='WARNING',
tbl=stp.mssg_tbl)
house_w_avg = np.nan
geo_w_avg = np.nan
return [duty, house_w_avg, geo_w_avg, rev_valve_stat[last_rev_valve]]
def main():
message("\n Restarted ...", mssgType='ADMIN')
mc = connectMemCache()
time.sleep(5)
with subprocess.Popen(rtl_cmd, stdout=subprocess.PIPE, text=True) as p:
while True:
signals = accumulate(p)
mc_result = mc.set("rtl", signals)
if not mc_result:
message("RTL failed to cache", mssgType='ERROR')
else:
message("Succesful cache", mssgType='SUCCESS')
async def getWELData(ip):
tic = time.time()
url = "http://" + ip + ":5150/data.xml"
post = {}
local_now = (dt.datetime.now()
.replace(microsecond=0)
.replace(tzinfo=TO_TZONE))
sunrise = sun.sunrise(LOC.observer, date=local_now.date(),
tzinfo=TO_TZONE).astimezone(DB_TZONE)
sunset = sun.sunset(LOC.observer, date=local_now.date(),
tzinfo=TO_TZONE).astimezone(DB_TZONE)
post['daylight'] = ((local_now > sunrise)
and (local_now < sunset)) * 1
try:
response = requests.get(url)
except ConnectionError:
message("Error in connecting to WEL, waiting 10 sec then trying again",
mssgType='WARNING')
time.sleep(10)
try:
response = requests.get(url)
except ConnectionError:
message("Second error in connecting to WEL, "
"excluding WEL from post.",
mssgType='ERROR')
return post
response_data = xmltodict.parse(response.content)['Devices']['Device']
for item in response_data:
try:
post[item['@Name']] = float(item['@Value'])
except ValueError:
post[item['@Name']] = item['@Value']
del post['Date']
del post['Time']
message([F"{'Getting WEL:': <20}", F"{time.time() - tic:.1f} s"],
mssgType='TIMING')
return post
def _stitch(self):
if self._data_source == 'WEL':
load_new = False
if self.data is not None:
if ((self.data.dateandtime.iloc[0] > self.timerange[0]) or
(self.data.dateandtime.iloc[-1] < self.timerange[1])):
load_new = True
else:
load_new = True
if load_new:
num_months = (
(self.timerange[1].year - self.timerange[0].year) * 12 +
self.timerange[1].month - self.timerange[0].month)
monthlist = [
self.timerange[0] + relativedelta(months=x)
for x in range(num_months + 1)
]
loadedstring = [
F'{month.year}-{month.month}' for month in monthlist
]
message(F'Loaded: {loadedstring}', mssgType='ADMIN')
datalist = [
self.read_log(self._dl_db_path + F'WEL_log_{month.year}' +
F'_{month.month:02d}.xls')
for month in monthlist
]
# print(datalist)
self.data = pd.concat(datalist)
# Shift power meter data by one sample for better alignment
self.data.HP_W = self.data.HP_W.shift(-1)
self.data.TAH_W = self.data.TAH_W.shift(-1)
tmask = ((self.data.index > self.timerange[0])
& (self.data.index < self.timerange[1]))
self.data = self.data[tmask]
if self._data_source == 'Pi':
query = {
'dateandtime': {
'$gte': self.timerange[0].astimezone(self._db_tzone),
'$lte': self.timerange[1].astimezone(self._db_tzone)
}
}
# print(F"#DEBUG: query: {query}")
self.data = pd.DataFrame(list(self._mongo_db.data.find(query)))
if len(self.data) == 0:
raise Exception("No data came back from mongo server.")
self.data.index = self.data['dateandtime']
self.data = self.data.drop(columns=['dateandtime'])
# print(self.data.columns)
self.data = self.data.tz_localize(self._db_tzone)
self.data = self.data.tz_convert(self._to_tzone)
# print(F"#DEBUG: timerange from: {self.data.index[-1]}"
# "to {self.data.index[0]}")
# Shift power meter data by one sample for better alignment
self.data.HP_W = self.data.HP_W.shift(-1)
self.data.TAH_W = self.data.TAH_W.shift(-1)
self.data = pd.concat((self.data, self._calced_cols(self.data)),
axis=1)
def plotRollMean(self,
vars,
axis_label="COP Rolling Mean",
height_mod=1,
disp_raw=True,
bottomPlot=False):
source = self._getDataSubset(vars)
# 3 * number of hours in displayed timerange
rolling_frame = (3 * self.resample_N *
(self.resample_N * self.resample_T).total_seconds() /
3600)
rolling_frame = int(
np.clip(rolling_frame, self.resample_N / 48, self.resample_N / 2))
try:
rolling_source = pd.DataFrame(
{'rolling_limit': source.dateandtime.iloc[-rolling_frame]},
index=[0])
except IndexError:
message(["Rolling frame IndexError:", F"{-rolling_frame}"],
tbl=self.mssg_tbl,
mssgType='WARNING')
rolling_source = pd.DataFrame(
{'rolling_limit': source.dateandtime.iloc[-1]}, index=[0])
lines = alt.Chart(source).transform_window(
rollmean='mean(value)',
frame=[-rolling_frame,
0]).mark_line(interpolate='basis', strokeWidth=2).encode(
x=alt.X(
'dateandtime:T',
# scale=alt.Scale(domain=self.resize),
axis=alt.Axis(grid=False, labels=False,
ticks=False),
title=None),
y=alt.Y('rollmean:Q',
scale=alt.Scale(zero=False),
axis=alt.Axis(orient='right', grid=True),
title=axis_label),
color=alt.Color('label',
legend=alt.Legend(title='Efficiencies',
orient='left',
offset=5)))
window_line = alt.Chart(rolling_source).mark_rule(
strokeDash=[5, 5], ).encode(x='rolling_limit:T',
color=alt.ColorValue('gold '))
rule = self._createRules(lines,
field='rollmean:Q',
timetext=bottomPlot,
timetextheightmod=height_mod)
latest_text = self._createLatestText(lines, 'rollmean:Q')
if disp_raw:
raw_lines = alt.Chart(source).mark_line(
interpolate='basis',
strokeWidth=2,
strokeDash=[1, 2],
opacity=0.8,
clip=True).encode(x=alt.X('dateandtime:T'),
y=alt.Y('value:Q'),
color='label')
plot = alt.layer(self._plotNightAlt(), lines, raw_lines, rule,
latest_text, window_line)
return plot
plot = alt.layer(self._plotNightAlt(), lines, rule, latest_text,
window_line)
return plot
def _plots(self):
tic = time.time()
if self._sensor_groups is None:
self._sensor_groups = [self.wthr_default, self.in_humid_default]
with st.spinner('Generating Plots'):
plot = alt.vconcat(
self.plotStatus().properties(
width=self.def_width,
height=self.def_height * self.stat_height_mod
),
self.plotMainMonitor(self._sensor_groups[0]).properties(
width=self.def_width,
height=self.def_height * self.stat_height_mod
),
self.plotMainMonitor(self.out_humid_default,
axis_label="Outdoor Humidity / %",
).properties(
width=self.def_width,
height=self.def_height * self.stat_height_mod
),
self.plotMainMonitor(self._sensor_groups[1],
axis_label="Indoor Humidity / %",
).properties(
width=self.def_width,
height=self.def_height * self.stat_height_mod
),
self.plotMainMonitor('rain_accum_R',
axis_label='Rain / mm',
).properties(
width=self.def_width,
height=self.def_height * self.cop_height_mod
),
self.plotMainMonitor('weather_station_W',
axis_label='Wind / km/h',
height_mod=self.cop_height_mod,
bottomPlot=True
).properties(
width=self.def_width,
height=self.def_height * self.cop_height_mod
),
spacing=self.def_spacing
).resolve_scale(
y='independent',
color='independent'
)
plot = plot.configure_axis(
labelFontSize=self.label_font_size,
titleFontSize=self.title_font_size,
titlePadding=41,
domain=False
).configure_legend(
labelFontSize=self.label_font_size,
titleFontSize=self.title_font_size
).configure_view(
cornerRadius=2
)
message([F"{'Altair plot gen:': <20}", F"{time.time() - tic:.2f} s"],
tbl=self.mssg_tbl, mssgType='TIMING')
return [plot]
def _serverStartup():
message("Server Started", mssgType='ADMIN')
def main():
_serverStartup()
st.markdown(F"""
<style>
.css-dm3ece{{
width: {90}%;
}}
.reportview-container .main .block-container{{
max-width: {1500}px;
padding-top: {5}px;
padding-right: {90}px;
padding-left: {10}px;
padding-bottom: {5}px;
}}
.css-1u96g9d{{
display: none
}}
</style>
<style class='css-1u96g9d'>
details {{
display: none;
}}
</style>
""",
unsafe_allow_html=True)
# -- sidebar --
st.sidebar.subheader("Monitor:")
stats_containers = [st.sidebar.beta_container() for x in range(3)]
which = st.sidebar.selectbox("Select Display Page",
['monit', 'pandw', 'wthr'],
index=0,
format_func=_whichFormatFunc)
st.sidebar.subheader("Plot Options:")
date_range = _date_select()
sensor_container = st.sidebar.beta_container()
max_samples = int(
np.clip((date_range[1] - date_range[0]).total_seconds() / 30, 260,
1440))
resample_N = st.sidebar.slider("Number of Data Samples",
min_value=10,
max_value=max_samples,
value=300,
step=10)
# -- main area --
st.header(F"{_whichFormatFunc(which)} Monitor")
stp = _page_select(resample_N, date_range, sensor_container, which)
stats = calc_stats(stp)
stats_containers[0].markdown(F"System Duty: `{stats[0]:.1f} %`"
F" `{stats[3]}`")
stats_containers[1].markdown(F"House Mean Power Use: `{stats[1]:.2f} kW`")
stats_containers[1].markdown(F"Geo Mean Power Use: `{stats[2]:.2f} kW | "
F"{100 * stats[2] / stats[1]:.0f} %`")
tic = time.time()
for plot in stp.plots:
st.altair_chart(plot)
message([F"{'Altair plot disp:': <20}", F"{time.time() - tic:.2f} s"],
tbl=stp.mssg_tbl,
mssgType='TIMING')
st.sidebar.markdown("[Github Project]"
"(https://github.com/TristanShoemaker/WELPi)")
post_id = connects.db.insert_one(post).inserted_id
message(F"Successful post @ UTC time: {utc_time}"
F" | post_id: {post_id}", mssgType='SUCCESS')
except DuplicateKeyError:
message("Tried to insert duplicate key "
F"{post['dateandtime'].strftime('%Y-%m-%d %H:%M:%S')}",
mssgType='WARNING')
async def main(interval):
while True:
then = time.time()
post = await getWELData(WEL_IP)
post.update(await getRtlData())
post.update(await getSenseData())
elapsed = time.time() - then
await asyncio.sleep(interval - elapsed)
post['dateandtime'] = (dt.datetime.utcnow()
.replace(microsecond=0)
.replace(tzinfo=DB_TZONE))
await send_post(post)
if __name__ == "__main__":
message("\n Restarted ...", mssgType='ADMIN')
message("Mongo Connected", mssgType='ADMIN')
connects = SourceConnects()
asyncio.run(main(30))
def connectMemCache():
ip = MONGO_IP + ":11211"
mc = Client([ip])
message("MemCache Connected", mssgType='ADMIN')
return mc
async def getSenseData():
tic = time.time()
try:
connects.sn.update_realtime()
except SenseAPITimeoutException:
message("Sense API timeout, trying reconnect...", mssgType='WARNING')
connects.sn = connectSense()
try:
connects.sn.update_realtime()
except SenseAPITimeoutException:
message("Second Sense API timeout, "
"excluding Sense from post.", mssgType='ERROR')
return {}
except socket.timeout as e:
message("Sense offline, excluding Sense from post."
F"\n Error: {e}", mssgType='ERROR')
return {}
except socket.timeout as e:
message("Sense offline, excluding Sense from post."
F"\n Error: {e}", mssgType='ERROR')
return {}
sense_post = connects.sn.get_realtime()
post = {}
post['solar_w'] = sense_post['solar_w']
post['house_w'] = sense_post['w']
try:
post['dehumidifier_w'] = [device for device in sense_post['devices']
if device['name'] == 'Dehumidifier '][0]['w']
except IndexError:
post['dehumidifier_w'] = 0
message("Dehumidifier not found in sense.", mssgType='WARNING')
try:
post['furnace_w'] = [device for device in sense_post['devices']
if device['name'] == 'Furnace'][0]['w']
except IndexError:
post['furnace_w'] = 0
message("Furnace not found in sense.", mssgType='WARNING')
try:
post['barn_pump_w'] = [device for device in sense_post['devices']
if device['name'] == 'Barn pump'][0]['w']
except IndexError:
post['barn_pump_w'] = 0
message("Barn pump not found in sense.", mssgType='WARNING')
try:
post['TES_sense_w'] = [device for device in sense_post['devices']
if device['name'] == 'TES'][0]['w']
except IndexError:
post['TES_sense_w'] = 0
message("TES not found in sense.", mssgType='WARNING')
message([F"{'Getting Sense:': <20}", F"{time.time() - tic:.1f} s"],
mssgType='TIMING')
return post