def check_dl_db(self, month=None, forcedl=False): if not os.path.exists(self._dl_db_path): os.mkdir(self._dl_db_path) if month is None: month = self._now - relativedelta(months=1) prev_db_path_xls = (self._dl_db_path + F'WEL_log_{month.year}' F'_{month.month:02d}.xls') this_month = self._now.date().month == month.month if (not os.path.exists(prev_db_path_xls)) or forcedl or this_month: prev_url = ('http://www.welserver.com/WEL1060/' F'WEL_log_{month.year}_{month.month:02d}') prev_db_path_zip = (self._dl_db_path + F'WEL_log_{month.year}' F'_{month.month:02d}.zip') try: message(F"Downloading {month.year}-{month.month}:\n", mssgType='ADMIN') download(prev_url + '.zip', prev_db_path_zip) os.system(F'unzip {prev_db_path_zip} -d {self._dl_db_path}' F';rm {prev_db_path_zip}') except HTTPError: try: download(prev_url + '.xls', prev_db_path_xls) except Exception: message(F"Error while downloading log: {HTTPError}", mssgType='ERROR')
def _plots(self): tic = time.time() # if self._sensor_groups is None: # self._sensor_groups = [self.in_default] with st.spinner('Generating Plots'): plot = alt.hconcat( self.plotNonTime( 'T_diff', 'T_diff_eff').properties(width=self.def_width), self.plotNonTime('solar_w', 'geo_tot_w').properties(width=self.def_width)) plot = plot.configure_axis( labelFontSize=self.label_font_size, titleFontSize=self.title_font_size, titlePadding=41, domain=False).configure_legend( labelFontSize=self.label_font_size, titleFontSize=self.title_font_size).configure_view( cornerRadius=2) message([F"{'Altair plot gen:': <20}", F"{time.time() - tic:.2f} s"], tbl=self.mssg_tbl, mssgType='TIMING') return [plot, plot]
def _getDataSubset(self, vars, id_vars='dateandtime', decimate_factor=1): if decimate_factor == 1: source = self.dat_resample else: source = self.dat_resample.resample(self.resample_T * decimate_factor).mean() source = source.reset_index() try: source = source.melt(id_vars=id_vars, value_vars=vars, var_name='label') except KeyError: goodKeys = [] badKeys = [] for key in vars: if key in source: goodKeys.append(key) else: badKeys.append(key) message(["Key(s) not found in db:", F"{badKeys}"], tbl=self.mssg_tbl, mssgType='WARNING') if not goodKeys: message("No valid keys selected, returning empty dataframe", mssgType='ERROR') return pd.DataFrame() source = source.melt(id_vars=id_vars, value_vars=goodKeys, var_name='label') return source
def _calced_cols(self, frame): out_frame = pd.DataFrame() heat_mask = frame.heat_1_b % 2 heat_mask[heat_mask == 0] = np.nan # Additional calculated columns frame['power_tot'] = frame.TAH_W + frame.HP_W try: out_frame['geo_tot_w'] = frame.TAH_W + frame.TES_sense_w except AttributeError: out_frame['geo_tot_w'] = frame.power_tot try: out_frame['base_load_w'] = np.abs(frame.house_w - out_frame['geo_tot_w'] - frame.dehumidifier_w) except AttributeError: pass try: out_frame['T_diff'] = np.abs( np.nanmean([ frame.fireplace_T, frame.D_room_T, frame.V_room_T, frame.T_room_T ]) - frame.outside_T) except AttributeError: out_frame['T_diff'] = np.abs(frame.living_T - frame.outside_T) out_frame['T_diff_eff'] = (frame.power_tot / out_frame.T_diff) # COP calculation air_density = 1.15 surface_area = 0.34 heat_capacity = 1.01 COP = (((air_density * surface_area * heat_capacity * frame.TAH_fpm) * (np.abs(frame.TAH_out_T - frame.TAH_in_T))) / (frame.power_tot / 1000)) COP[COP > 5] = np.nan COP = COP * heat_mask out_frame['COP'] = COP # WEL COP calculation well_gpm = 13.6 gpm_to_lpm = 0.0630902 out_frame['well_W'] = ((well_gpm * gpm_to_lpm) * 4.186 * (np.abs(frame.loop_out_T - frame.loop_in_T))) well_COP = out_frame.well_W / (frame.power_tot / 1000) well_COP[well_COP > 10] = np.nan well_COP = well_COP * heat_mask out_frame['well_COP'] = well_COP # Reset rain accumulation every 24 hrs try: rain_offset = (frame.groupby( frame.index.date)['weather_station_R'].transform( lambda x: np.mean(x.iloc[-10:-1]))) out_frame['rain_accum_R'] = (frame['weather_station_R'] - rain_offset) except KeyError: message("Weather station rain data not present in selection", mssgType='WARNING') return out_frame
async def send_post(post): utc_time = post['dateandtime'].strftime('%Y-%m-%d %H:%M:%S') try: post_id = connects.db.insert_one(post).inserted_id message(F"Successful post @ UTC time: {utc_time}" F" | post_id: {post_id}", mssgType='SUCCESS') except DuplicateKeyError: message("Tried to insert duplicate key " F"{post['dateandtime'].strftime('%Y-%m-%d %H:%M:%S')}", mssgType='WARNING')
async def getRtlData(): tic = time.time() post = connects.mc.get('rtl') if post is None: message("RTL data not found in memCache, " "excluding RTL from post.", mssgType='WARNING') return {} else: message([F"{'Getting RTL:': <20}", F"{time.time() - tic:.3f} s"], mssgType='TIMING') return post
def makeWEL(self, date_range, force_refresh=False): tic = time.time() if not force_refresh: dat = _cachedWELData(date_range) else: dat = WELData(timerange=date_range, mongo_connection=_cachedMongoConnect()) self.resample_T = (dat.timerange[1] - dat.timerange[0]) / self.resample_N self.dat_resample = dat.data.resample(self.resample_T).mean() message([F"{'WEL Data init:': <20}", F"{time.time() - tic:.2f} s"], tbl=self.mssg_tbl, mssgType='TIMING')
def _plots(self): tic = time.time() if self._sensor_groups is None: self._sensor_groups = [self.in_default] with st.spinner('Generating Plots'): plot = alt.vconcat( self.plotStatus().properties(width=self.def_width, height=self.def_height * self.stat_height_mod), self.plotMainMonitor(self._sensor_groups[0]).properties( width=self.def_width, height=self.def_height * self.pwr_height_mod), self.plotPowerStack([ 'solar_w', 'base_load_w', 'dehumidifier_w', 'geo_tot_w' ], ).properties(width=self.def_width, height=self.def_height * self.pwr_height_mod), # self.plotMainMonitor(self._sensor_groups[1]).properties( # width=self.def_width, # height=self.def_height * self.stat_height_mod # ), self.plotRollMean( ['COP', 'well_COP'], axis_label="COP").properties( width=self.def_width, height=self.def_height * self.stat_height_mod), self.plotRollMean(['T_diff_eff'], axis_label="Defficiency / W/°C", disp_raw=False, bottomPlot=True, height_mod=self.stat_height_mod).properties( width=self.def_width, height=self.def_height * self.stat_height_mod, ), spacing=self.def_spacing).resolve_scale(y='independent', color='independent') plot = plot.configure_axis( labelFontSize=self.label_font_size, titleFontSize=self.title_font_size, titlePadding=41, domain=False).configure_legend( labelFontSize=self.label_font_size, titleFontSize=self.title_font_size).configure_view( cornerRadius=2) message([F"{'Altair plot gen:': <20}", F"{time.time() - tic:.2f} s"], tbl=self.mssg_tbl, mssgType='TIMING') return [plot]
def accumulate(p): signals = pd.DataFrame() tic = time.time() for line in p.stdout: packet = processLine(line) signals = signals.append(packet, ignore_index=True) if time.time() - tic >= 29: break message("Found Signals:", mssgType='HEADER') [ print(F"{22 * ' '}{idx: <25}{value}", flush=True) for idx, value in signals.count().items() ] signals.drop_duplicates(inplace=True) return signals.mean().to_dict()
def processLine(line): line = json.loads(line) packet = {} try: id = F"{line['id']}_{line['message_type']}" except KeyError: id = str(line['id']) try: for quantity in id_to_name[id]['sensors']: sensor_name = (F"{id_to_name[id]['name']}_" F"{quantity_short[quantity]}") packet[sensor_name] = float(line[quantity]) return packet except KeyError: message([F"Unknown Sensor ID: {id}", F"\n{line}"], mssgType='WARNING')
def connectSense(): sn = Senseable() if platform.system() == 'Linux': path = "/home/ubuntu/WEL/WELPi/sense_info.txt" elif platform.system() == 'Darwin': path = "./sense_info.txt" sense_info = open(path).read().strip().split() try: sn.authenticate(*sense_info) except Exception as e: message("Error in authenticating with Sense, " F"excluding Sense from post. \n Error: {e}", mssgType='ERROR') sn.rate_limit = 10 message("Sense Connected", mssgType='ADMIN') return sn
def __init__(self, data_source='Pi', timerange=None, WEL_download=False, dl_db_path='../log_db/', mongo_connection=None, calc_cols=True): self._calc_cols = calc_cols self._data_source = data_source self._dl_db_path = dl_db_path self._now = dt.datetime.now().astimezone(self._to_tzone) if timerange is None: self.timerange = self.time_from_args() elif type(timerange[0]) is str: self.timerange = self.time_from_args(timerange) else: self.timerange = self.timeCondition(timerange) self.timerange = [ time.replace( tzinfo=self._to_tzone) if time.tzinfo is None else time for time in self.timerange ] if self._data_source == 'WEL': self.refresh_db() if WEL_download: dat_url = ("http://www.welserver.com/WEL1060/" + F"WEL_log_{self._now.year}" + F"_{self._now.month:02d}.xls") downfilepath = (self._dl_db_path + F"WEL_log_{self._now.year}" + F"_{self._now.month:02d}.xls") downfile = download(dat_url, downfilepath) if os.path.exists(downfilepath): move(downfile, downfilepath) self._stitch() elif self._data_source == 'Pi': if mongo_connection is None: self._mongo_db = mongoConnect() else: self._mongo_db = mongo_connection self._stitch() else: message("Valid data sources are 'Pi' or 'WEL'", mssgType='WARNING') quit()
def calc_stats(stp): last_rev_valve = np.round(stp.dat_resample['rev_valve_b'][-1] % 2) rev_valve_stat = {1: "Cooling", 0: "Heating"} N = len(stp.dat_resample) heat_2_count = (stp.dat_resample['heat_2_b'] % 2).sum() heat_1_count = (stp.dat_resample['heat_1_b'] % 2).sum() - heat_2_count # Heat 1 is ~80% of full power duty = 100 * ((0.8 * heat_1_count + heat_2_count) / N) try: house_w_avg = stp.dat_resample['house_w'].mean() / 1000 geo_w_avg = stp.dat_resample['geo_tot_w'].mean() / 1000 except KeyError: message("House power data not available", mssgType='WARNING', tbl=stp.mssg_tbl) house_w_avg = np.nan geo_w_avg = np.nan return [duty, house_w_avg, geo_w_avg, rev_valve_stat[last_rev_valve]]
def main(): message("\n Restarted ...", mssgType='ADMIN') mc = connectMemCache() time.sleep(5) with subprocess.Popen(rtl_cmd, stdout=subprocess.PIPE, text=True) as p: while True: signals = accumulate(p) mc_result = mc.set("rtl", signals) if not mc_result: message("RTL failed to cache", mssgType='ERROR') else: message("Succesful cache", mssgType='SUCCESS')
async def getWELData(ip): tic = time.time() url = "http://" + ip + ":5150/data.xml" post = {} local_now = (dt.datetime.now() .replace(microsecond=0) .replace(tzinfo=TO_TZONE)) sunrise = sun.sunrise(LOC.observer, date=local_now.date(), tzinfo=TO_TZONE).astimezone(DB_TZONE) sunset = sun.sunset(LOC.observer, date=local_now.date(), tzinfo=TO_TZONE).astimezone(DB_TZONE) post['daylight'] = ((local_now > sunrise) and (local_now < sunset)) * 1 try: response = requests.get(url) except ConnectionError: message("Error in connecting to WEL, waiting 10 sec then trying again", mssgType='WARNING') time.sleep(10) try: response = requests.get(url) except ConnectionError: message("Second error in connecting to WEL, " "excluding WEL from post.", mssgType='ERROR') return post response_data = xmltodict.parse(response.content)['Devices']['Device'] for item in response_data: try: post[item['@Name']] = float(item['@Value']) except ValueError: post[item['@Name']] = item['@Value'] del post['Date'] del post['Time'] message([F"{'Getting WEL:': <20}", F"{time.time() - tic:.1f} s"], mssgType='TIMING') return post
def _stitch(self): if self._data_source == 'WEL': load_new = False if self.data is not None: if ((self.data.dateandtime.iloc[0] > self.timerange[0]) or (self.data.dateandtime.iloc[-1] < self.timerange[1])): load_new = True else: load_new = True if load_new: num_months = ( (self.timerange[1].year - self.timerange[0].year) * 12 + self.timerange[1].month - self.timerange[0].month) monthlist = [ self.timerange[0] + relativedelta(months=x) for x in range(num_months + 1) ] loadedstring = [ F'{month.year}-{month.month}' for month in monthlist ] message(F'Loaded: {loadedstring}', mssgType='ADMIN') datalist = [ self.read_log(self._dl_db_path + F'WEL_log_{month.year}' + F'_{month.month:02d}.xls') for month in monthlist ] # print(datalist) self.data = pd.concat(datalist) # Shift power meter data by one sample for better alignment self.data.HP_W = self.data.HP_W.shift(-1) self.data.TAH_W = self.data.TAH_W.shift(-1) tmask = ((self.data.index > self.timerange[0]) & (self.data.index < self.timerange[1])) self.data = self.data[tmask] if self._data_source == 'Pi': query = { 'dateandtime': { '$gte': self.timerange[0].astimezone(self._db_tzone), '$lte': self.timerange[1].astimezone(self._db_tzone) } } # print(F"#DEBUG: query: {query}") self.data = pd.DataFrame(list(self._mongo_db.data.find(query))) if len(self.data) == 0: raise Exception("No data came back from mongo server.") self.data.index = self.data['dateandtime'] self.data = self.data.drop(columns=['dateandtime']) # print(self.data.columns) self.data = self.data.tz_localize(self._db_tzone) self.data = self.data.tz_convert(self._to_tzone) # print(F"#DEBUG: timerange from: {self.data.index[-1]}" # "to {self.data.index[0]}") # Shift power meter data by one sample for better alignment self.data.HP_W = self.data.HP_W.shift(-1) self.data.TAH_W = self.data.TAH_W.shift(-1) self.data = pd.concat((self.data, self._calced_cols(self.data)), axis=1)
def plotRollMean(self, vars, axis_label="COP Rolling Mean", height_mod=1, disp_raw=True, bottomPlot=False): source = self._getDataSubset(vars) # 3 * number of hours in displayed timerange rolling_frame = (3 * self.resample_N * (self.resample_N * self.resample_T).total_seconds() / 3600) rolling_frame = int( np.clip(rolling_frame, self.resample_N / 48, self.resample_N / 2)) try: rolling_source = pd.DataFrame( {'rolling_limit': source.dateandtime.iloc[-rolling_frame]}, index=[0]) except IndexError: message(["Rolling frame IndexError:", F"{-rolling_frame}"], tbl=self.mssg_tbl, mssgType='WARNING') rolling_source = pd.DataFrame( {'rolling_limit': source.dateandtime.iloc[-1]}, index=[0]) lines = alt.Chart(source).transform_window( rollmean='mean(value)', frame=[-rolling_frame, 0]).mark_line(interpolate='basis', strokeWidth=2).encode( x=alt.X( 'dateandtime:T', # scale=alt.Scale(domain=self.resize), axis=alt.Axis(grid=False, labels=False, ticks=False), title=None), y=alt.Y('rollmean:Q', scale=alt.Scale(zero=False), axis=alt.Axis(orient='right', grid=True), title=axis_label), color=alt.Color('label', legend=alt.Legend(title='Efficiencies', orient='left', offset=5))) window_line = alt.Chart(rolling_source).mark_rule( strokeDash=[5, 5], ).encode(x='rolling_limit:T', color=alt.ColorValue('gold ')) rule = self._createRules(lines, field='rollmean:Q', timetext=bottomPlot, timetextheightmod=height_mod) latest_text = self._createLatestText(lines, 'rollmean:Q') if disp_raw: raw_lines = alt.Chart(source).mark_line( interpolate='basis', strokeWidth=2, strokeDash=[1, 2], opacity=0.8, clip=True).encode(x=alt.X('dateandtime:T'), y=alt.Y('value:Q'), color='label') plot = alt.layer(self._plotNightAlt(), lines, raw_lines, rule, latest_text, window_line) return plot plot = alt.layer(self._plotNightAlt(), lines, rule, latest_text, window_line) return plot
def _plots(self): tic = time.time() if self._sensor_groups is None: self._sensor_groups = [self.wthr_default, self.in_humid_default] with st.spinner('Generating Plots'): plot = alt.vconcat( self.plotStatus().properties( width=self.def_width, height=self.def_height * self.stat_height_mod ), self.plotMainMonitor(self._sensor_groups[0]).properties( width=self.def_width, height=self.def_height * self.stat_height_mod ), self.plotMainMonitor(self.out_humid_default, axis_label="Outdoor Humidity / %", ).properties( width=self.def_width, height=self.def_height * self.stat_height_mod ), self.plotMainMonitor(self._sensor_groups[1], axis_label="Indoor Humidity / %", ).properties( width=self.def_width, height=self.def_height * self.stat_height_mod ), self.plotMainMonitor('rain_accum_R', axis_label='Rain / mm', ).properties( width=self.def_width, height=self.def_height * self.cop_height_mod ), self.plotMainMonitor('weather_station_W', axis_label='Wind / km/h', height_mod=self.cop_height_mod, bottomPlot=True ).properties( width=self.def_width, height=self.def_height * self.cop_height_mod ), spacing=self.def_spacing ).resolve_scale( y='independent', color='independent' ) plot = plot.configure_axis( labelFontSize=self.label_font_size, titleFontSize=self.title_font_size, titlePadding=41, domain=False ).configure_legend( labelFontSize=self.label_font_size, titleFontSize=self.title_font_size ).configure_view( cornerRadius=2 ) message([F"{'Altair plot gen:': <20}", F"{time.time() - tic:.2f} s"], tbl=self.mssg_tbl, mssgType='TIMING') return [plot]
def _serverStartup(): message("Server Started", mssgType='ADMIN')
def main(): _serverStartup() st.markdown(F""" <style> .css-dm3ece{{ width: {90}%; }} .reportview-container .main .block-container{{ max-width: {1500}px; padding-top: {5}px; padding-right: {90}px; padding-left: {10}px; padding-bottom: {5}px; }} .css-1u96g9d{{ display: none }} </style> <style class='css-1u96g9d'> details {{ display: none; }} </style> """, unsafe_allow_html=True) # -- sidebar -- st.sidebar.subheader("Monitor:") stats_containers = [st.sidebar.beta_container() for x in range(3)] which = st.sidebar.selectbox("Select Display Page", ['monit', 'pandw', 'wthr'], index=0, format_func=_whichFormatFunc) st.sidebar.subheader("Plot Options:") date_range = _date_select() sensor_container = st.sidebar.beta_container() max_samples = int( np.clip((date_range[1] - date_range[0]).total_seconds() / 30, 260, 1440)) resample_N = st.sidebar.slider("Number of Data Samples", min_value=10, max_value=max_samples, value=300, step=10) # -- main area -- st.header(F"{_whichFormatFunc(which)} Monitor") stp = _page_select(resample_N, date_range, sensor_container, which) stats = calc_stats(stp) stats_containers[0].markdown(F"System Duty: `{stats[0]:.1f} %`" F" `{stats[3]}`") stats_containers[1].markdown(F"House Mean Power Use: `{stats[1]:.2f} kW`") stats_containers[1].markdown(F"Geo Mean Power Use: `{stats[2]:.2f} kW | " F"{100 * stats[2] / stats[1]:.0f} %`") tic = time.time() for plot in stp.plots: st.altair_chart(plot) message([F"{'Altair plot disp:': <20}", F"{time.time() - tic:.2f} s"], tbl=stp.mssg_tbl, mssgType='TIMING') st.sidebar.markdown("[Github Project]" "(https://github.com/TristanShoemaker/WELPi)")
post_id = connects.db.insert_one(post).inserted_id message(F"Successful post @ UTC time: {utc_time}" F" | post_id: {post_id}", mssgType='SUCCESS') except DuplicateKeyError: message("Tried to insert duplicate key " F"{post['dateandtime'].strftime('%Y-%m-%d %H:%M:%S')}", mssgType='WARNING') async def main(interval): while True: then = time.time() post = await getWELData(WEL_IP) post.update(await getRtlData()) post.update(await getSenseData()) elapsed = time.time() - then await asyncio.sleep(interval - elapsed) post['dateandtime'] = (dt.datetime.utcnow() .replace(microsecond=0) .replace(tzinfo=DB_TZONE)) await send_post(post) if __name__ == "__main__": message("\n Restarted ...", mssgType='ADMIN') message("Mongo Connected", mssgType='ADMIN') connects = SourceConnects() asyncio.run(main(30))
def connectMemCache(): ip = MONGO_IP + ":11211" mc = Client([ip]) message("MemCache Connected", mssgType='ADMIN') return mc
async def getSenseData(): tic = time.time() try: connects.sn.update_realtime() except SenseAPITimeoutException: message("Sense API timeout, trying reconnect...", mssgType='WARNING') connects.sn = connectSense() try: connects.sn.update_realtime() except SenseAPITimeoutException: message("Second Sense API timeout, " "excluding Sense from post.", mssgType='ERROR') return {} except socket.timeout as e: message("Sense offline, excluding Sense from post." F"\n Error: {e}", mssgType='ERROR') return {} except socket.timeout as e: message("Sense offline, excluding Sense from post." F"\n Error: {e}", mssgType='ERROR') return {} sense_post = connects.sn.get_realtime() post = {} post['solar_w'] = sense_post['solar_w'] post['house_w'] = sense_post['w'] try: post['dehumidifier_w'] = [device for device in sense_post['devices'] if device['name'] == 'Dehumidifier '][0]['w'] except IndexError: post['dehumidifier_w'] = 0 message("Dehumidifier not found in sense.", mssgType='WARNING') try: post['furnace_w'] = [device for device in sense_post['devices'] if device['name'] == 'Furnace'][0]['w'] except IndexError: post['furnace_w'] = 0 message("Furnace not found in sense.", mssgType='WARNING') try: post['barn_pump_w'] = [device for device in sense_post['devices'] if device['name'] == 'Barn pump'][0]['w'] except IndexError: post['barn_pump_w'] = 0 message("Barn pump not found in sense.", mssgType='WARNING') try: post['TES_sense_w'] = [device for device in sense_post['devices'] if device['name'] == 'TES'][0]['w'] except IndexError: post['TES_sense_w'] = 0 message("TES not found in sense.", mssgType='WARNING') message([F"{'Getting Sense:': <20}", F"{time.time() - tic:.1f} s"], mssgType='TIMING') return post