def plot_live_minutes(self):
"""returns plotly graph with minute information up to 10 hours back.
example:
:plot_live_minutes()
"""
date = Runtime.td('date_live')
time = Runtime.td('time_live')
e = read_youless().read_minutes()
live = e['watts'][-1]
# maxusage = max(e['watts']) # variable to calculate max height of figure
graphtitle = Youless.lang('liveminutegraphtitle').format(
live, date, time)
fig = px.line(
x=e['time'],
y=e['watts'],
title=graphtitle,
# width=1300, # static figure width
# height=(maxusage + (maxusage / 3) # calculate max figure height
height=500, # static figure height
labels={
"x": Youless.lang('T'), # x-axis naming
"y": Youless.lang('W') # y-axis naming
})
logger.debug("Plotting live minutes %s %s" % (date, time))
return fig
def read_days(self, etype):
"""read all Electricity or Gas hour values from Youless 120 with a maximum history of 70 days back (youless max).
returns a list with tuples of data.
example:
read_days(etype)
args:
:etype is 'E' or 'G' for Electricity or Gas
"""
self.__urltype = self.__ele if etype == 'E' and etype != (
'G', 'S') else self.__gas if etype == 'G' else self.__snul
self.max_page = Youless.web("maxDayPage")
self.min_page = Youless.web("minDayPage")
self.max_hour = Youless.web("maxHour")
self.min_hour = Youless.web("minHour")
self.counter = self.max_page
self.return_list = []
while (self.counter >= self.min_page):
self.api = requests.get(self.__url + self.__urltype + self.__json +
self.__day + str(self.counter))
readapi = self.api.json()
json_date = datetime.datetime.strptime(readapi['tm'],
'%Y-%m-%dT%H:%M:%S')
date = json_date.date()
year = json_date.date().strftime('%Y')
week_no = json_date.date().strftime('%W')
month_no = json_date.date().strftime('%m')
month_name = json_date.date().strftime('%B')
month_day = json_date.date().strftime('%d')
year_day = json_date.date().strftime('%j')
raw_values = readapi['val']
lst = []
self.hours = self.min_hour
for y, s in enumerate(raw_values):
try:
raw_values[y] = s.strip()
raw_values[y] = float(s.replace(',', '.'))
except AttributeError:
pass
except IndexError:
break
finally:
lst.append(raw_values[y])
if raw_values[y] is None:
lst.pop()
strlst = '{}'.format(lst)
task = (str(date), int(year), int(week_no), int(month_no),
month_name, int(month_day), int(year_day), strlst)
self.return_list.append(task)
lst.clear()
self.counter -= 1
return self.return_list
def read_months(self, etype):
"""Function to retrieve days per month from Youless 120 up to 11 months back for Electricity and Gas.
returns a list with tuples of data.
eaxmple:
read_months(etype)
args:
:etype is 'E' or 'G' for Electricity or Gas
"""
self.__urltype = self.__ele if etype == 'E' and etype != (
'G', 'S') else self.__gas if etype == 'G' else self.__snul
self.__type = self.__kwh if etype == 'E' and etype != 'G' else self.__m3
self.max_page = Youless.web("maxMonthPage")
self.min_page = Youless.web("minMonthPage")
self.counter = self.max_page
self.return_list = []
while (self.counter >= self.min_page):
self.api = requests.get(self.__url + self.__urltype + self.__json +
self.__month + str(self.counter))
readapi = self.api.json()
json_date = datetime.datetime.strptime(readapi['tm'],
'%Y-%m-%dT%H:%M:%S')
date = json_date.date()
year = json_date.date().strftime('%Y')
month_no = json_date.date().strftime('%m')
month_name = json_date.date().strftime('%B')
raw_values = readapi['val']
lst = []
for y, s in enumerate(raw_values):
try:
raw_values[y] = s.strip()
raw_values[y] = float(s.replace(',', '.'))
except AttributeError:
pass
except IndexError:
break
finally:
lst.append(raw_values[y])
if raw_values[y] is None:
lst.pop()
strlst = '{}'.format(lst)
task = (str(date), int(year), int(month_no), month_name, strlst)
self.return_list.append(task)
lst.clear()
self.counter -= 1
return self.return_list
def plot_month_day(self, year, month, etype):
"""Reads from table yeardays_X and returns figure.
plot one month of the year into a graph with daily totals.
examples:
:plot_month_day(2021, 1, 'E')
:plot_month_day(2021, 'January', 'G')
args:
:year as integer
:month as integer or string
:etype as string ('E' for Electricity, 'G' for Gas)
"""
self.year = year
self.month = month
self.etype = etype
self.table = Youless.sql("dbtables")[self.etype][0]
self.columns = [
Youless.lang("D"),
Youless.lang("KH"),
Youless.lang("KWH")
] if self.etype == 'E' else [
Youless.lang("D"),
Youless.lang("KM"),
Youless.lang("M3")
] if self.etype == 'G' else None
if (type(self.month) == str):
self.mN = datetime.datetime.strptime(self.month, '%B')
self.month = int(self.mN.date().strftime('%m'))
self.monthName = datetime.date(1900, self.month, 1).strftime('%B')
lst = self.data.retrieve_month(
self.year, self.month, self.table) # retrieve data from database
if lst == 0:
logger.error("No Data")
return 0
data = {}
high = max(lst[2]) + max(lst[2]) / 3
totalUsage = 0
for n, elem in enumerate(lst[2]):
data[n +
1] = elem # Add kWh values to dictionary with the day as key
totalUsage += elem
df = pd.DataFrame(data.items(), columns=self.columns[:2])
fig = px.bar(
df,
x=df[self.columns[0]],
y=df[self.columns[1]],
range_y=(0, high),
title=(Youless.lang('yearmonthtitle') %
(self.monthName, self.year, self.columns[1], totalUsage,
self.columns[2])),
)
logger.debug("Plotting month %d of year %d" % (self.month, self.year))
return fig
def plot_year_month(self, year, etype):
"""Reads from table yeardays_X and returns figure.
plot the year with month totals into a graph.
example:
:plot_year(2021, 'E')
args:
:year as integer
:etype as string ('E' for Electricity, 'G' for Gas)
"""
self.year = year
self.etype = etype
self.table = Youless.sql("dbtables")[self.etype][0]
self.columns = [
Youless.lang("M"),
Youless.lang("KH"),
Youless.lang("KWH")
] if self.etype == 'E' else [
Youless.lang("M"),
Youless.lang("KM"),
Youless.lang("M3")
] if self.etype == 'G' else None
self.type = 1 # equals year with month totals
lst = self.data.retrieve_year(self.year, self.type, self.table)
if lst == 0:
logger.error("No Data")
return 0
data = {}
months = {}
totalUsage = 0.0
maxLst = []
for n, elem in enumerate(lst):
data[lst[n][1]] = lst[n][3]
months[lst[n][1]] = datetime.date(1900, int(lst[n][1]),
1).strftime('%B')
totalUsage += lst[n][3]
maxLst.append(lst[n][3])
high = max(maxLst) + max(maxLst) / 3
df = pd.DataFrame(data.items(), columns=(self.columns[:2]))
fig = px.bar(
df,
x=df[self.columns[0]],
y=df[self.columns[1]],
range_y=(0, high),
title=(Youless.lang('yeartitle') %
(self.year, self.columns[1], totalUsage, self.columns[2])),
)
fig.update_layout(xaxis=dict(tickmode='array',
tickvals=list(months.keys()),
ticktext=list(months.values())))
logger.debug("Plotting year %d" % (self.year))
return fig
def main():
if not youless_sql.isSqlite3Db(path):
logger.warning("Database {} non existant, creating database".format(Settings.dbname))
for v in Youless.sql("dbtables").values():
for i in v:
con = sl.connect(path)
with con:
con.execute(Youless.sql("queries")[i])
logger.warning("Table {} created".format(i))
else:
logger.warning("Database {} exists, checking and creating tables".format(Settings.dbname))
for v in Youless.sql("dbtables").values():
for i in v:
if (youless_sql.is_table_exists(i)):
logger.warning("Table {} already exists, doing nothing".format(i))
else:
logger.warning("Table {} does not exist, creating".format(i))
con = sl.connect(path)
with con: # CREATE TABLE
con.execute(Youless.sql("queries")[i])
def read_minutes(self):
"""read per minute energy data from Youless 120, data is always 1 minute behind live.
returns a dictionary with 2 lists consisting of time values and watts values.
example return:
{'time': ['value', 'value'], 'watts': ['value', 'value']}
"""
self.maxPage = Youless.web("maxMinutePage")
self.minPage = Youless.web("minMinutePage")
self.maxMinute = Youless.web("maxMinute")
self.minMinute = Youless.web("minMinute")
self.counter = self.maxPage
data = {}
time = []
watts = []
while (self.counter >= self.minPage):
self.api = requests.get(self.__url + self.__ele + self.__json +
self.__D + str(self.counter)).json()
self.date = datetime.datetime.strptime(self.api['tm'],
'%Y-%m-%dT%H:%M:%S')
i = self.minMinute
while (i <= self.maxMinute):
h = self.date.hour
m = self.date.minute + i
if (m >= 60):
m -= 60
h += 1
self.time = '%02d:%02d' % (h, m) # str(self.date.time())[0:5]
self.watt = int(self.api['val'][i])
time.append(self.time)
watts.append(self.watt)
logger.debug('Time: {} Usage: {} Watt'.format(
self.time, self.watt))
i += 1
self.counter -= 1
data['time'] = time
data['watts'] = watts
logger.debug(data)
return data
def plot_year_day(self, year, etype):
"""Reads from table yeardays_X and returns figure.
plot the year with daily totals into a graph.
example:
:plot_year_day(2021, 'E')
args:
:year as integer
:etype as string ('E' for Electricity, 'G' for Gas)
"""
self.year = year
self.etype = etype
self.table = Youless.sql("dbtables")[self.etype][0]
self.columns = [
Youless.lang("D"),
Youless.lang("KH"),
Youless.lang("KWH")
] if self.etype == 'E' else [
Youless.lang("D"),
Youless.lang("KM"),
Youless.lang("M3")
] if self.etype == 'G' else None
self.type = 2 # equals year with month totals
lst = self.data.retrieve_year(self.year, self.type, self.table)
if lst == 0:
logger.error("No Data")
return 0
data = {}
totalUsage = 0.0
maxLst = []
for n, elem in enumerate(lst):
self.month = int(lst[n][1])
for y, value in enumerate(lst[n][3]):
tempkey = ("%d-%d-%d" % (self.year, self.month, y + 1)
) # year-month-day
data[tempkey] = value
totalUsage += value
maxLst.append(value)
high = max(maxLst) + max(maxLst) / 3
df = pd.DataFrame(data.items(), columns=self.columns[:2])
fig = px.bar(
df,
x=df[self.columns[0]],
y=df[self.columns[1]],
range_y=(0, high),
title=(Youless.lang('yeardaytitle') %
(self.year, self.columns[1], totalUsage, self.columns[2])),
)
logger.debug("Plotting year %d" % (self.year))
return fig
def is_table_exists(table):
"""youless_sql.is_table_exists(table_name) -> True if exists, False if not"""
con = sl.connect(path)
with con:
check = Youless.sql("queries")["table_exist"]
table = (table,)
run = con.execute(check, table)
e = int(run.fetchone()[0]) # returns 1 if exists
rtn = True if (e != 0) else False # 1 == True else False
logger.debug("table {} existence is {}".format(table, rtn))
return rtn
def retrieve_year(self, year, totals, table):
"""Retrieve data from table yeardays_X and return list with items.
Retrieves available data for given year per month and day in values.
Returns either month totals or day totals for the entire year.
example:
retrieve_year(2021, 2, 'yeardays_g')
args:
:year as integer
:totals as integer (1 is month totals, 2 is day per month totals)
:table as string
"""
self.year = year
self.totals = totals
self.table = table
if (self.totals != 1 and self.totals != 2):
logger.error("error: type can only be 1 or 2")
return "error: type can only be 1 or 2"
logger.debug("Starting year retrieval %d" % (self.year))
self.cur = self.conn.cursor()
with self.conn:
self.query = (Youless.sql("queries")["so_yeardays"] %
(self.table, str(self.year)))
data = self.conn.execute(self.query)
rows = len(data.fetchall())
logger.debug("Rows retrieved: %d" % rows)
if (rows >= 1):
data = self.conn.execute(self.query)
self.lst = []
for row in data:
self.values = row[4] # x is string representation of list
self.values = ast.literal_eval(
self.values) # convert x to real list
total = 0
for v, n in enumerate(self.values):
self.values[v] = float(self.values[v])
total += float(self.values[v])
if (self.totals == 1): # month totals
self.tmplst = [[row[1], row[2], row[3], int(total)]]
elif (self.totals == 2): # day totals
self.tmplst = [[row[1], row[2], row[3], self.values]]
self.lst.extend(self.tmplst)
self.tmplst.clear()
else:
return 0
self.conn.close()
logger.debug("Retrieved list:")
logger.debug(self.lst)
return self.lst
def plot_live(self):
"""returns plotly graph with live information.
example:
:plot_live()
"""
date = Runtime.td('date_live')
time = Runtime.td('secs_live')
if (len(livedict['timelst']) == 20):
livedict['timelst'].pop(0)
livedict['timelst'].append(time)
lvd = read_youless().read_live()
live = lvd['pwr']
if (len(livedict['wattlst']) == 20):
livedict['wattlst'].pop(0)
livedict['wattlst'].append(live)
total = lvd['cnt']
graphtitle = Youless.lang('livegraphtitle').format(
live, date, time, total, Youless.lang('KWH'))
maxusage = max(livedict['wattlst'])
fig = px.line(
x=livedict['timelst'],
y=livedict['wattlst'],
title=graphtitle,
range_y=(0, (maxusage + (maxusage / 3))),
width=500, # static figure widt
# height = (maxusage + (maxusage / 3) # calculate max figure height
height=500, # static figure height
labels={
"x": Youless.lang('T'), # x-axis naming
"y": Youless.lang('W') # y-axis naming
})
logger.debug("Plotting live %s %s" % (date, time))
return fig
def retrieve_day(self, year, month, day, table):
"""Retrieve data from table dayhours_X and return list with items.
Retrieves day data for given year, month and day in a list with hours and values.
example:
retrieve_day(2021, 1, 19, 'dayhours_e')
args:
:year as integer
:month as integer
:day as integer
:table as string
"""
self.year = year
self.month = month
self.day = day
self.table = table
logger.debug("Starting day retrieval %d %d %d from table %s" %
(self.year, self.month, self.day, self.table))
self.cur = self.conn.cursor()
with self.conn:
self.query = (
Youless.sql("queries")["s_dayhours"] %
(self.table, str(self.year), str(self.month), str(self.day)))
data = self.cur.execute(self.query)
rows = len(self.cur.fetchall())
logger.debug("Rows retrieved: %d" % rows)
if (rows >= 1):
data = self.cur.execute(
self.query
) # execute query again because fetchall() mangles the data to a list of tuples with strings
for row in data:
self.values = row[7] # x is string representation of list
self.values = ast.literal_eval(
self.values) # convert x to real list
for v, n in enumerate(self.values):
self.values[v] = float(self.values[v])
else:
return 0
self.conn.close()
self.lst = [self.year, self.month, self.day, self.values]
logger.debug("Retrieved list:")
logger.debug(self.lst)
return self.lst
def parse_months(self, etype, data):
"""Function to parse the information retrieved by retrieve_months
example:
parse_months(etype, data)
args:
:etype is 'E' or 'G' for Electricity or Gas
:data is a list with tuples retrieved by retrieve_months
"""
self.__table = Youless.sql("dbtables")[etype][0]
self.__type = self.__kwh if etype == 'E' and etype != 'G' else self.__m3
logger.debug("Connected to table {}".format(self.__table))
for item in data:
write_data().write_yeardays(self.conn, item, self.__table,
self.__type)
self.conn.close()
logger.debug("Database connection closed...")
def parse_tenminutes(self, etype, data):
"""Function to parse the information retrieved by read_ten_minutes
example:
parse_tenminutes(etype, data)
args:
:etype is 'E' or 'G' for Electricity or Gas
:data is a list with tuples retrieved by read_ten_minutes
"""
self.__table = Youless.sql("dbtables")[etype][2]
self.__type = self.__watt if etype == 'E' and etype != 'G' else self.__ltr
logger.debug("Connected to table {}".format(self.__table))
for key in data.keys():
write_data().write_tenminutes(self.conn, (key, data[key]),
self.__table, self.__type)
self.conn.close()
logger.debug("Database connection closed...")
def check_existence(self, **kwargs):
"""function to check if item exists in sqlite database.\n
returns 1 if row exists and 0 if not.
:table= str
:column= str
:item= str/int (int gets transformed to str in process)
"""
if self.conn is None:
return None
self.table = kwargs.get('table')
self.column = kwargs.get('column')
self.item = kwargs.get('item')
self.q = Youless.sql('queries')['if_exists']
self.query = self.q.format(self.table, self.column, self.item)
self.cur = self.conn.cursor()
with self.conn:
self.data = self.cur.execute(self.query)
self.outcome = [row for row in self.data]
self.conn.close()
return int(self.outcome[0][0])
def retrieve_month(self, year, month, table):
"""Retrieve data from table yeardays_X and return list.
Retrieves monthdata for given month of given year in a list with days and values
example:
retrieve_month(2021, 1, 'yeardays_e')
args:
:year as integer
:month as integer
:table as string
"""
self.year = year
self.month = month
self.table = table
logger.debug("Starting month retrieval %d %d from table %s" %
(self.year, self.month, self.table))
self.cur = self.conn.cursor()
with self.conn:
self.query = (Youless.sql("queries")["s_yeardays"] %
(self.table, str(self.year), str(self.month)))
data = self.conn.execute(self.query)
rows = len(data.fetchall())
logger.debug("Rows retrieved: %d" % rows)
if (rows >= 1):
data = self.conn.execute(self.query)
for row in data:
self.values = row[4] # x is string representation of list
self.values = ast.literal_eval(
self.values) # convert x to real list
for v, n in enumerate(self.values):
self.values[v] = float(self.values[v])
else:
return 0
self.conn.close()
self.lst = [self.year, self.month, self.values]
logger.debug("Retrieved list:")
logger.debug(self.lst)
return self.lst
def get_item(self, **kwargs):
"""returns item from database corresponding to the given query and kwargs
examples:
retrieve_custom_data().get_item(query='select_one_and',
select=('*'),
table='dayhours_e',
id='year',
item='2021',
id2='yearday',
item2='44')
retrieve_custom_data().get_item(query='select_one',
select='yearday,watt',
table='dayhours_e',
id='year',
item='2021')
kwargs:
:query= str
:select= str
:table= str
:id= str
:item= str
:id2= str (optional)
:item2= str (optional)
"""
if self.conn is None:
return None
self.q = Youless.sql('queries')[kwargs.get('query')]
self.query = self.q % tuple(kwargs.values())[1:]
self.cur = self.conn.cursor()
with self.conn:
self.data = self.cur.execute(self.query)
self.outcome = [row for row in self.data]
self.conn.close()
return self.outcome
def __init__(self) -> None:
self.elist = Youless.sql('energytypes')[
'list'] # retrieve energy types
"""
import sys
import datetime
# web
import requests
# Youless setup
from LS120 import Youless
# initialize logging
import logging
logger = logging.getLogger(__name__)
logger.debug("read_youless.py started")
sys.stdin.reconfigure(encoding=Youless.web("ENCODING")) # set encoding
sys.stdout.reconfigure(encoding=Youless.web("ENCODING")) # set encoding
class read_youless:
"""class to read data from the Youless LS120"""
def __init__(self) -> None:
self.__headers = Youless.web("HEADERS") # acceptable html headers
self.__url = Youless.web("URL") # base url
self.__ele = Youless.web("ELE")
self.__gas = Youless.web("GAS")
self.__snul = Youless.web("S0")
self.__json = Youless.web("JSON")
self.__month = Youless.web("M")
self.__day = Youless.web("W")
self.__stats = Youless.web("STATS")
def plot_day_hour(self, year, month, day, etype):
"""Reads from table dayhours_X and return figure.
Plot one day of the year into a graph with hourly totals.
examples:
:plot_day_hour(2021, 1, 19, 'E')
:plot_day_hour(2021, January, 19, 'G')
args:
:year as integer
:month as integer or string
:day as integer
:etype as string ('E' for Electricity, 'G' for Gas)
"""
self.year = year
self.month = month
self.day = day
self.etype = etype
self.table = Youless.sql("dbtables")[self.etype][1]
self.columns = [
Youless.lang("U"),
Youless.lang("W"),
Youless.lang("KWH")
] if self.etype == 'E' else [
Youless.lang("U"),
Youless.lang("L"),
Youless.lang("M3")
] if self.etype == 'G' else None
if (type(self.month) == str):
self.mN = datetime.datetime.strptime(self.month, '%B')
self.month = int(self.mN.date().strftime('%m'))
self.monthName = datetime.date(1900, self.month, 1).strftime('%B')
lst = self.data.retrieve_day(self.year, self.month, self.day,
self.table) # retrieve data from database
if lst == 0:
logger.error("No Data")
return 0
data = {}
high = max(lst[3]) + max(lst[3]) / 3
totalWatt = 0
for n, elem in enumerate(lst[3]):
data[
n +
1] = elem # Add watt values to dictionary with the hour as key
totalWatt += elem
df = pd.DataFrame(data.items(), columns=self.columns[:2])
fig = px.bar(
df,
x=df[self.columns[0]],
y=df[self.columns[1]],
range_y=(0, high),
title=(Youless.lang('dayhourtitle') %
(self.monthName, self.day, self.year, self.columns[1],
float(totalWatt / 1000), self.columns[2])),
)
logger.debug("Plotting month %d day %d of year %d from table %s" %
(self.month, self.day, self.year, self.table))
return fig
def create_dash_page(self, ip, port):
"""Plot one or more graphs with dash and plotly for testing purposes"""
self.ip = ip
self.port = port
# define the app
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
# define tables to show on page
live_table_e = web_elements.table_header(Youless.lang('LIVE_TEN_E'),
"e_tenMinuteGraph")
live_table_g = web_elements.table_header(Youless.lang('LIVE_TEN_G'),
"g_tenMinuteGraph")
dual_energy_table = web_elements.dual_table(Youless.lang('TOYE'),
"e_today", "e_yesterday")
dual_gas_table = web_elements.dual_table(Youless.lang('TOYE'),
"g_today", "g_yesterday")
# define layout with defined tables
app.layout = web_elements.layout_with_intervals(
live_table_e, live_table_g, dual_energy_table, dual_gas_table)
# start callback with update interval
@app.callback( # update every minute (60 seconds)
Output('e_tenMinuteGraph',
'figure'), # electricity ten minute graph output
Output('g_tenMinuteGraph',
'figure'), # gas ten minute graph output
Input('interval-component-quarterhour',
'n_intervals') # interval input
)
# the callback function
def update_tenminutegraph_live(n):
fig1 = plot_live().plot_live_ten_minutes(
etype='E',
start=datetime.datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0) -
datetime.timedelta(days=2)) # 2 days back from today)
fig2 = plot_live().plot_live_ten_minutes(
etype='G',
start=datetime.datetime(2021, 7, 22)) # plot live minute graph
return fig1, fig2
# start the callback with update interval
@app.callback([
Output('e_today', 'figure'),
Output('e_yesterday', 'figure'),
Output('g_today', 'figure'),
Output('g_yesterday', 'figure')
], [Input('interval-component-halfhour', 'n_intervals')])
def multi_output(n): # the callback function
fig1 = plot_dbdata().plot_day_hour(Runtime.td('year_now'),
Runtime.td('month_now'),
Runtime.td('day_now'),
self.elist[0])
fig2 = plot_dbdata().plot_day_hour(Runtime.td('year_now'),
Runtime.td('month_now'),
Runtime.td('day_yesterday'),
self.elist[0])
fig3 = plot_dbdata().plot_day_hour(Runtime.td('year_now'),
Runtime.td('month_now'),
Runtime.td('day_now'),
self.elist[1])
fig4 = plot_dbdata().plot_day_hour(Runtime.td('year_now'),
Runtime.td('month_now'),
Runtime.td('day_yesterday'),
self.elist[1])
return fig1, fig2, fig3, fig4
# run the webserver
app.run_server(debug=Dash_Settings.DASHDEBUG, host=ip, port=port)
def create_dash_page(self, ip, port, debug, reloader):
"""Plot all information with plotly and dash"""
self.ip = ip
self.port = port
self.debug = debug
self.reloader = reloader
# define the app
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
# define tables to show on page
live_table = web_elements.dual_table(Youless.lang('LIVE'), "liveGraph",
"minuteGraph")
live_ten_minutes_e_table = web_elements.table_header(
Youless.lang('LIVE_TEN_E'), "e_tenMinuteGraph")
live_ten_minutes_g_table = web_elements.table_header(
Youless.lang('LIVE_TEN_G'), "g_tenMinuteGraph")
today_yesterday_table = web_elements.quad_table_header(
Youless.lang('TOYE'), "e_today", "e_yesterday", "g_today",
"g_yesterday")
thismonth_lastmonth_table = web_elements.quad_table_header(
Youless.lang('CMLM'), "e_currmonth", "e_lastmonth", "g_currmonth",
"g_lastmonth")
thisyear_lastyear_month_table = web_elements.quad_table_header(
Youless.lang('TYLYM'), "e_year1", "e_year2", "g_year1", "g_year2")
thisyear_lastyear_day_table = web_elements.quad_table_header(
Youless.lang('TYLYD'), "e_fullyear1", "e_fullyear2", "g_fullyear1",
"g_fullyear2")
# define layout with defined tables
app.layout = web_elements.layout_with_intervals(
live_table, live_ten_minutes_e_table, live_ten_minutes_g_table,
today_yesterday_table, thismonth_lastmonth_table,
thisyear_lastyear_month_table, thisyear_lastyear_day_table)
# start callback with update interval
@app.callback( # update every 15 minutes (quarter hour)
Output('updatedb', 'children'), # database update output
Input('interval-component-quarterhour',
'n_intervals') # interval input
)
# the callback function
def quarter_hour(n):
db.main() # update database
return
# start callback with update interval
@app.callback( # live update (every 5 seconds)
Output('liveGraph', 'figure'), # live graph output
Input('interval-component-live', 'n_intervals') # interval input
)
# the callback function
def update_graph_live(n):
fig = plot_live().plot_live() # plot live graph
return fig
# start callback with update interval
@app.callback( # update every minute (60 seconds)
Output('minuteGraph', 'figure'), # minute graph output
Input('interval-component-minute', 'n_intervals') # interval input
)
# the callback function
def update_minutegraph_live(n):
fig = plot_live().plot_live_minutes() # plot live minute graph
return fig
# start callback with update interval
@app.callback( # update every 10 minutes (600 seconds)
[
Output('e_tenMinuteGraph', 'figure'),
Output('g_tenMinuteGraph', 'figure'),
], [Input('interval-component-tenminutes', 'n_intervals')])
def update_tenminutegraph(n):
fig1 = plot_live().plot_live_ten_minutes(
etype='E',
start=datetime.datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0) -
datetime.timedelta(days=2)) # 2 days back from today)
fig2 = plot_live().plot_live_ten_minutes(
etype='G',
start=datetime.datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0) -
datetime.timedelta(days=2)) # 2 days back from today)
return fig1, fig2
# start callback with update interval
@app.callback( # update every 30 minutes (half hour)
Output('e_today', 'figure'), # energy today output
Output('g_today', 'figure'), # gas today output
Input('interval-component-halfhour',
'n_intervals') # interval input
)
def update_halfhour(n): # the callback function
fig1 = plot_dbdata().plot_day_hour(
Runtime.td('year_now'), Runtime.td('month_now'),
Runtime.td('day_now'), self.elist[0]) # plot energy today
fig2 = plot_dbdata().plot_day_hour(Runtime.td('year_now'),
Runtime.td('month_now'),
Runtime.td('day_now'),
self.elist[1]) # plot gas today
return fig1, fig2
# start callback with update interval
@app.callback( # update every 360 minutes (six hours)
[
Output('e_yesterday', 'figure'), # energy yesterday
Output('g_yesterday', 'figure'), # gas yesterday
Output('e_currmonth', 'figure'), # energy current month
Output('e_lastmonth', 'figure'), # energy last month
Output('g_currmonth', 'figure'), # gas current month
Output('g_lastmonth', 'figure'), # gas last month
Output('e_year1', 'figure'), # energy current year per month
Output('e_year2', 'figure'), # energy last year per month
Output('g_year1', 'figure'), # gas current year per month
Output('g_year2', 'figure'), # gas last year per month
Output('e_fullyear1', 'figure'), # energy current year per day
Output('e_fullyear2', 'figure'), # energy last year per day
Output('g_fullyear1', 'figure'), # gas current year per day
Output('g_fullyear2', 'figure') # gas last year per day
],
[
Input('interval-component-sixhours',
'n_intervals') # interval input
])
def update_sixhours(n): # the callback function
fig1 = plot_dbdata().plot_day_hour(
Runtime.td('year_now'), Runtime.td('month_now'),
Runtime.td('day_yesterday'),
self.elist[0]) # plot energy yesterday
fig2 = plot_dbdata().plot_day_hour(
Runtime.td('year_now'), Runtime.td('month_now'),
Runtime.td('day_yesterday'),
self.elist[1]) # plot gas yesterday
fig3 = plot_dbdata().plot_month_day(
Runtime.td('year_now'), Runtime.td('month_now'),
self.elist[0]) # plot energy current month
fig4 = plot_dbdata().plot_month_day(
Runtime.td('year_now'), Runtime.td('last_month'),
self.elist[0]) # plot energy last month
fig5 = plot_dbdata().plot_month_day(
Runtime.td('year_now'), Runtime.td('month_now'),
self.elist[1]) # plot gas current month
fig6 = plot_dbdata().plot_month_day(
Runtime.td('year_now'), Runtime.td('last_month'),
self.elist[1]) # plot gas last month
fig7 = plot_dbdata().plot_year_month(
Runtime.td('year_now'),
self.elist[0]) # plot energy current year per month
fig8 = plot_dbdata().plot_year_month(
Runtime.td('last_year'),
self.elist[0]) # plot energy last year per month
fig9 = plot_dbdata().plot_year_month(
Runtime.td('year_now'),
self.elist[1]) # plot gas current year per month
fig10 = plot_dbdata().plot_year_month(
Runtime.td('last_year'),
self.elist[1]) # plot gas last year per month
fig11 = plot_dbdata().plot_year_day(
Runtime.td('year_now'),
self.elist[0]) # plot energy current year per day
fig12 = plot_dbdata().plot_year_day(
Runtime.td('last_year'),
self.elist[0]) # plot energy last year per day
fig13 = plot_dbdata().plot_year_day(
Runtime.td('year_now'),
self.elist[1]) # plot gas current year per day
fig14 = plot_dbdata().plot_year_day(
Runtime.td('last_year'),
self.elist[1]) # plot gas last year per day
return fig1, fig2, fig3, fig4, fig5, fig6, fig7, fig8, fig9, fig10, fig11, fig12, fig13, fig14
# Display json structure in frame below graph
# @app.callback(
# Output("structure", "children"),
# [Input("graph", "figure")])
# def display_structure(fig_json):
# return json.dumps(fig_json, indent=2)
# run the webserver
app.run_server(debug=self.debug,
host=self.ip,
port=self.port,
use_reloader=self.reloader)
def plot_hours(self, *args):
"""Reads from table dayhours_X and return figure.
Plot hours from given starthour up to endhour.
Plot spans a max of 2 days with a minimum of 1 hour and a maximum of 24 hours.
Minimum arguments is 5, maximum is 7 in this order:
:energytype: 'E', year: 2021, month: 3, startday: 2, starthour: 5, endday: 3, endhour: 6
:energytype: 'E', year: 2021, month: 3, startday: 2, starthour: 5, endhour: 6
:energytype: 'E', year: 2021, month: 3, startday: 2, starthour: 5
plot_hours(arguments) examples:
:plot_hours("E", 2021, 3, 2, 12, 3, 11)
:plot_hours("E", 2021, 3, 2, 12, 11)
:plot_hours("E", 2021, 3, 2, 12)
"""
if (len(args) > 7):
return logger.warning("Maximum arguments of 7 exceeded")
elif (len(args) < 5):
return logger.warning("Minimum arguments is 5")
self.etype = args[0]
self.table = Youless.sql("dbtables")[self.etype][1]
self.year = args[1]
self.month = args[2]
self.startday = args[3]
self.starthour = args[4]
if (len(args) == 7):
self.endday = args[5]
self.endhour = args[6]
lst = self.data.retrieve_hours(
self.table, self.year, self.month, self.startday,
self.starthour, self.endday,
self.endhour) # retrieve data from database
elif (len(args) == 6):
self.endhour = args[5]
lst = self.data.retrieve_hours(
self.table, self.year, self.month, self.startday,
self.starthour, self.endhour) # retrieve data from database
else:
lst = self.data.retrieve_hours(
self.table, self.year, self.month, self.startday,
self.starthour) # retrieve data from database
if lst == 0:
logger.error("No Data")
return 0
self.columns = [
Youless.lang("U"),
Youless.lang("W"),
Youless.lang("KWH")
] if self.etype == 'E' else [
Youless.lang("U"),
Youless.lang("L"),
Youless.lang("M3")
] if self.etype == 'G' else None
self.hours = lst[-1]
data = {}
highlst = []
total = 0
for n, v in enumerate(self.hours):
data[self.hours[n][0]] = int(self.hours[n][1])
highlst.append(self.hours[n][1])
total += int(self.hours[n][1])
high = int(max(highlst) + max(highlst) / 3)
df = pd.DataFrame(data.items(), columns=self.columns[:2])
logger.debug(df)
self.month = datetime.date(1900, int(self.month), 1).strftime('%B')
self.title = (Youless.lang('dayhourtitle') %
(self.month, self.startday, self.year, self.columns[1],
int(total / 1000), self.columns[2]) if
('endday'
not in dir(self)) else Youless.lang('customhourtitle') %
(self.startday, self.endday, self.month, self.year,
self.columns[1], int(total / 1000), self.columns[2]))
fig = px.bar(
df,
x=df[self.columns[0]],
y=df[self.columns[1]],
range_y=(0, high),
title=(self.title),
height=500,
)
fig.update_layout(
xaxis_type=
'category' # change x axis type so that plotly does not arrange overlapping day hours
)
logger.debug(fig)
return fig
def __init__(self) -> None:
global update_db
update_db = -1
self.elist = Youless.sql('energytypes')[
'list'] # retrieve energy types
from LS120 import Runtime, Youless
from LS120.plotly_graphs import plot_live, plot_dbdata
import LS120.db_auto_import as db
# Dash setup
from dash_settings import Dash_Settings, ip_validation
from dash_web_elements import web_elements # dash web page elements
# initialize logging
# from LS120 import logger
import logging
logger = logging.getLogger(__name__)
logger.debug("dash_allgraphs_live.py started")
# set language
Youless.youless_locale()
# database variable
update_db = 1
class all_graphs_view:
"""class to create dash page with all graphs from LS120.plotly_graphs.plot_data"""
def __init__(self) -> None:
global update_db
update_db = -1
self.elist = Youless.sql('energytypes')[
'list'] # retrieve energy types
def create_dash_page(self, ip, port, debug, reloader):
def plot_live_ten_minutes(self, **kwargs):
"""returns plotly graph with ten minute information up to 10 days back.
:when supplied it returns a graph from date start=
examples:
:plot_live_ten_minutes(etype='E')
:plot_live_ten_minutes(etype='G', start=datetime.datetime(2020, 1, 31, 0, 0))
kwargs:
:etype= str 'E' or 'G' (mandatory)
:start= datetime.datetime object with date
example: datetime.datetime(2020, 1, 31, 0, 0)
"""
if 'etype' not in kwargs:
return None
else:
self.etype = kwargs.get('etype')
self.ename = Youless.lang(
"W") if self.etype == 'E' else Youless.lang(
"L") if self.etype == 'G' else None
if 'start' in kwargs:
self.start_date = kwargs.get(
'start') # 2021-01-01 00:00:00 <class 'datetime.datetime'>
date_now = Runtime.td('date_live')
time_now = Runtime.td('time_live')
d = read_youless().read_ten_minutes(self.etype)
today = d.get(Runtime.td('current_date'))
live_usage = list(today[1][0].values())[0]
graphtitle = Youless.lang('tenminutegraphtitle').format(
live_usage, self.ename, date_now, time_now)
if 'start_date' in dir(self):
del_lst = []
for key in d.keys():
s_key = datetime.datetime.strptime(
key, '%Y-%m-%d'
) # 2021-01-01 00:00:00 <class 'datetime.datetime'>
if self.start_date > s_key:
del_lst.append(key)
for i in del_lst:
del d[i]
fig_dict = {} # dictionary for storing date time, values
date_time = [] # list for storing date time strings
values = [] # list for storing usage values
for key in d.keys(): # for date key in d
for sub_key in d[key][1]: # for every time key in the list
date_time.append(f'{key} {list(sub_key.keys())[0]}')
values.append(list(sub_key.values())[0])
fig_dict['time'] = date_time
fig_dict['usage'] = values
fig = px.line(
x=fig_dict['time'],
y=fig_dict['usage'],
title=graphtitle,
height=500, # static figure height
labels={
"x": Youless.lang('T'), # x-axis naming
"y": self.ename # y-axis naming
})
logger.debug("Plotting live ten minutes %s %s" % (date_now, time_now))
return fig
def __init__(self) -> None:
self.__headers = Youless.web("HEADERS") # acceptable html headers
self.__url = Youless.web("URL") # base url
self.__ele = Youless.web("ELE")
self.__gas = Youless.web("GAS")
self.__snul = Youless.web("S0")
self.__json = Youless.web("JSON")
self.__month = Youless.web("M")
self.__day = Youless.web("W")
self.__stats = Youless.web("STATS")
self.__D = Youless.web("D")
self.__H = Youless.web("H")
self.__kwh = Youless.sql("valuenaming")[0]
self.__m3 = Youless.sql("valuenaming")[1]
self.__watt = Youless.sql("valuenaming")[2]
self.__ltr = Youless.sql("valuenaming")[3]
def retrieve_hours(self, table, year, month, startday, starthour, *args):
"""Retrieve data from table dayhours_X and return list with items.
Retrieve hour data for given year, month and days with a minimum of 1 hour and a maximum of 24 hours in a list with hours and values.
Since we dont know if the end day/hour is the same as the start day/hour we use *args to accept this.
When only startday and starthour is given, only that single hour will be retrieved.
Examples:
retrieve_hours('dayhours_g', 2021, 3, 2, 3, 3, 6) # year: 2021, month: 3, startday: 2, starthour: 5, endday: 3, endhour: 6\n
retrieve_hours('dayhours_e', 2020, 10, 1, 12) # year: 2020, month: 10, startday: 1, starthour: 12\n
retrieve_hours('dayhours_e', 2020, 11, 2, 11, 18) # year: 2020, month: 11, startday: 2, starthour: 11, endhour: 18\n
args:
:table as string
:year as integer
:month as integer (min is 1, max is 12)
:startday as integer (min is 1, max is 30)
:endday as integer (min is 2, max is 31)
:starthour as integer (min is 0 (00:00), max is 23 (23:00))
:endhour as integer (min is 1 (01:00), max is 24 (24:00))
"""
self.table = table
self.year = year
self.month = month
self.startday = startday
self.starthour = starthour
self.hourlist = []
if (len(args) == 2):
self.endday = args[0]
self.endhour = args[1]
self.query = (Youless.sql("queries")["s_dayhours2"] %
(self.table, self.year, self.month, self.startday,
self.endday))
self.lst = [
self.year, self.month, self.startday, self.starthour,
self.endday, self.endhour, self.hourlist
]
elif (len(args) == 1):
self.endhour = args[0]
self.query = (Youless.sql("queries")["s_dayhours"] %
(self.table, self.year, self.month, self.startday))
self.lst = [
self.year, self.month, self.startday, self.starthour,
self.endhour, self.hourlist
]
else:
self.query = (Youless.sql("queries")["s_dayhours"] %
(self.table, self.year, self.month, self.startday))
self.lst = [
self.year, self.month, self.startday, self.starthour,
self.hourlist
]
logger.debug("Starting hour retrieval from table %s" % self.table)
self.cur = self.conn.cursor()
with self.conn:
data = self.cur.execute(self.query)
rows = (len(self.cur.fetchall()))
logger.debug("Rows retrieved: %d" % rows)
if (rows >= 1):
data = self.cur.execute(
self.query
) # execute query again because fetchall() mangles the data to a list of tuples with strings
rowcount = 0
for row in data:
rowcount += 1
self.values = row[7] # x is string representation of list
self.values = ast.literal_eval(
self.values) # convert x to real list
for n, v in enumerate(self.values):
self.values[n] = int(self.values[n])
if (rowcount == 1) and (n >= self.starthour):
self.hourlist.append((n, v))
if (len(args) == 1) and (n == self.endhour):
break
elif (rowcount == 2) and (n <= self.endhour):
self.hourlist.append((n, v))
else:
return 0
self.conn.close()
logger.debug("Retrieved list:")
logger.debug(self.lst)
return self.lst
def get_dayhours_average(self, **kwargs):
"""gets data from table dayhours_X based on yearday numbers and return average for:
:specified weekday in a year,
:specified week in a year,
:specified month in a year,
:specified weekday in a month in a year.
kwargs:
:year= int: 2021
:day= dayname
:month= monthname
:etype= 'E' or 'G'
"""
if 'year' in kwargs: # check if year is passed as keyword argument
self.year = kwargs.get('year')
self.start = parser.parse(
f'{self.year} 1 1'
) # a full year is always from january 1st to december 31st
self.end = parser.parse(
f'{self.year} 12 31'
) # a full year is always from january 1st to december 31st
# self.start = datetime(self.year, 1, 1) # a full year is always from january 1st to december 31st
# self.end = datetime(self.year, 12, 31) # a full year is always from january 1st to december 31st
if 'year' in kwargs and 'month' in kwargs:
self.month = kwargs.get('month').lower().capitalize()
self.start = parser.parse(
f'{self.year} {self.month} 1'
) # a month always starts at day 1, it is always the same
self.end = parser.parse(
f'{self.year} {self.month} {calendar.monthrange(self.year, self.start.month)[1]}'
) # get last day of month from calendar
# self.start = datetime(self.year, datetime.strptime(self.month, '%B').month, 1) # a month always starts at day 1, it is always the same
# self.end = datetime(self.year,
# datetime.strptime(self.month, '%B').month,
# calendar.monthrange(self.year, datetime.strptime(self.month, '%B').month)[1]) # get last day of month from calendar
if 'day' in kwargs:
self.day = kwargs.get('day').lower().capitalize()
if 'week' in kwargs:
self.week = kwargs.get('week')
self.start = retrieve_custom_data().get_date_range_from_week(
year=self.year, week=self.week)[2]
self.end = retrieve_custom_data().get_date_range_from_week(
year=self.year, week=self.week)[3]
if 'etype' in kwargs:
self.table = Youless.sql('dbtables')[kwargs.get('etype')][1]
self.select = Youless.sql('av_select')[kwargs.get('etype')]
if 'day' not in kwargs:
self.get_yeardays = retrieve_custom_data().get_yeardays(
start=self.start, end=self.end)
else:
self.get_yeardays = retrieve_custom_data().get_yeardays(
start=self.start, end=self.end, day=self.day)
logger.debug("get_yeardays: %s" % self.get_yeardays)
check_exist = [
yearday.lstrip('0') for yearday in self.get_yeardays
if retrieve_custom_data().check_existence(
table=self.table, column='yearday', item=yearday) == 1
]
logger.debug("exists yearday check: %s" % check_exist)
if check_exist == []:
return None
self.get_item = [[
i for i in retrieve_custom_data().get_item(query='select_one_and',
select=self.select,
table=self.table,
id='year',
item=self.year,
id2='yearday',
item2=n)
] for n in check_exist]
logger.debug("get_item: %s" % self.get_item)
lists = [list(elem[0]) for elem in self.get_item
] # remove nested tuple and change to list
lists = [
[elem[0], ast.literal_eval(elem[1])] for elem in lists
] # convert every 2nd item in the list to a real list (was a string)
lists = [[elem[0], int(sum(elem[1]) / len(elem[1]))] for elem in lists
] # get the sum of all values of retrieved day
average = int(sum(elem[1] for elem in lists) /
len(lists)) # get the average over all said days
# return the average and the amount of days the average is based on including the provided kwargs
if 'day' in dir(self):
if 'month' in dir(self):
return self.year, self.day, self.month, self.table, average, len(
lists)
else:
return self.year, self.day, self.table, average, len(lists)
elif 'month' in dir(self):
return self.year, self.month, self.table, average, len(lists)
else:
return self.year, self.week, self.table, average, len(lists)
def read_ten_minutes(self, etype):
"""Read per ten minute data from Youless 120.
Returned data is always 10 minutes behind last occuring rounded 10 minute interval.
e.g. at 18.58hours the newest entry is 18.40hours and at 19.00hours it will be 18.50hours
Returns dictionary with date as key containing a lists with time:value key/value dictionaries\n
example:
read_ten_minutes(etype)
args:
:etype is 'E' or 'G' for Electricity or Gas
"""
self.__urltype = self.__ele if etype == 'E' and etype != (
'G', 'S') else self.__gas if etype == 'G' else self.__snul
self.max_page = Youless.web("max_tenminutepage")
self.min_page = Youless.web("min_tenminutepage")
self.max_ten = Youless.web("max_ten")
self.min_ten = Youless.web("min_ten")
self.counter = self.min_page
self.period = 10 # 10 minute steps
data = {}
while (self.counter <= self.max_page):
self.api = requests.get(self.__url + self.__urltype + self.__json +
self.__H + str(self.counter)).json()
self.date = datetime.datetime.strptime(self.api['tm'],
'%Y-%m-%dT%H:%M:%S')
hour = self.date.hour
minute = self.date.minute - 10 # current time is 10 minutes less then json start time. subtract 10 minutes from json start time
year = self.date.date().strftime('%Y')
week_no = self.date.date().strftime('%W')
month_no = self.date.date().strftime('%m')
month_name = self.date.date().strftime('%B')
month_day = self.date.date().strftime('%d')
year_day = self.date.date().strftime('%j')
if self.date.date().strftime('%Y-%m-%d') not in data.keys():
data[self.date.date().strftime('%Y-%m-%d')] = []
data[self.date.date().strftime('%Y-%m-%d')].insert(
0,
(year, week_no, month_no, month_name, month_day, year_day))
data[self.date.date().strftime('%Y-%m-%d')].insert(1, [])
i = self.min_ten
while (i <= self.max_ten):
minute += self.period
if minute >= 60:
minute = 0
hour += 1
if hour >= 24:
hour = 0
self.date += datetime.timedelta(days=1)
self.time = '%02d:%02d' % (hour, minute
) # str(self.date.time())[0:5]
self.value = int(self.api['val'][i])
logger.debug('Time: {} Usage: {} {}'.format(
self.time, self.value, etype))
data[self.date.date().strftime('%Y-%m-%d')][1].insert(
0, {self.time: self.value})
i += 1
self.counter += 1
for key in data.keys(): # for date in dictionary
sorted_values = sorted(
data[key][1], key=lambda d: list(d.keys()), reverse=True
) # order the list hour/value dictionaries in reverse
data[key][
1] = sorted_values # write the sorted list over the original list
logger.debug(data)
return data