# ### Script settings
#
# In[ ]:
number_of_days = 7
# ### We create a dataframe per sensor
# In[ ]:
# Load houseprint from cache if possible, otherwise build it from source
try:
hp_filename = os.path.join(c.get('data', 'folder'), 'hp_anonymous.pkl')
hp = houseprint.load_houseprint_from_file(hp_filename)
print("Houseprint loaded from {}".format(hp_filename))
except Exception as e:
print(e)
print("Because of this error we try to build the houseprint from source")
hp = houseprint.Houseprint()
# In[ ]:
start = pd.Timestamp(time() - number_of_days * 86400, unit='s')
# In[ ]:
sensors = hp.get_sensors()
#sensors.remove('b325dbc1a0d62c99a50609e919b9ea06')
import numpy as np
import matplotlib.pyplot as plt
import pytz
import datetime as dt
script_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
# add the path to opengrid to sys.path
sys.path.append(os.path.join(script_dir, os.pardir, os.pardir))
from opengrid.library import houseprint
from opengrid.library import fluksoapi
# script settings ############################################################
path_to_data = os.path.abspath('/usr/local/data')
# get all sensors
hp = houseprint.load_houseprint_from_file('hp_anonymous.pkl')
sensors = []
for flukso_id, d in hp.fluksosensors.items():
for sensor_id, s in d.items():
# sensor_id is 1-6, s is {}
try:
sensors.append(s['Sensor'])
except:
pass
print("{} sensors found".format(len(sensors)))
for sensor in sensors:
# create a single csv
csv=fluksoapi.consolidate(folder=path_to_data, sensor=sensor)
try:
from opengrid.library import houseprint
except ImportError:
sys.path.append(c.get('backend', 'opengrid'))
from opengrid.library import houseprint
app = Flask(__name__)
SECRET_KEY = "secret_key" # TODO add a real key in the config file
app.config.from_object(__name__)
try:
hp = houseprint.Houseprint()
except:
print("Connection failed, loading houseprint from cache")
hp = houseprint.load_houseprint_from_file("cache_hp.hp")
else:
hp.save("cache_hp.hp")
@app.route("/")
@app.route("/index")
def index():
return render_template('index.html')
@app.route("/data")
def data():
devices = hp.get_devices()
devices.sort(key=lambda x: x.key)
return render_template('data.html', fluksos=devices)
try:
from opengrid.library import houseprint
except ImportError:
sys.path.append(c.get('backend', 'opengrid'))
from opengrid.library import houseprint
app = Flask(__name__)
SECRET_KEY = "secret_key" # TODO add a real key in the config file
app.config.from_object(__name__)
try:
hp = houseprint.Houseprint()
except:
print("Connection failed, loading houseprint from cache")
hp = houseprint.load_houseprint_from_file("cache_hp.hp")
else:
hp.save("cache_hp.hp")
@app.route("/")
@app.route("/index")
def index():
return render_template('index.html')
@app.route("/data")
def data():
devices = hp.get_devices()
devices.sort(key=lambda x: x.key)
return render_template('data.html', fluksos=devices)
import pytz
import datetime as dt
script_dir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
# add the path to opengrid to sys.path
sys.path.append(os.path.join(script_dir, os.pardir, os.pardir))
from opengrid.library import houseprint
from opengrid.library import fluksoapi
# script settings ############################################################
path_to_data = os.path.abspath(
'/home/roel/data/work/opengrid/backup_data_in_EDT/data')
# get all sensors
hp = houseprint.load_houseprint_from_file('hp_anonymous.pkl')
sensors = []
for flukso_id, d in hp.fluksosensors.items():
for sensor_id, s in d.items():
# sensor_id is 1-6, s is {}
try:
sensors.append(s['Sensor'])
except:
pass
print("{} sensors found".format(len(sensors)))
for sensor in sensors:
# create a single csv
csv = fluksoapi.consolidate(folder=path_to_data, sensor=sensor)
def compute(sensorid, start_model, end_model):
end = pd.Timestamp('now', tz='Europe/Brussels')
# Create houseprint from saved file, if not available, parse the google spreadsheet
try:
hp_filename = os.path.join(c.get('data', 'folder'), 'hp_anonymous.pkl')
hp = houseprint.load_houseprint_from_file(hp_filename)
print("Houseprint loaded from {}".format(hp_filename))
except Exception as e:
print(e)
print(
"Because of this error we try to build the houseprint from source")
hp = houseprint.Houseprint()
hp.init_tmpo()
# Load the cached daily data
sensor = hp.find_sensor(sensorid)
cache = caching.Cache(variable='{}_daily_total'.format(sensor.type))
df_day = cache.get(sensors=[sensor])
df_day.rename(columns={sensorid: sensor.type}, inplace=True)
# Load the cached weather data, clean up and compose a combined dataframe
weather = forecastwrapper.Weather(location=(50.8024, 4.3407),
start=start_model,
end=end)
irradiances = [
(0, 90), # north vertical
(90, 90), # east vertical
(180, 90), # south vertical
(270, 90), # west vertical
]
orientations = [0, 90, 180, 270]
weather_data = weather.days(
irradiances=irradiances,
wind_orients=orientations,
heating_base_temperatures=[0, 6, 8, 10, 12, 14, 16, 18]).dropna(axis=1)
weather_data.drop([
'icon', 'summary', 'moonPhase', 'windBearing', 'temperatureMaxTime',
'temperatureMinTime', 'apparentTemperatureMaxTime',
'apparentTemperatureMinTime', 'uvIndexTime', 'sunsetTime',
'sunriseTime'
],
axis=1,
inplace=True)
# Add columns for the day-of-week
for i, d in zip(range(7), [
'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday',
'Sunday'
]):
weather_data[d] = 0
weather_data.loc[weather_data.index.weekday == i, d] = 1
weather_data = weather_data.applymap(float)
data = pd.concat([df_day, weather_data], axis=1).dropna()
data = data.tz_convert('Europe/Brussels')
df = data.resample(rule='MS').sum()
if len(df) < 2:
print("Not enough data for building a monthly reference model")
sys.exit(1)
# monthly model, statistical validation
mv = regression.MVLinReg(df.ix[:end_model], sensor.type, p_max=0.03)
figures = mv.plot(df=df)
figures[0].savefig(os.path.join(c.get('data', 'folder'), 'figures',
'multivar_model_' + sensorid + '.png'),
dpi=100)
figures[1].savefig(os.path.join(c.get('data', 'folder'), 'figures',
'multivar_results_' + sensorid + '.png'),
dpi=100)
# weekly model, statistical validation
df = data.resample(rule='W').sum()
if len(df.ix[:end_model]) < 4:
print("Not enough data for building a weekly reference model")
sys.exit(1)
mv = regression.MVLinReg(df.ix[:end_model], sensor.type, p_max=0.02)
if len(df.ix[end_model:]) > 0:
figures = mv.plot(model=False, bar_chart=True, df=df.ix[end_model:])
figures[0].savefig(os.path.join(
c.get('data', 'folder'), 'figures',
'multivar_prediction_weekly_' + sensorid + '.png'),
dpi=100)
import zipfile
script_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
# add the path to opengrid to sys.path
sys.path.append(os.path.join(script_dir, os.pardir, os.pardir))
from opengrid.library import houseprint
from opengrid.library import fluksoapi
##############################################################################
path_to_data = os.path.abspath('/usr/local/data')
path_to_webserver = os.path.abspath('/var/www/private')
# consolidate for yesterday
yesterday = dt.datetime.now() - dt.timedelta(days=1)
# get all sensors
hp = houseprint.load_houseprint_from_file(os.path.join(script_dir, 'hp_anonymous.pkl'))
sensors = hp.get_sensors()
print("{} sensors found".format(len(sensors)))
# create a empty zip-file with the date as filename
zipfilename = yesterday.strftime(format="%Y%m%d")
with zipfile.ZipFile(os.path.join(path_to_webserver, zipfilename+'.zip'), 'w') as myzip:
for sensor in sensors:
# create a csv with the data of the given day
csv = fluksoapi.consolidate_sensor(path_to_data, sensor, file_type='csv', dt_day=yesterday)
# add to myzip
myzip.write(csv, arcname=os.path.split(csv)[-1])
# and remove the file. original files are kept
os.remove(csv)
