def get_data(starttime, endtime):
"""
@param starttime - start time in [seconds]
@param endtime - end time in [seconds]
"""
data = pd.DataFrame()
# loop through the loging file to collect all the data between the
# start and end time
for fname in os.listdir(configs.getLogDirPath(CONFIG_FILE_NAME)):
if os.path.isfile(
os.path.join(configs.getLogDirPath(CONFIG_FILE_NAME),
fname)) == False:
continue
try:
filetime = time.mktime(time.strptime(fname, file_name_pattern))
except ValueError:
#print fname, 'matchs not the filepattern', file_name_pattern
continue
if starttime <= filetime and endtime + 24 * 60 * 60 >= filetime:
#print fname
data = data.append(get_data_from_file(fname))
# create a new column seconds
mask = (data[TIMESTAMP] >= starttime) & (data[TIMESTAMP] <=
endtime + 24 * 60 * 60)
if len(data) <= 0:
return data
else:
return data[mask]
def plot_humidity_chart(inner_humidity,
outer_humidity,
figsize=configs.fig_size,
title=None,
file_name='humidity.png'):
fig, ax = plt.subplots(figsize=figsize)
# plot the data
p1, = ax.plot(inner_humidity[configs.TIMESTAMP],
inner_humidity[configs.HUMIDITY],
'-',
label=INNER_HUMIDITY,
linewidth=1)
p2, = ax.plot(outer_humidity[configs.TIMESTAMP],
outer_humidity[configs.HUMIDITY],
'--',
label=OUTER_HUMIDITY,
linewidth=1)
# set labels
ax.set_xlabel("time")
ax.set_ylabel(u'humidity [%]')
# text size of the axes
tkw = dict(size=4, width=1.5)
# colors of the axes
ax.tick_params(axis='y', colors=p1.get_color(), **tkw)
# set the ticks of the x axis for each 6th value (full hour)
plt.xticks(inner_humidity[configs.TIMESTAMP][::6],
inner_humidity[configs.TIME].map(lambda x: x[:-3])[::6],
rotation=45)
plt.grid(linestyle='dashed', color='lightgray')
plt.legend()
# get the mean date of the time values for the name of the file
print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + file_name)
plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + file_name,
format='png',
bbox_inches='tight')
plt.close()
def getDataFromFile(fileName,
pathToLogDir=configs.getLogDirPath(CONFIG_FILE_NAME)):
"""
Fetch the data from the wind log file. The file contains the time
(milliseconds since 01.01.1970) when the magnet of the
hemispherical cup anemometer passed the hall sensor. One row per time.
The method returns the times as numpy array of type float.
"""
data = np.genfromtxt(pathToLogDir + fileName, delimiter=',', dtype=float)
return data
def get_data_from_file(
file_name, path_to_log_dir=configs.getLogDirPath(CONFIG_FILE_NAME)):
"""
Fetch the data from the DHT22 log file. The file contains date, time
temperature and humidity.
Returns
-------
pandas DataFrame containing the log data in the file
"""
if os.path.isfile(os.path.join(path_to_log_dir, file_name)):
with open(os.path.join(path_to_log_dir, file_name), 'r') as data_file:
try:
df = pd.read_csv(
data_file, # relative python path to subdirectory
sep=',', # Tab-separated value file.
quotechar="'", # single quote allowed as quote character
dtype={
DATE: str,
TIME: str,
TEMPERATURE: float,
PRESSURE: float,
HUMIDITY: float
},
# Parse the salary column as an integer
# usecols=['name', 'birth_date', 'salary']. # Only load the three columns specified.
# parse_dates=['birth_date'], # Intepret the birth_date column as a date
# skiprows=10, # Skip the first 10 rows of the file
na_values=['.', '??',
'-'], # Take any '.', '-' or '??' values as NA
)
df[TIMESTAMP] = (
df[DATE] + ' ' + df[TIME]).map(lambda x: time.mktime(
time.strptime(x, datePattern + ' ' + timePattern)))
return df
except pd.errors.EmptyDataError:
return pd.DataFrame()
else:
return pd.DataFrame()
def log():
# get path to log directory
pathToLogDir = configs.getLogDirPath(CONFIG_FILE_NAME)
# current date
currentDate = datetime.now()
# create name of current file of day
filename = os.path.join(pathToLogDir,
currentDate.strftime(file_name_pattern))
print(filename)
with open(filename, 'a') as logfile:
# create new datawriter
datawriter = csv.DictWriter(logfile,
delimiter=',',
lineterminator='\n',
fieldnames=field_names)
# check whether the file is new or empty
if logfile.tell() == 0:
datawriter.writeheader()
data = {}
# get current time {HH:MM:SS}
time = datetime.now().strftime(timePattern)
# get sensor data
temperature, pressure, humidity = bme280.readBME280All()
data[DATE] = currentDate.strftime(datePattern)
data[TIME] = time
data[TEMPERATURE] = "{0:.2f}".format(temperature)
data[PRESSURE] = "{0:.2f}".format(pressure)
data[HUMIDITY] = "{0:.2f}".format(humidity)
print(data)
# write a now row into logging file
datawriter.writerow(data)
import numpy as np
import os
import re
from weatherStation.util import beaufort as btf, logBME280, logDHT22, aneometer
from weatherStation.config import configs
# raspberry pi wind log file name pattern
fileNamePattern = aneometer.filePattern
# bme280 file pattern
bmeFilePattern = logBME280.file_name_pattern
# DHT22 file pattern
dht22FilePattern = logDHT22.file_name_pattern
# Dateipfad in dem sich die Logdateinen befinden
windlogsDir = configs.getLogDirPath(aneometer.CONFIG_FILE_NAME)
bme280Dir = configs.getLogDirPath(logBME280.CONFIG_FILE_NAME)
dht22Dir = configs.getLogDirPath(logDHT22.CONFIG_FILE_NAME)
CONFIG_FILE_NAME = 'website'
app = Flask(__name__)
def load_config_parameter():
configs.get_config_parameter(CONFIG_FILE_NAME)
host_pattern = r'^host=((\d{1,3}\.){3}(\d{1,3}))'
port_pattern = r'^port=(\d{1,5})'
debug_pattern = r'^debug=(true|false)?'
parameter = dict(host='127.0.0.1', port=4242, debug=False)
def log():
# get path to log directory
pathToLogDir = configs.getLogDirPath(CONFIG_FILE_NAME)
# current date
current_date = datetime.now()
# create name of current file of day
filename = os.path.join(pathToLogDir,
current_date.strftime(file_name_pattern))
with open(filename, 'a') as logfile:
# create new data_writer
data_writer = csv.DictWriter(logfile,
delimiter=',',
lineterminator='\n',
fieldnames=field_names)
# check whether the file is new or empty
if logfile.tell() == 0:
data_writer.writeheader()
data = {}
# get current time {HH:MM:SS}
time_str = datetime.now().strftime(timePattern)
#TODO
# get log date of the last x minutes
# filter out the outliers
# get the data of the last 60 minutes -> the last 6 values
#values = get_24_hours_data(time.time())[-6]
# get sensor data, as float, if an error occurred a tuple of (None, None) will
# return
humidity, temperature = Adafruit_DHT.read_retry(Adafruit_DHT.DHT22, 18)
data[DATE] = current_date.strftime(datePattern)
data[TIME] = time_str
# The sensor DHT22 does not measure the pressure.
data[PRESSURE] = "-"
# In case both values are None
# the delivered values from the sensor are invalid.
# Additionally check the valid temperature and humidity
# range according to the data sheet of the DHT22
# {@link https://www.sparkfun.com/datasheets/Sensors/Temperature/DHT22.pdf}
# TODO Furthermore check if the current measured values
# are outliers
# check the range of temperature
if (-40 <= temperature <= 80) or temperature is not None:
data[TEMPERATURE] = "{0:.2f}".format(temperature)
else:
data[TEMPERATURE] = '-'
# check the range of the humidity
if (0 <= humidity <= 100) or humidity is not None:
data[HUMIDITY] = "{0:.2f}".format(humidity)
else:
data[HUMIDITY] = '-'
# Try to approximate the current value with the paste
# values since 60 minutes ago
#TODO hier noch weiter machen
# ~ values = values.append(data)
# ~ outliers = util.find_outlier(values)
# ~ if outliers[-1] == true:
# write a now row into logging file
data_writer.writerow(data)
def plot_24_hours(
date=time.time(), figsize=configs.fig_size, title=None,
file_name=None):
if title is not None:
plt.title(title)
data = get_24_hours_data(date)
print(data[TIME])
fig, host = plt.subplots(figsize=figsize)
fig.subplots_adjust(right=0.9)
par1 = host.twinx()
# par2 = host.twinx()
# Offset the right spine of par2. The ticks and label have already been
# placed on the right by twinx above.
# par2.spines["right"].set_position(("axes", 1.1))
# Having been created by twinx, par2 has its frame off, so the line of its
# detached spine is invisible. First, activate the frame but make the patch
# and spines invisible.
# make_patch_spines_invisible(par2)
# Second, show the right spine.
# par2.spines["right"].set_visible(True)
# plot the data
p1, = host.plot(data[TIMESTAMP],
data[TEMPERATURE],
'r-',
label=TEMPERATURE,
linewidth=1)
p2, = par1.plot(data[TIMESTAMP],
data[HUMIDITY],
'g--',
label=HUMIDITY,
linewidth=1)
# p3, = par2.plot(data[TIMESTAMP], data[PRESSURE], 'b:', label=TEMPERATURE, linewidth=1)
# host.set_xlim(0, 2)
# host.set_ylim(0, 2)
# par1.set_ylim(ymax=100)
# par2.set_ylim(1, 65)
# set labels
host.set_xlabel("time")
host.set_ylabel(TEMPERATURE + u' [°C]')
par1.set_ylabel(HUMIDITY + u' [%]')
# par2.set_ylabel(PRESSURE + ' [hPa]')
# get the color of plotted lines
host.yaxis.label.set_color(p1.get_color())
par1.yaxis.label.set_color(p2.get_color())
# par2.yaxis.label.set_color(p3.get_color())
# text size of the axes
tkw = dict(size=4, width=1.5)
# colors of the axes
host.tick_params(axis='y', colors=p1.get_color(), **tkw)
par1.tick_params(axis='y', colors=p2.get_color(), **tkw)
# par2.tick_params(axis='y', colors=p3.get_color(), **tkw)
host.tick_params(axis='x', **tkw)
# set the ticks of the x axis for each 6th value (full hour)
host.set_xticks(data[TIMESTAMP][::6])
host.set_xticklabels(data[TIME].map(lambda x: x[:-3])[::6])
host.tick_params(axis='x',
rotation=45,
grid_linestyle='dashed',
grid_color='lightgray')
host.tick_params(axis='y', grid_alpha=0)
host.grid(True)
# plt.grid(axis='both', linestyle='dashed', color='lightgray')
# get the mean date of the time values for the name of the file
if file_name is None:
# file_name = 'hourValues_windlogs_'+time.strftime(datePattern, data[TIMESTAMP][int(len(data[TIMESTAMP])/2)])+'.png'
file_name = 'testDHT22.png'
print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + file_name)
plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + file_name,
format='png',
bbox_inches='tight')
plt.close()
def plotWindVelocityCharts(winlogsDir=configs.getLogDirPath(CONFIG_FILE_NAME)):
for root, dirs, files in os.walk(windlogsDir):
files = sorted(files)
newFormateTime = time.mktime(time.strptime('2018-03-05', '%Y-%m-%d'))
print(newFormateTime)
for fname in files:
if fname.replace('.csv', '.png') not in files:
if newFormateTime > time.mktime(
time.strptime(fname, 'windLogs_%Y-%m-%d.csv')):
makeWindLogDia(os.path.join(root, fname))
else:
makeWindLogDia2(os.path.join(root, fname))
# walk over each file
for fname in files:
# allow only .csv files, otherwise continue
if '.csv' not in fname or 'houreValues.csv' == fname:
continue
# does the data already plotted
if alreadyAsPNG(files, fname[-14:-4]):
print('data of file -', fname, '- already plotted')
continue
# time of data file in seconds
timeOfDataFile = time.mktime(
time.strptime(fname, 'windLogs_%Y-%m-%d.csv'))
if startDate >= timeOfDataFile:
print('file', fname, 'ignored (startDate)')
continue
if newFormateTime > timeOfDataFile:
# old formate: from,average,min,max
# print 'Get data from ',fname,'...'
# print 'file',fname,'ignored (oldFileFormate)'
# continue
print('Get data from ', fname, '...')
try:
data = np.array([
float(x.split(',')[0])
for x in open(windlogsDir + '/' +
fname, 'r').readlines()
],
dtype=float)
except ValueError:
print('ValueError')
lineNumber = 1
for x in open(windlogsDir + '/' + fname, 'r').readlines():
try:
xfloat = float(x)
lineNumber = lineNumber + 1
except ValueError:
print('ValueError in file', fname)
print(x, 'in line', lineNumber + 1,
'cannot convert in a float!')
break
break
startTime = time.mktime(
time.strptime(fname + '-' + str(0),
'windLogs_%Y-%m-%d.csv-%H'))
xValues = []
yValues = []
startTime = time.mktime(
time.strptime(fname + '-' + str(0),
'windLogs_%Y-%m-%d.csv-%H'))
xValues = []
yValues = []
#
#
#
# Log script version 2. Difference to version 1 is that every signal of
# the anemometer will save into the log file since the 5th marche 2018.
SAVE_DIRECTORY = configs.getLogDirPath(CONFIG_FILE_NAME)
logs = []
def getCurrentTime():
"""
# Get the current time in miliseconds
"""
return time.time() * 1000
def getLogFileName():
"""
Get the name of the log file which has
the pattern 'windLogs_%Y-%m-%d.csv'.
def plot_hour_wind_log_chart(data,
fileName=None,
figsize=configs.fig_size,
title=None):
"""
Make a plot of the given wind log data. Before plotting the values averaging
before for each time interval.
@param data - hours and wind velocity values
Parameters
----------
data : hours and wind velocity values, data[0] - time in milliseconds, data[1] - wind velocity as hour average [km/h]
fileName : file name of the chart to save
figsize : size of the chart in inches
title : title of the chart
"""
# generate labels
labels = [[], []]
for x in data[0]:
# get full hour time before xValue[-1]
x_label_time = round(x / 3600000. - 0.5) * 3600000
# add xticks label coordinate (time value on x axis)
labels[0].append(x_label_time)
# add the corresponding time as string
labels[1].append(
time.strftime('%H:%M', time.localtime(x_label_time / 1000.)))
# get full hour time after xValue[-1]
x_label_time = round(data[0][-1] / 3600000. + 0.5) * 3600000
labels[0].append(x_label_time)
labels[1].append(
time.strftime('%H:%M', time.localtime(x_label_time / 1000.)))
# todo Den Part noch auslagern !!! In dieser Methode besser nur die Daten generieren.
# print 'Start plotting ...'
# Plot the wind velocity chart
fig = plt.figure(figsize=figsize)
ax = plt.subplot(111)
if title is not None:
plt.title(title)
plt.xlabel('time')
plt.ylabel('wind velocity [km/h]')
ax.plot(data[0], data[1], '-', label='hourly average', linewidth=1)
# generate the x axis labels
timeinterval = data[0][1] - data[0][0]
plt.xticks(labels[0], labels[1], rotation=45)
# print 'legend'
# plt.legend()
plt.ylim(ymin=0)
plt.xlim(labels[0][0] - timeinterval / 2.0,
labels[0][-1] + 3 * timeinterval / 2.0)
plt.grid(linestyle='dashed', color='lightgray')
# get the mean date of the time values for the name of the file
if fileName is None:
fileName = 'hourValues_windlogs_' + time.strftime(
aneometer.datePattern,
time.localtime(data[0][int(len(data[0]) / 2)] / 1000.)) + '.png'
print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + fileName)
plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + fileName,
format='png',
bbox_inches='tight')
plt.close()
def plot_group_chart(inner,
outer,
figsize=configs.fig_size_group,
title=None,
file_name='groupChart.png'):
fig, (ax1, ax2, ax3) = plt.subplots(3,
figsize=figsize,
sharex=True,
gridspec_kw={'hspace': 0})
# plot the data
ax1.plot(inner[configs.TIMESTAMP],
inner[configs.TEMPERATURE],
'r-',
label=INNER_TEMPERATURE,
linewidth=1)
ax1.plot(outer[configs.TIMESTAMP],
outer[configs.TEMPERATURE],
'r--',
label=OUTER_TEMPERATURE,
linewidth=1)
ax2.plot(inner[configs.TIMESTAMP],
inner[configs.HUMIDITY],
'g-',
label=INNER_HUMIDITY,
linewidth=1)
ax2.plot(outer[configs.TIMESTAMP],
outer[configs.HUMIDITY],
'g--',
label=OUTER_HUMIDITY,
linewidth=1)
ax3.plot(inner[configs.TIMESTAMP],
inner[configs.PRESSURE],
'b-',
label=PRESSURE,
linewidth=1)
# set labels
plt.xlabel('time')
ax1.set_ylabel(u'temperature [°C]')
ax2.set_ylabel(u'humidity [%]')
ax3.set_ylabel(u'pressure [hPa]')
# text size of the axes
tkw = dict(size=4, width=1.5)
# colors of the axes
ax1.tick_params(axis='y', colors='black', **tkw)
ax2.tick_params(axis='y', colors='black', **tkw)
ax3.tick_params(axis='y', colors='black', **tkw)
# set the ticks of the x axis for each 6th value (full hour)
plt.xticks(inner[configs.TIMESTAMP][::6],
inner[configs.TIME].map(lambda x: x[:-3])[::6],
rotation=45)
ax1.grid(linestyle='dashed', color='lightgray')
ax2.grid(linestyle='dashed', color='lightgray')
ax3.grid(linestyle='dashed', color='lightgray')
ax1.legend()
ax2.legend()
ax3.legend()
# add title
if title is not None:
st = fig.suptitle(title)
st.set_y(0.95)
fig.subplots_adjust(top=0.91)
# get the mean date of the time values for the name of the file
print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + file_name)
plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + file_name,
format='png',
bbox_inches='tight')
plt.close()