def get_data(starttime, endtime): """ @param starttime - start time in [seconds] @param endtime - end time in [seconds] """ data = pd.DataFrame() # loop through the loging file to collect all the data between the # start and end time for fname in os.listdir(configs.getLogDirPath(CONFIG_FILE_NAME)): if os.path.isfile( os.path.join(configs.getLogDirPath(CONFIG_FILE_NAME), fname)) == False: continue try: filetime = time.mktime(time.strptime(fname, file_name_pattern)) except ValueError: #print fname, 'matchs not the filepattern', file_name_pattern continue if starttime <= filetime and endtime + 24 * 60 * 60 >= filetime: #print fname data = data.append(get_data_from_file(fname)) # create a new column seconds mask = (data[TIMESTAMP] >= starttime) & (data[TIMESTAMP] <= endtime + 24 * 60 * 60) if len(data) <= 0: return data else: return data[mask]
def plot_humidity_chart(inner_humidity, outer_humidity, figsize=configs.fig_size, title=None, file_name='humidity.png'): fig, ax = plt.subplots(figsize=figsize) # plot the data p1, = ax.plot(inner_humidity[configs.TIMESTAMP], inner_humidity[configs.HUMIDITY], '-', label=INNER_HUMIDITY, linewidth=1) p2, = ax.plot(outer_humidity[configs.TIMESTAMP], outer_humidity[configs.HUMIDITY], '--', label=OUTER_HUMIDITY, linewidth=1) # set labels ax.set_xlabel("time") ax.set_ylabel(u'humidity [%]') # text size of the axes tkw = dict(size=4, width=1.5) # colors of the axes ax.tick_params(axis='y', colors=p1.get_color(), **tkw) # set the ticks of the x axis for each 6th value (full hour) plt.xticks(inner_humidity[configs.TIMESTAMP][::6], inner_humidity[configs.TIME].map(lambda x: x[:-3])[::6], rotation=45) plt.grid(linestyle='dashed', color='lightgray') plt.legend() # get the mean date of the time values for the name of the file print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + file_name) plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + file_name, format='png', bbox_inches='tight') plt.close()
def getDataFromFile(fileName, pathToLogDir=configs.getLogDirPath(CONFIG_FILE_NAME)): """ Fetch the data from the wind log file. The file contains the time (milliseconds since 01.01.1970) when the magnet of the hemispherical cup anemometer passed the hall sensor. One row per time. The method returns the times as numpy array of type float. """ data = np.genfromtxt(pathToLogDir + fileName, delimiter=',', dtype=float) return data
def get_data_from_file( file_name, path_to_log_dir=configs.getLogDirPath(CONFIG_FILE_NAME)): """ Fetch the data from the DHT22 log file. The file contains date, time temperature and humidity. Returns ------- pandas DataFrame containing the log data in the file """ if os.path.isfile(os.path.join(path_to_log_dir, file_name)): with open(os.path.join(path_to_log_dir, file_name), 'r') as data_file: try: df = pd.read_csv( data_file, # relative python path to subdirectory sep=',', # Tab-separated value file. quotechar="'", # single quote allowed as quote character dtype={ DATE: str, TIME: str, TEMPERATURE: float, PRESSURE: float, HUMIDITY: float }, # Parse the salary column as an integer # usecols=['name', 'birth_date', 'salary']. # Only load the three columns specified. # parse_dates=['birth_date'], # Intepret the birth_date column as a date # skiprows=10, # Skip the first 10 rows of the file na_values=['.', '??', '-'], # Take any '.', '-' or '??' values as NA ) df[TIMESTAMP] = ( df[DATE] + ' ' + df[TIME]).map(lambda x: time.mktime( time.strptime(x, datePattern + ' ' + timePattern))) return df except pd.errors.EmptyDataError: return pd.DataFrame() else: return pd.DataFrame()
def log(): # get path to log directory pathToLogDir = configs.getLogDirPath(CONFIG_FILE_NAME) # current date currentDate = datetime.now() # create name of current file of day filename = os.path.join(pathToLogDir, currentDate.strftime(file_name_pattern)) print(filename) with open(filename, 'a') as logfile: # create new datawriter datawriter = csv.DictWriter(logfile, delimiter=',', lineterminator='\n', fieldnames=field_names) # check whether the file is new or empty if logfile.tell() == 0: datawriter.writeheader() data = {} # get current time {HH:MM:SS} time = datetime.now().strftime(timePattern) # get sensor data temperature, pressure, humidity = bme280.readBME280All() data[DATE] = currentDate.strftime(datePattern) data[TIME] = time data[TEMPERATURE] = "{0:.2f}".format(temperature) data[PRESSURE] = "{0:.2f}".format(pressure) data[HUMIDITY] = "{0:.2f}".format(humidity) print(data) # write a now row into logging file datawriter.writerow(data)
import numpy as np import os import re from weatherStation.util import beaufort as btf, logBME280, logDHT22, aneometer from weatherStation.config import configs # raspberry pi wind log file name pattern fileNamePattern = aneometer.filePattern # bme280 file pattern bmeFilePattern = logBME280.file_name_pattern # DHT22 file pattern dht22FilePattern = logDHT22.file_name_pattern # Dateipfad in dem sich die Logdateinen befinden windlogsDir = configs.getLogDirPath(aneometer.CONFIG_FILE_NAME) bme280Dir = configs.getLogDirPath(logBME280.CONFIG_FILE_NAME) dht22Dir = configs.getLogDirPath(logDHT22.CONFIG_FILE_NAME) CONFIG_FILE_NAME = 'website' app = Flask(__name__) def load_config_parameter(): configs.get_config_parameter(CONFIG_FILE_NAME) host_pattern = r'^host=((\d{1,3}\.){3}(\d{1,3}))' port_pattern = r'^port=(\d{1,5})' debug_pattern = r'^debug=(true|false)?' parameter = dict(host='127.0.0.1', port=4242, debug=False)
def log(): # get path to log directory pathToLogDir = configs.getLogDirPath(CONFIG_FILE_NAME) # current date current_date = datetime.now() # create name of current file of day filename = os.path.join(pathToLogDir, current_date.strftime(file_name_pattern)) with open(filename, 'a') as logfile: # create new data_writer data_writer = csv.DictWriter(logfile, delimiter=',', lineterminator='\n', fieldnames=field_names) # check whether the file is new or empty if logfile.tell() == 0: data_writer.writeheader() data = {} # get current time {HH:MM:SS} time_str = datetime.now().strftime(timePattern) #TODO # get log date of the last x minutes # filter out the outliers # get the data of the last 60 minutes -> the last 6 values #values = get_24_hours_data(time.time())[-6] # get sensor data, as float, if an error occurred a tuple of (None, None) will # return humidity, temperature = Adafruit_DHT.read_retry(Adafruit_DHT.DHT22, 18) data[DATE] = current_date.strftime(datePattern) data[TIME] = time_str # The sensor DHT22 does not measure the pressure. data[PRESSURE] = "-" # In case both values are None # the delivered values from the sensor are invalid. # Additionally check the valid temperature and humidity # range according to the data sheet of the DHT22 # {@link https://www.sparkfun.com/datasheets/Sensors/Temperature/DHT22.pdf} # TODO Furthermore check if the current measured values # are outliers # check the range of temperature if (-40 <= temperature <= 80) or temperature is not None: data[TEMPERATURE] = "{0:.2f}".format(temperature) else: data[TEMPERATURE] = '-' # check the range of the humidity if (0 <= humidity <= 100) or humidity is not None: data[HUMIDITY] = "{0:.2f}".format(humidity) else: data[HUMIDITY] = '-' # Try to approximate the current value with the paste # values since 60 minutes ago #TODO hier noch weiter machen # ~ values = values.append(data) # ~ outliers = util.find_outlier(values) # ~ if outliers[-1] == true: # write a now row into logging file data_writer.writerow(data)
def plot_24_hours( date=time.time(), figsize=configs.fig_size, title=None, file_name=None): if title is not None: plt.title(title) data = get_24_hours_data(date) print(data[TIME]) fig, host = plt.subplots(figsize=figsize) fig.subplots_adjust(right=0.9) par1 = host.twinx() # par2 = host.twinx() # Offset the right spine of par2. The ticks and label have already been # placed on the right by twinx above. # par2.spines["right"].set_position(("axes", 1.1)) # Having been created by twinx, par2 has its frame off, so the line of its # detached spine is invisible. First, activate the frame but make the patch # and spines invisible. # make_patch_spines_invisible(par2) # Second, show the right spine. # par2.spines["right"].set_visible(True) # plot the data p1, = host.plot(data[TIMESTAMP], data[TEMPERATURE], 'r-', label=TEMPERATURE, linewidth=1) p2, = par1.plot(data[TIMESTAMP], data[HUMIDITY], 'g--', label=HUMIDITY, linewidth=1) # p3, = par2.plot(data[TIMESTAMP], data[PRESSURE], 'b:', label=TEMPERATURE, linewidth=1) # host.set_xlim(0, 2) # host.set_ylim(0, 2) # par1.set_ylim(ymax=100) # par2.set_ylim(1, 65) # set labels host.set_xlabel("time") host.set_ylabel(TEMPERATURE + u' [°C]') par1.set_ylabel(HUMIDITY + u' [%]') # par2.set_ylabel(PRESSURE + ' [hPa]') # get the color of plotted lines host.yaxis.label.set_color(p1.get_color()) par1.yaxis.label.set_color(p2.get_color()) # par2.yaxis.label.set_color(p3.get_color()) # text size of the axes tkw = dict(size=4, width=1.5) # colors of the axes host.tick_params(axis='y', colors=p1.get_color(), **tkw) par1.tick_params(axis='y', colors=p2.get_color(), **tkw) # par2.tick_params(axis='y', colors=p3.get_color(), **tkw) host.tick_params(axis='x', **tkw) # set the ticks of the x axis for each 6th value (full hour) host.set_xticks(data[TIMESTAMP][::6]) host.set_xticklabels(data[TIME].map(lambda x: x[:-3])[::6]) host.tick_params(axis='x', rotation=45, grid_linestyle='dashed', grid_color='lightgray') host.tick_params(axis='y', grid_alpha=0) host.grid(True) # plt.grid(axis='both', linestyle='dashed', color='lightgray') # get the mean date of the time values for the name of the file if file_name is None: # file_name = 'hourValues_windlogs_'+time.strftime(datePattern, data[TIMESTAMP][int(len(data[TIMESTAMP])/2)])+'.png' file_name = 'testDHT22.png' print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + file_name) plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + file_name, format='png', bbox_inches='tight') plt.close()
def plotWindVelocityCharts(winlogsDir=configs.getLogDirPath(CONFIG_FILE_NAME)): for root, dirs, files in os.walk(windlogsDir): files = sorted(files) newFormateTime = time.mktime(time.strptime('2018-03-05', '%Y-%m-%d')) print(newFormateTime) for fname in files: if fname.replace('.csv', '.png') not in files: if newFormateTime > time.mktime( time.strptime(fname, 'windLogs_%Y-%m-%d.csv')): makeWindLogDia(os.path.join(root, fname)) else: makeWindLogDia2(os.path.join(root, fname)) # walk over each file for fname in files: # allow only .csv files, otherwise continue if '.csv' not in fname or 'houreValues.csv' == fname: continue # does the data already plotted if alreadyAsPNG(files, fname[-14:-4]): print('data of file -', fname, '- already plotted') continue # time of data file in seconds timeOfDataFile = time.mktime( time.strptime(fname, 'windLogs_%Y-%m-%d.csv')) if startDate >= timeOfDataFile: print('file', fname, 'ignored (startDate)') continue if newFormateTime > timeOfDataFile: # old formate: from,average,min,max # print 'Get data from ',fname,'...' # print 'file',fname,'ignored (oldFileFormate)' # continue print('Get data from ', fname, '...') try: data = np.array([ float(x.split(',')[0]) for x in open(windlogsDir + '/' + fname, 'r').readlines() ], dtype=float) except ValueError: print('ValueError') lineNumber = 1 for x in open(windlogsDir + '/' + fname, 'r').readlines(): try: xfloat = float(x) lineNumber = lineNumber + 1 except ValueError: print('ValueError in file', fname) print(x, 'in line', lineNumber + 1, 'cannot convert in a float!') break break startTime = time.mktime( time.strptime(fname + '-' + str(0), 'windLogs_%Y-%m-%d.csv-%H')) xValues = [] yValues = []
startTime = time.mktime( time.strptime(fname + '-' + str(0), 'windLogs_%Y-%m-%d.csv-%H')) xValues = [] yValues = [] # # # # Log script version 2. Difference to version 1 is that every signal of # the anemometer will save into the log file since the 5th marche 2018. SAVE_DIRECTORY = configs.getLogDirPath(CONFIG_FILE_NAME) logs = [] def getCurrentTime(): """ # Get the current time in miliseconds """ return time.time() * 1000 def getLogFileName(): """ Get the name of the log file which has the pattern 'windLogs_%Y-%m-%d.csv'.
def plot_hour_wind_log_chart(data, fileName=None, figsize=configs.fig_size, title=None): """ Make a plot of the given wind log data. Before plotting the values averaging before for each time interval. @param data - hours and wind velocity values Parameters ---------- data : hours and wind velocity values, data[0] - time in milliseconds, data[1] - wind velocity as hour average [km/h] fileName : file name of the chart to save figsize : size of the chart in inches title : title of the chart """ # generate labels labels = [[], []] for x in data[0]: # get full hour time before xValue[-1] x_label_time = round(x / 3600000. - 0.5) * 3600000 # add xticks label coordinate (time value on x axis) labels[0].append(x_label_time) # add the corresponding time as string labels[1].append( time.strftime('%H:%M', time.localtime(x_label_time / 1000.))) # get full hour time after xValue[-1] x_label_time = round(data[0][-1] / 3600000. + 0.5) * 3600000 labels[0].append(x_label_time) labels[1].append( time.strftime('%H:%M', time.localtime(x_label_time / 1000.))) # todo Den Part noch auslagern !!! In dieser Methode besser nur die Daten generieren. # print 'Start plotting ...' # Plot the wind velocity chart fig = plt.figure(figsize=figsize) ax = plt.subplot(111) if title is not None: plt.title(title) plt.xlabel('time') plt.ylabel('wind velocity [km/h]') ax.plot(data[0], data[1], '-', label='hourly average', linewidth=1) # generate the x axis labels timeinterval = data[0][1] - data[0][0] plt.xticks(labels[0], labels[1], rotation=45) # print 'legend' # plt.legend() plt.ylim(ymin=0) plt.xlim(labels[0][0] - timeinterval / 2.0, labels[0][-1] + 3 * timeinterval / 2.0) plt.grid(linestyle='dashed', color='lightgray') # get the mean date of the time values for the name of the file if fileName is None: fileName = 'hourValues_windlogs_' + time.strftime( aneometer.datePattern, time.localtime(data[0][int(len(data[0]) / 2)] / 1000.)) + '.png' print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + fileName) plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + fileName, format='png', bbox_inches='tight') plt.close()
def plot_group_chart(inner, outer, figsize=configs.fig_size_group, title=None, file_name='groupChart.png'): fig, (ax1, ax2, ax3) = plt.subplots(3, figsize=figsize, sharex=True, gridspec_kw={'hspace': 0}) # plot the data ax1.plot(inner[configs.TIMESTAMP], inner[configs.TEMPERATURE], 'r-', label=INNER_TEMPERATURE, linewidth=1) ax1.plot(outer[configs.TIMESTAMP], outer[configs.TEMPERATURE], 'r--', label=OUTER_TEMPERATURE, linewidth=1) ax2.plot(inner[configs.TIMESTAMP], inner[configs.HUMIDITY], 'g-', label=INNER_HUMIDITY, linewidth=1) ax2.plot(outer[configs.TIMESTAMP], outer[configs.HUMIDITY], 'g--', label=OUTER_HUMIDITY, linewidth=1) ax3.plot(inner[configs.TIMESTAMP], inner[configs.PRESSURE], 'b-', label=PRESSURE, linewidth=1) # set labels plt.xlabel('time') ax1.set_ylabel(u'temperature [°C]') ax2.set_ylabel(u'humidity [%]') ax3.set_ylabel(u'pressure [hPa]') # text size of the axes tkw = dict(size=4, width=1.5) # colors of the axes ax1.tick_params(axis='y', colors='black', **tkw) ax2.tick_params(axis='y', colors='black', **tkw) ax3.tick_params(axis='y', colors='black', **tkw) # set the ticks of the x axis for each 6th value (full hour) plt.xticks(inner[configs.TIMESTAMP][::6], inner[configs.TIME].map(lambda x: x[:-3])[::6], rotation=45) ax1.grid(linestyle='dashed', color='lightgray') ax2.grid(linestyle='dashed', color='lightgray') ax3.grid(linestyle='dashed', color='lightgray') ax1.legend() ax2.legend() ax3.legend() # add title if title is not None: st = fig.suptitle(title) st.set_y(0.95) fig.subplots_adjust(top=0.91) # get the mean date of the time values for the name of the file print('savefig', configs.getLogDirPath(CONFIG_FILE_NAME) + file_name) plt.savefig(configs.getLogDirPath(CONFIG_FILE_NAME) + file_name, format='png', bbox_inches='tight') plt.close()