def addRecords(self, forecastID, parserID, values): if(self.database.isOpen()): minMaxMap = {} for value in values: dayTimestamp = rmGetStartOfDay(value.timestamp) minMax = minMaxMap.get(dayTimestamp) if minMax is None: minMax = self.getMinMax(parserID, dayTimestamp) minMaxMap[dayTimestamp] = minMax minMax["minTemperature"] = self.__min(self.__min(value.minTemperature, value.temperature), minMax["minTemperature"]) minMax["maxTemperature"] = self.__max(self.__max(value.maxTemperature, value.temperature), minMax["maxTemperature"]) minMax["minRH"] = self.__min(self.__min(value.minRh, value.rh), minMax["minRH"]) minMax["maxRH"] = self.__max(self.__min(value.maxRh, value.rh), minMax["maxRH"]) valuesToInsert = [] for value in values: dayTimestamp = rmGetStartOfDay(value.timestamp) minMax = minMaxMap[dayTimestamp] valuesToInsert.append((forecastID, parserID, value.timestamp, value.temperature, minMax["minTemperature"], minMax["maxTemperature"], value.rh, minMax["minRH"], minMax["maxRH"], value.wind, value.solarRad, value.skyCover, value.rain, value.et0, value.pop, value.qpf, value.condition, value.pressure, value.dewPoint, value.userData)) self.clearHistory(parserID, False) self.database.executeMany("INSERT INTO parserData(forecastID, parserID, timestamp, "\ "temperature, minTemperature, maxTemperature, rh, minRh, maxRh, "\ "wind, solarRad, skyCover, rain, et0, pop, qpf, "\ "condition, pressure, dewPoint, userData) "\ "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", valuesToInsert) self.database.commit()
def parseStationYesterdayDataNoKey(self, data): #daily summary for yesterday try: l = RMWeatherDataLimits() temperature = self.__toFloat(data["TemperatureAvgC"]) mintemp = self.__toFloat(data["TemperatureLowC"]) maxtemp = self.__toFloat(data["TemperatureHighC"]) rh = self.__toFloat(data["HumidityAvg"]) minrh = self.__toFloat(data["HumidityLow"]) maxrh = self.__toFloat(data["HumidityHigh"]) dewpoint = self.__toFloat(data["DewpointAvgC"]) wind = self.__toFloat(data["WindSpeedAvgKMH"]) maxpressure = self.__toFloat(data["PressureMaxhPa"]) minpressure = self.__toFloat(data["PressureMinhPa"]) rain = self.__toFloat( data["PrecipitationSumCM"]) * 10.0 # from cm to mm if wind is not None: wind = wind / 3.6 # converted from kmetersph to mps if maxpressure is not None: maxpressure = l.sanitize(RMWeatherDataType.PRESSURE, maxpressure / 10.0) # converted to from hpa to kpa if minpressure is not None: minpressure = l.sanitize(RMWeatherDataType.PRESSURE, minpressure / 10.0) pressure = None if maxpressure is not None and minpressure is not None: pressure = (maxpressure + minpressure) / 2.0 #log.info("rh:%s minrh: %s maxrh: %s pressure: %s temp: %s mintemp: %s maxtemp: %s" % (rh, minrh, maxrh, pressure, temperature, mintemp, maxtemp)) timestamp = rmCurrentDayTimestamp() timestamp = rmGetStartOfDay(timestamp - 12 * 3600) self.addValue(RMParser.dataType.TEMPERATURE, timestamp, temperature, False) self.addValue(RMParser.dataType.MINTEMP, timestamp, mintemp, False) self.addValue(RMParser.dataType.MAXTEMP, timestamp, maxtemp, False) self.addValue(RMParser.dataType.RH, timestamp, rh, False) self.addValue(RMParser.dataType.MINRH, timestamp, minrh, False) self.addValue(RMParser.dataType.MAXRH, timestamp, maxrh, False) self.addValue(RMParser.dataType.WIND, timestamp, wind, False) self.addValue(RMParser.dataType.RAIN, timestamp, rain, False) self.addValue(RMParser.dataType.DEWPOINT, timestamp, dewpoint, False) self.addValue(RMParser.dataType.PRESSURE, timestamp, pressure, False) except Exception, e: self.lastKnownError = "ERROR: Failed to get historical data" log.error("%s: %s" % (self.lastKnownError, e))
def getMinMax(self, parserID, dayTimestamp): ### Min and Max are computed only from the last forecast for that day. results = { "minTemperature": None, "maxTemperature": None, "minRH": None, "maxRH": None } if self.database.isOpen(): minForecastTimestamp = None maxForecastTimestamp = None maxDayTimestamp = dayTimestamp + 86400 rows = self.database.execute("SELECT f.timestamp, pd.timestamp, pd.temperature, pd.minTemperature, pd.maxTemperature, pd.rh, pd.minRh, pd.maxRh "\ "FROM forecast f, parserData pd "\ "WHERE pd.parserID=? AND ?<=pd.timestamp AND pd.timestamp<=? AND pd.forecastID=f.ID ORDER BY pd.forecastID DESC", (parserID, dayTimestamp - 2 * 86400, dayTimestamp + 2 * 86400)) for row in rows: timestamp = rmGetStartOfDay(row[1]) if dayTimestamp <= timestamp < maxDayTimestamp: forecastTimestamp = rmGetStartOfDay(row[0]) if maxForecastTimestamp is None: minForecastTimestamp = forecastTimestamp maxForecastTimestamp = forecastTimestamp + 86400 if minForecastTimestamp <= forecastTimestamp < maxForecastTimestamp: minTemp = self.__val(row[3], row[2]) maxTemp = self.__val(row[4], row[2]) minRH = self.__val(row[6], row[5]) maxRH = self.__val(row[7], row[5]) results["minTemperature"] = self.__min(results["minTemperature"], minTemp) results["maxTemperature"] = self.__max(results["maxTemperature"], maxTemp) results["minRH"] = self.__min(results["minRH"], minRH) results["maxRH"] = self.__max(results["maxRH"], maxRH) return results
def perform(self): # Try to connect, and if it fail try to auto discover the device if self._connect() or self._discover(): # Successfully connected to the GW1000 device, let's retrieve live data live_data = self._get_live_data() if self.startOfDayTimestamp == 0: # First usage, initialization of the start of the day variable self.startOfDayTimestamp = rmGetStartOfDay( self.currentTimestamp) # Check if the live data is for a new day elif rmGetStartOfDay( self.currentTimestamp) != self.startOfDayTimestamp: # Report historical data of yesterday self._report_observations() # Reset the observations data for a new day self._reset_observations() # Parser live data and add observations self._parse_live_data(live_data) # A new observation, increment the observation counter self.observation_counter += 1 self._report_observations()
def __getSimpleForecast(self): try: tuple = datetime.datetime.fromtimestamp(int( time.time())).timetuple() dayTimestamp = int( datetime.datetime(tuple.tm_year, tuple.tm_mon, tuple.tm_mday).strftime("%s")) maxDayTimestamp = dayTimestamp + globalSettings.parserDataSizeInDays * 86400 simpleForecast = self.jsonResponse["forecast"]["simpleforecast"][ "forecastday"] timestamp = [] temperatureMax = [] temperatureMin = [] wind = [] humidity = [] qpf = [] condition = [] for dayF in simpleForecast: tt = self.__toInt(dayF["date"]["epoch"]) tt = rmGetStartOfDay(tt) if tt > maxDayTimestamp: break timestamp.append(self.__toInt(tt)) temperatureMax.append(self.__toFloat(dayF["high"]["celsius"])) temperatureMin.append(self.__toFloat(dayF["low"]["celsius"])) windValue = self.__toFloat(dayF["avewind"]["kph"]) if windValue is not None: wind.append(windValue / 3.6) # convertred from kmetersph to meterps humidity.append(self.__toFloat(dayF["avehumidity"])) qpf.append(self.__toFloat(dayF["qpf_allday"]["mm"])) condition.append(self.conditionConvert(dayF["conditions"])) temperatureMax = zip(timestamp, temperatureMax) temperatureMin = zip(timestamp, temperatureMin) wind = zip(timestamp, wind) humidity = zip(timestamp, humidity) qpf = zip(timestamp, qpf) condition = zip(timestamp, condition) self.addValues(RMParser.dataType.RH, humidity) self.addValues(RMParser.dataType.MAXTEMP, temperatureMax) self.addValues(RMParser.dataType.MINTEMP, temperatureMin) self.addValues(RMParser.dataType.QPF, qpf) self.addValues(RMParser.dataType.WIND, wind) self.addValues(RMParser.dataType.CONDITION, condition) except: log.error("Failed to get simple forecast")
def importFromSprinklerV1Db(self, filePath): if self.database.isOpen(): v1DB = RMDatabase(filePath) if not v1DB.open(): return False result = False try: data = OrderedDict() valuesToInsert = [] rows = v1DB.execute( "SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag FROM %s ORDER BY ts_started, zid" % self._tableName ) for row in rows: dayTimestamp = rmGetStartOfDay(int(row[0])) dayData = data.get(dayTimestamp, None) if dayData is None: dayData = {"token": uuid.uuid4().hex, "tokenTimestamp": dayTimestamp} data[dayTimestamp] = dayData valuesToInsert.append( [ row[0], row[1], row[2], row[3], row[4], row[5], row[6], dayData["token"], dayData["tokenTimestamp"], ] ) if valuesToInsert: self.database.execute("DELETE FROM %s" % self._tableName) self.database.executeMany( "INSERT INTO %s(ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp) " "VALUES(?,?,?,?,?,?,?,?,?)" % self._tableName, valuesToInsert, ) self.database.commit() result = True except Exception, e: log.exception(e) v1DB.close() return result
def getRecordsByParserName(self, parserName): results = OrderedDict() if self.database.isOpen(): #SELECT f.timestamp, f.processed, pd.* FROM parser p, forecast f, parserData pd WHERE p.name='ForecastIO Parser' AND p.id == pd.parserID AND f.id == pd.forecastID ORDER BY f.id DESC, pd.timestamp DESC; records = self.database.execute("SELECT f.timestamp, f.processed, pd.* FROM parser p, forecast f, parserData pd "\ "WHERE p.name=? AND p.id == pd.parserID AND f.id == pd.forecastID "\ "ORDER BY f.id DESC, pd.timestamp ASC", (parserName, )) for row in records: forecast = RMForecastInfo(row[2], row[0], row[1]) weatherData = RMWeatherData(row[4]) weatherData.temperature = row[5] weatherData.minTemperature = row[6] weatherData.maxTemperature = row[7] weatherData.rh = row[8] weatherData.minRh = row[9] weatherData.maxRh = row[10] weatherData.wind = row[11] weatherData.solarRad = row[12] weatherData.skyCover = row[13] weatherData.rain = row[14] weatherData.et0 = row[15] weatherData.pop = row[16] weatherData.qpf = row[17] weatherData.condition = row[18] weatherData.pressure = row[19] weatherData.dewPoint = row[20] weatherData.userData = row[21] dayTimestamp = rmGetStartOfDay(weatherData.timestamp) forecastValues = None if forecast in results: forecastValues = results[forecast] else: forecastValues = OrderedDict() results[forecast] = forecastValues dailyValues = None if dayTimestamp in forecastValues: dailyValues = forecastValues[dayTimestamp] else: dailyValues = [] forecastValues[dayTimestamp] = dailyValues dailyValues.append([weatherData, ]) return results
def importFromSprinklerV1Db(self, filePath): if (self.database.isOpen()): v1DB = RMDatabase(filePath) if not v1DB.open(): return False result = False try: data = OrderedDict() valuesToInsert = [] rows = v1DB.execute( "SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag FROM %s ORDER BY ts_started, zid" % self._tableName) for row in rows: dayTimestamp = rmGetStartOfDay(int(row[0])) dayData = data.get(dayTimestamp, None) if dayData is None: dayData = { "token": uuid.uuid4().hex, "tokenTimestamp": dayTimestamp } data[dayTimestamp] = dayData valuesToInsert.append([ row[0], row[1], row[2], row[3], row[4], row[5], row[6], dayData["token"], dayData["tokenTimestamp"] ]) if valuesToInsert: self.database.execute("DELETE FROM %s" % self._tableName) self.database.executeMany("INSERT INTO %s(ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp) "\ "VALUES(?,?,?,?,?,?,?,?,?)" % self._tableName, valuesToInsert) self.database.commit() result = True except Exception, e: log.exception(e) v1DB.close() return result
def deleteRecordsHistoryByDayThreshold(self, parserID, minDayTimestampThresold, maxDayTimestampThresold, commit = True): if(self.database.isOpen()): # Delete very old data rows = self.database.execute("DELETE FROM parserData WHERE timestamp<?", (minDayTimestampThresold, )) #SELECT f.ID, f.timestamp, p.rowid, p.timestamp, p.temperature, p.rh, p.wind, p.solarRad, p.skyCover, p.rain, p.et0, p.pop, p.qpf, p.condition, p.pressure, p.dewPoint, p.archived FROM parserData p, forecast f WHERE f.ID=p.forecastID ORDER BY p.timestamp DESC, p.forecastID DESC # Compute new data query = "SELECT f.ID, f.timestamp, p.rowid, p.timestamp, p.temperature, p.rh, p.wind, p.solarRad, p.skyCover, p.rain, p.et0, p.pop, p.qpf, p.condition, p.pressure, p.dewPoint, p.archived "\ "FROM parserData p, forecast f WHERE f.ID=p.forecastID AND p.parserID=? ORDER BY p.timestamp DESC, p.forecastID DESC" rows = self.database.execute(query, (parserID, )) tempData = OrderedDict() rowIdsToDelete = [] rowIdsToKeep = [] newData = [] lastDayTimestamp = None lastForecastID = None for row in rows: forecastID = row[0] forecastDayTimestamp = rmGetStartOfDay(row[1]) dayTimestamp = rmGetStartOfDay(row[3]) if lastDayTimestamp != dayTimestamp: # Enter a new day lastDayTimestamp = dayTimestamp lastForecastID = forecastID if maxDayTimestampThresold <= forecastDayTimestamp: continue if lastForecastID != forecastID: # We want only the last forecast for this day rowIdsToDelete.append(str(row[2])) continue timestamp = rmNormalizeTimestamp(row[3]) #timestampOffset = timestamp - dayTimestamp dayData = tempData.get(dayTimestamp, None) if dayData is None: tempData[dayTimestamp] = dayData = {} dayData["count"] = 0 dayData["data"] = { "rowid": row[2], "forecastID": forecastID, "condition": None, "archived": row[16] } rowIdsToKeep.append(str(row[2])) else: rowIdsToDelete.append(str(row[2])) dayData["count"] += 1 dayData = dayData["data"] dayData["temperature"] = self.__sum(dayData.get("temperature", None), row[4]) dayData["rh"] = self.__sum(dayData.get("rh", None), row[5]) dayData["wind"] = self.__sum(dayData.get("wind", None), row[6]) dayData["solarRad"] = self.__sum(dayData.get("solarRad", None), row[7]) dayData["skyCover"] = self.__sum(dayData.get("skyCover", None), row[8]) dayData["rain"] = self.__sum(dayData.get("rain", None), row[9]) dayData["et0"] = row[10] dayData["pop"] = self.__sum(dayData.get("pop", None), row[11]) dayData["qpf"] = self.__sum(dayData.get("qpf", None), row[12]) dayData["pressure"] = self.__sum(dayData.get("pressure", None), row[14]) dayData["dewPoint"] = self.__sum(dayData.get("dewPoint", None), row[15]) if row[13]: # and 43200 <= timestampOffset <= 50400: # between 12-14. dayData["condition"] = row[13] for dayTimestamp in tempData: dayData = tempData[dayTimestamp] count = dayData["count"] dayData = dayData["data"] if not dayData["archived"]: if count > 1: dayData["temperature"] = self.__avg(dayData.get("temperature", None), count) dayData["rh"] = self.__avg(dayData.get("rh", None), count) dayData["wind"] = self.__avg(dayData.get("wind", None), count) dayData["solarRad"] = self.__avg(dayData.get("solarRad", None), count) dayData["skyCover"] = self.__avg(dayData.get("skyCover", None), count) dayData["rain"] = self.__avg(dayData.get("rain", None), count) dayData["pop"] = self.__avg(dayData.get("pop", None), count) dayData["pressure"] = self.__avg(dayData.get("pressure", None), count) dayData["dewPoint"] = self.__avg(dayData.get("dewPoint", None), count) newData.append((dayTimestamp, dayData["temperature"], dayData["rh"], dayData["wind"], dayData["solarRad"], dayData["skyCover"], dayData["rain"], dayData["et0"], dayData["pop"], dayData["qpf"], dayData["condition"], dayData["pressure"], dayData["dewPoint"], dayData["rowid"])) # Delete unnecessary data if rowIdsToDelete: self.database.execute("DELETE FROM parserData WHERE rowid IN(%s)" % ",".join(rowIdsToDelete)) # Update computed data if newData: query = "UPDATE parserData SET timestamp=?, temperature=?, rh=?, wind=?, solarRad=?, skyCover=?, rain=?, et0=?, pop=?, qpf=?, condition=?, pressure=?, dewPoint=?, archived=1 WHERE rowid=?" self.database.executeMany(query, newData) self.database.execute("DELETE FROM forecast WHERE processed <> 0 AND ID NOT IN (SELECT DISTINCT forecastID FROM parserData)") if commit: self.database.commit()
def parseStationDataWithKey(self, jsonData): # daily summary for yesterday tsToday = rmCurrentDayTimestamp() tsYesterDay = rmDeltaDayFromTimestamp(tsToday, -1) l = RMWeatherDataLimits() try: dailysummary = jsonData['summaries'] for observation in dailysummary: tsDay = observation.get('epoch', None) tsDay = rmGetStartOfDay(tsDay) if tsDay == tsYesterDay: temperature = self.__toFloat( observation['metric']['tempAvg']) mintemp = self.__toFloat(observation['metric']['tempLow']) maxtemp = self.__toFloat(observation['metric']['tempHigh']) rh = self.__toFloat(observation["humidityAvg"]) minrh = self.__toFloat(observation["humidityLow"]) maxrh = self.__toFloat(observation["humidityHigh"]) dewpoint = self.__toFloat( observation['metric']["dewptAvg"]) wind = self.__toFloat( observation['metric']["windspeedAvg"]) if wind is not None: wind = wind / 3.6 # converted from kmetersph to mps maxpressure = self.__toFloat( observation['metric']["pressureMax"]) minpressure = self.__toFloat( observation['metric']["pressureMin"]) if maxpressure is not None: maxpressure = l.sanitize( RMWeatherDataType.PRESSURE, maxpressure / 10.0) # converted to from hpa to kpa if minpressure is not None: minpressure = l.sanitize(RMWeatherDataType.PRESSURE, minpressure / 10.0) pressure = None if maxpressure is not None and minpressure is not None: pressure = (maxpressure + minpressure) / 2.0 rain = self.__toFloat(observation['metric']["precipTotal"]) self.addValue(RMParser.dataType.TEMPERATURE, tsYesterDay, temperature, False) self.addValue(RMParser.dataType.MINTEMP, tsYesterDay, mintemp, False) self.addValue(RMParser.dataType.MAXTEMP, tsYesterDay, maxtemp, False) self.addValue(RMParser.dataType.RH, tsYesterDay, rh, False) self.addValue(RMParser.dataType.MINRH, tsYesterDay, minrh, False) self.addValue(RMParser.dataType.MAXRH, tsYesterDay, maxrh, False) self.addValue(RMParser.dataType.WIND, tsYesterDay, wind, False) self.addValue(RMParser.dataType.RAIN, tsYesterDay, rain, False) self.addValue(RMParser.dataType.DEWPOINT, tsYesterDay, dewpoint, False) self.addValue(RMParser.dataType.PRESSURE, tsYesterDay, pressure, False) return True return False except: self.lastKnownError = "Warning: Failed to get yesterday data summary" log.info(self.lastKnownError) return False
def _reset_observations(self): self.observations = dict.fromkeys(self.observations, None) self.observation_counter = 0 self.startOfDayTimestamp = rmGetStartOfDay(self.currentTimestamp)
def getRecordsEx(self, minTimestamp, maxTimestamp, withManualPrograms=False): if (self.database.isOpen()): #if withManualPrograms: # sqlCondition = " AND usersch_id != 0 " # sqlConditionForced = " WHERE usersch_id != 0" #else: # sqlCondition = "" # sqlConditionForced = "" if minTimestamp and maxTimestamp: records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "\ "WHERE ?<=tokenTimestamp AND tokenTimestamp<? GROUP BY tokenTimestamp "\ "ORDER BY tokenTimestamp" % self._tableName, (minTimestamp, maxTimestamp) ) elif minTimestamp: records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "\ "WHERE ?<=tokenTimestamp GROUP BY tokenTimestamp "\ "ORDER BY tokenTimestamp" % self._tableName, (minTimestamp, ) ) elif maxTimestamp: records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec), SUM(user_sec) userDuration duration, usersch_id FROM %s "\ "WHERE tokenTimestamp<? GROUP BY tokenTimestamp "\ "ORDER BY tokenTimestamp" % self._tableName, (maxTimestamp, ) ) else: records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "\ "GROUP BY tokenTimestamp ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName ) tempResults = OrderedDict() for row in records: if not withManualPrograms and int(row[3]) == 0: continue dayTimestamp = rmGetStartOfDay(int(row[0])) totalDurations = tempResults.get( dayTimestamp, None) # List with real and user durations if totalDurations is None: totalDurations = [int(row[1]), int(row[2])] else: totalDurations[0] += int(row[1]) totalDurations[1] += int(row[2]) tempResults[dayTimestamp] = totalDurations results = {"days": []} for dayTimestamp in tempResults: date = rmTimestampToDateAsString(dayTimestamp, "%Y-%m-%d") day = { "dayTimestamp": dayTimestamp, "date": date, "realDuration": tempResults.get(dayTimestamp)[0], "userDuration": tempResults.get(dayTimestamp)[1] } results["days"].append(day) return results return None
def getRecords(self, minTimestamp, maxTimestamp): if (self.database.isOpen()): if minTimestamp and maxTimestamp: records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\ "WHERE ?<=tokenTimestamp AND tokenTimestamp<? "\ "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (minTimestamp, maxTimestamp) ) elif minTimestamp: records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\ "WHERE ?<=tokenTimestamp "\ "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (minTimestamp, ) ) elif maxTimestamp: records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\ "WHERE tokenTimestamp<? "\ "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (maxTimestamp, ) ) else: records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\ "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName ) tempResults = OrderedDict() for row in records: token = row[7] dayTimestamp = rmGetStartOfDay(int(row[8])) dayGroup = tempResults.get(dayTimestamp, None) if dayGroup is None: dayGroup = OrderedDict() tempResults[dayTimestamp] = dayGroup programGroup = dayGroup.get(token, None) if programGroup is None: programGroup = OrderedDict() dayGroup[token] = programGroup zones = programGroup.get(row[1], None) if zones is None: zones = OrderedDict() programGroup[row[1]] = zones zone = zones.get(row[2], None) if zone is None: zone = OrderedDict() zone["uid"] = row[2] zone["flag"] = row[6] zone["cycles"] = [] zones[row[2]] = zone cycles = zone["cycles"] info = OrderedDict() info["id"] = len(cycles) + 1 info["startTime"] = rmTimestampToDateAsString(row[0]) info["startTimestamp"] = row[0] info["userDuration"] = row[3] info["machineDuration"] = row[4] info["realDuration"] = row[5] cycles.append(info) results = {"days": []} for dayTimestamp in tempResults: programs = [] day = { "date": rmTimestampToDateAsString(dayTimestamp, "%Y-%m-%d"), "dateTimestamp": dayTimestamp, "programs": programs } results["days"].append(day) dayGroup = tempResults[dayTimestamp] for token in dayGroup: tempPrograms = dayGroup[token] for programId in tempPrograms: zones = [] program = OrderedDict() program["id"] = programId program["zones"] = zones programs.append(program) tempZones = tempPrograms[programId] for zoneId in tempZones: zones.append(tempZones[zoneId]) return results return None
def getRecordsEx(self, minTimestamp, maxTimestamp, withManualPrograms=False): if self.database.isOpen(): # if withManualPrograms: # sqlCondition = " AND usersch_id != 0 " # sqlConditionForced = " WHERE usersch_id != 0" # else: # sqlCondition = "" # sqlConditionForced = "" if minTimestamp and maxTimestamp: records = self.database.execute( "SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s " "WHERE ?<=tokenTimestamp AND tokenTimestamp<? GROUP BY tokenTimestamp " "ORDER BY tokenTimestamp" % self._tableName, (minTimestamp, maxTimestamp), ) elif minTimestamp: records = self.database.execute( "SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s " "WHERE ?<=tokenTimestamp GROUP BY tokenTimestamp " "ORDER BY tokenTimestamp" % self._tableName, (minTimestamp,), ) elif maxTimestamp: records = self.database.execute( "SELECT tokenTimestamp, SUM(real_sec), SUM(user_sec) userDuration duration, usersch_id FROM %s " "WHERE tokenTimestamp<? GROUP BY tokenTimestamp " "ORDER BY tokenTimestamp" % self._tableName, (maxTimestamp,), ) else: records = self.database.execute( "SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s " "GROUP BY tokenTimestamp ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName ) tempResults = OrderedDict() for row in records: if not withManualPrograms and int(row[3]) == 0: continue dayTimestamp = rmGetStartOfDay(int(row[0])) totalDurations = tempResults.get(dayTimestamp, None) # List with real and user durations if totalDurations is None: totalDurations = [int(row[1]), int(row[2])] else: totalDurations[0] += int(row[1]) totalDurations[1] += int(row[2]) tempResults[dayTimestamp] = totalDurations results = {"days": []} for dayTimestamp in tempResults: date = rmTimestampToDateAsString(dayTimestamp, "%Y-%m-%d") day = { "dayTimestamp": dayTimestamp, "date": date, "realDuration": tempResults.get(dayTimestamp)[0], "userDuration": tempResults.get(dayTimestamp)[1], } results["days"].append(day) return results return None
def getRecords(self, minTimestamp, maxTimestamp): if self.database.isOpen(): if minTimestamp and maxTimestamp: records = self.database.execute( "SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s " "WHERE ?<=tokenTimestamp AND tokenTimestamp<? " "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (minTimestamp, maxTimestamp), ) elif minTimestamp: records = self.database.execute( "SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s " "WHERE ?<=tokenTimestamp " "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (minTimestamp,), ) elif maxTimestamp: records = self.database.execute( "SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s " "WHERE tokenTimestamp<? " "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (maxTimestamp,), ) else: records = self.database.execute( "SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s " "ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName ) tempResults = OrderedDict() for row in records: token = row[7] dayTimestamp = rmGetStartOfDay(int(row[8])) dayGroup = tempResults.get(dayTimestamp, None) if dayGroup is None: dayGroup = OrderedDict() tempResults[dayTimestamp] = dayGroup programGroup = dayGroup.get(token, None) if programGroup is None: programGroup = OrderedDict() dayGroup[token] = programGroup zones = programGroup.get(row[1], None) if zones is None: zones = OrderedDict() programGroup[row[1]] = zones zone = zones.get(row[2], None) if zone is None: zone = OrderedDict() zone["uid"] = row[2] zone["flag"] = row[6] zone["cycles"] = [] zones[row[2]] = zone cycles = zone["cycles"] info = OrderedDict() info["id"] = len(cycles) + 1 info["startTime"] = rmTimestampToDateAsString(row[0]) info["startTimestamp"] = row[0] info["userDuration"] = row[3] info["machineDuration"] = row[4] info["realDuration"] = row[5] cycles.append(info) results = {"days": []} for dayTimestamp in tempResults: programs = [] day = { "date": rmTimestampToDateAsString(dayTimestamp, "%Y-%m-%d"), "dateTimestamp": dayTimestamp, "programs": programs, } results["days"].append(day) dayGroup = tempResults[dayTimestamp] for token in dayGroup: tempPrograms = dayGroup[token] for programId in tempPrograms: zones = [] program = OrderedDict() program["id"] = programId program["zones"] = zones programs.append(program) tempZones = tempPrograms[programId] for zoneId in tempZones: zones.append(tempZones[zoneId]) return results return None
def perform(self): station = self.params.get("station", None) if station is None or station == "": station = "10637" log.debug("No station set, using Frankfurt am Main (%s)" % station) url = "http://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/single_stations/" + str( station) + "/kml/MOSMIX_L_LATEST_" + str(station) + ".kmz" try: datafile = self.openURL(url) if datafile is None: self.lastKnownError = "Cannot read data from DWD Service." return else: log.debug("Successfully loaded the KML file") kmz = zipfile.ZipFile(BufferedRandomReader(datafile), 'r') for name in kmz.namelist(): kml = kmz.read(name) root = ElementTree.fromstring(kml) ns = { 'xmlns': "http://www.opengis.net/kml/2.2", 'dwd': 'https://opendata.dwd.de/weather/lib/pointforecast_dwd_extension_V1_0.xsd' } # temporary storage of forecast values tmp = dict() forecastDict = dict() timestamps = [] forecasts = dict() # Find all forecasts for element in root.findall('.//dwd:Forecast', ns): forecasts.update({ element.attrib['{https://opendata.dwd.de/weather/lib/pointforecast_dwd_extension_V1_0.xsd}elementName']: element[0].text.split() }) # Find all timestamps for element in root.findall('.//dwd:TimeStep', namespaces=ns): timestamps.append(element.text) for timestep in timestamps: for measure, values in forecasts.iteritems(): tmp.update({measure: values.pop(0)}) forecastDict.update({timestep: dict(tmp)}) # Add retreived data to DB for time, forecast in forecastDict.iteritems(): timestamp = rmTimestampFromDateAsString( time[:-5], "%Y-%m-%dT%H:%M:%S") yesterdayTimestamp = rmGetStartOfDay(timestamp - 12 * 60 * 60) if timestamp is None: log.debug( "Cannot convert timestamp: %s to unix timestamp" % time) continue # Temperature if forecast['TTT'] != '-': TTT = float(forecast['TTT']) - 273.15 self.addValue(RMParser.dataType.TEMPERATURE, timestamp, TTT) # Minimum temperature last 24h if forecast['TN'] != '-': TN = float(forecast['TN']) - 273.15 self.addValue(RMParser.dataType.MINTEMP, timestamp - 12 * 60 * 60, TN) # Maximum temperature last 24h if forecast['TX'] != '-': TX = float(forecast['TX']) - 273.15 self.addValue(RMParser.dataType.MINTEMP, timestamp - 12 * 60 * 60, TX) # Windspeed if forecast['FF'] != '-': FF = float(forecast['FF']) self.addValue(RMParser.dataType.WIND, timestamp, FF) # Precipation last 24h if forecast['RRdc'] != '-': RRdc = float(forecast['RRdc']) self.addValue(RMParser.dataType.QPF, yesterdayTimestamp, RRdc) # Atmospheric pressure if forecast['PPPP'] != '-': PPPP = float(forecast['PPPP']) / 1000 self.addValue(RMParser.dataType.PRESSURE, timestamp, PPPP) # Dewpoint if forecast['Td'] != '-': Td = float(forecast['Td']) - 273.15 self.addValue(RMParser.dataType.DEWPOINT, timestamp, Td) except Exception as e: log.error("*** Error running DWD parser") log.exception(e)
def wfUDPData(self): air_count = 0 sky_count = 0 temp_total = 0 humd_total = 0 pres_total = 0 wind_total = 0 srad_total = 0 rain_total = 0 dewp_total = 0 bufferSize = 1024 # whatever you need port = 50222 day_of_year = 0 log.debug("Start listening on port %d" % port) try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(('0.0.0.0', port)) except: log.Error("Socket failure") log.debug("Start recieve loop") while self.started: hub = s.recvfrom(bufferSize) data = json.loads(hub[0]) # hub is a truple (json, ip, port) now = datetime.now() # Check if this is a new day, if so if day_of_year != now.timetuple().tm_yday: # clear counters air_count = 0 sky_count = 0 temp_total = 0 humd_total = 0 pres_total = 0 wind_total = 0 srad_total = 0 rain_total = 0 dewp_total = 0 self.report = { 'temperature': 0, 'humidity': 0, 'pressure': 0, 'dewpoint': 0, 'wind': 0, 'srad': 0, 'rain': 0, 'max_temp': -100, 'min_temp': 100, 'max_humid': 0, 'min_humid': 100 } day_of_year = now.timetuple().tm_yday self.parserData[1] = self.parserData[0] self.newDay += 1 # signal that just rolled into a new day, need to send a yesterday summary onetime # reset yesterday's timestamp to start of day if 'ts' in self.parserData[1]: self.parserData[1]['ts'] = rmGetStartOfDay( self.parserData[1]['ts']) #log.debug("type = %s broadcast s/n = %s TEMPEST target: %s" % (data["type"], data["serial_number"], self.params["TempestSerialNum"])) debugMsg = "Observation: " if (((data["type"] == "obs_air") and (data["serial_number"] == self.params["AirSerialNumber"])) or ((data["type"] == "obs_st") and (data["serial_number"] == self.params["TempestSerialNum"]))): if data["type"] == "obs_air": pres_idx = 1 # UDP packet data indexes for Air device temp_idx = 2 humd_idx = 3 else: pres_idx = 6 # UDP packet data indexes for TEMPEST device temp_idx = 7 humd_idx = 8 air_count += 1 temp_total += data["obs"][0][temp_idx] self.report["temperature"] = float(temp_total) / float( air_count) humd_total += data["obs"][0][humd_idx] self.report["humidity"] = float(humd_total) / float(air_count) # report pressure in hpa so convert from mb to hpa pres_total += (data["obs"][0][pres_idx] / 10.0) self.report["pressure"] = float(pres_total) / float(air_count) # Calculate dewpoint b = (17.625 * data["obs"][0][temp_idx]) / ( 243.04 + data["obs"][0][temp_idx]) rh = float(data["obs"][0][humd_idx]) / 100.0 c = math.log(rh) dewpoint = (243.04 * (c + b)) / (17.625 - c - b) dewp_total += dewpoint self.report["dewpoint"] = dewp_total / air_count # Track Min/Max if (data["obs"][0][temp_idx] > self.report["max_temp"]): self.report["max_temp"] = data["obs"][0][temp_idx] if (data["obs"][0][temp_idx] < self.report["min_temp"]): self.report["min_temp"] = data["obs"][0][temp_idx] if (data["obs"][0][humd_idx] > self.report["max_humid"]): self.report["max_humid"] = data["obs"][0][humd_idx] if (data["obs"][0][humd_idx] < self.report["min_humid"]): self.report["min_humid"] = data["obs"][0][humd_idx] debugMsg += "Temp (dF)= %.2f, " % ( (float(data["obs"][0][temp_idx]) * 9 / 5) + 32 ) # convert degC to degF debugMsg += "Humid = %.2f, " % data["obs"][0][humd_idx] debugMsg += "Press (inHg) = %.2f, " % ( float(data["obs"][0][pres_idx]) / 33.8639) if (((data["type"] == "obs_sky") and (data["serial_number"] == self.params["SkySerialNumber"])) or ((data["type"] == "obs_st") and (data["serial_number"] == self.params["TempestSerialNum"]))): if data["type"] == "obs_sky": wind_idx = 5 # UDP Packet data indexes for Sky device srad_idx = 10 rain_idx = 11 else: wind_idx = 2 # UDP packet data indexes for TEMPEST device srad_idx = 11 rain_idx = 12 sky_count += 1 wind_total += data["obs"][0][wind_idx] self.report["wind"] = float(wind_total) / float(sky_count) srad_total += data["obs"][0][srad_idx] self.report["srad"] = float(srad_total) / float(sky_count) rain_total += data["obs"][0][rain_idx] self.report["rain"] = rain_total debugMsg += "Cumulative Rain (inch) = %.2f" % (rain_total / 25.4) log.debug(debugMsg) ts = mod_time.mktime(now.timetuple()) + now.microsecond / 1e6 self.parserData[0] = {'ts': ts, 'report': self.report} log.debug("Receive thread exiting") s.close() self.started = False
def wfUDPData(self): air_count = 0 sky_count = 0 temp_total = 0 humd_total = 0 pres_total = 0 wind_total = 0 srad_total = 0 rain_total = 0 dewp_total = 0 prev_air_hour = 0 prev_sky_hour = 0 prev_air_day = 0 prev_sky_day = 0 bufferSize = 1024 # whatever you need port = 50222 day_of_year = 0 log.debug("Start listening on port %d" % port) try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(('0.0.0.0', port)) except: log.Error("Socket failure") log.debug("Start recieve loop") while self.started: hub = s.recvfrom(bufferSize) data = json.loads(hub[0]) # hub is a truple (json, ip, port) now = datetime.now() # Check if this is a new day, if so if day_of_year != now.timetuple().tm_yday: # clear counters air_count = 0 sky_count = 0 temp_total = 0 humd_total = 0 pres_total = 0 dewp_total = 0 srad_total = 0 rain_total = 0 self.report = { 'temperature': 0, 'humidity': 0, 'pressure': 0, 'dewpoint': 0, 'wind': 0, 'srad': 0, 'rain': 0, 'max_temp': -100, 'min_temp': 100, 'max_humid': 0, 'min_humid': 100 } day_of_year = now.timetuple().tm_yday self.parserData[1] = self.parserData[0] # reset yesterday's timestamp to start of day if 'ts' in self.parserData[1]: self.parserData[1]['ts'] = rmGetStartOfDay(self.parserData[1]['ts']) #log.debug("type = %s s/n = %s" % (data["type"], data["serial_number"])) if (data["type"] == "obs_air") and (data["serial_number"] == self.params["AirSerialNumber"]): air_count += 1 temp_total += data["obs"][0][2] self.report["temperature"] = float(temp_total) / float(air_count) humd_total += data["obs"][0][3] self.report["humidity"] = float(humd_total) / float(air_count) # report pressure in hpa so convert from mb to hpa pres_total += (data["obs"][0][1] / 10.0) self.report["pressure"] = float(pres_total) / float(air_count) # Calculate dewpoint b = (17.625 * data["obs"][0][2]) / (243.04 + data["obs"][0][2]) rh = float(data["obs"][0][3]) / 100.0 c = math.log(rh) dewpoint = (243.04 * (c + b)) / (17.625 - c - b) dewp_total += dewpoint self.report["dewpoint"] = dewp_total / air_count # Track Min/Max if (data["obs"][0][2] > self.report["max_temp"]): self.report["max_temp"] = data["obs"][0][2] if (data["obs"][0][2] < self.report["min_temp"]): self.report["min_temp"] = data["obs"][0][2] if (data["obs"][0][3] > self.report["max_humid"]): self.report["max_humid"] = data["obs"][0][3] if (data["obs"][0][3] < self.report["min_humid"]): self.report["min_humid"] = data["obs"][0][3] if (data["type"] == "obs_sky") and (data["serial_number"] == self.params["SkySerialNumber"]): sky_count += 1 wind_total += data["obs"][0][5] self.report["wind"] = float(wind_total) / float(sky_count) srad_total += data["obs"][0][10] self.report["srad"] = float(srad_total) / float(sky_count) rain_total += data["obs"][0][3] self.report["rain"] = rain_total ts = mod_time.mktime(now.timetuple()) + now.microsecond / 1e6 self.parserData[0] = { 'ts': ts, 'report':self.report } log.debug("Receive thread exiting") s.close() self.started = False