def addRecords(self, forecastID, parserID, values):
if(self.database.isOpen()):
minMaxMap = {}
for value in values:
dayTimestamp = rmGetStartOfDay(value.timestamp)
minMax = minMaxMap.get(dayTimestamp)
if minMax is None:
minMax = self.getMinMax(parserID, dayTimestamp)
minMaxMap[dayTimestamp] = minMax
minMax["minTemperature"] = self.__min(self.__min(value.minTemperature, value.temperature), minMax["minTemperature"])
minMax["maxTemperature"] = self.__max(self.__max(value.maxTemperature, value.temperature), minMax["maxTemperature"])
minMax["minRH"] = self.__min(self.__min(value.minRh, value.rh), minMax["minRH"])
minMax["maxRH"] = self.__max(self.__min(value.maxRh, value.rh), minMax["maxRH"])
valuesToInsert = []
for value in values:
dayTimestamp = rmGetStartOfDay(value.timestamp)
minMax = minMaxMap[dayTimestamp]
valuesToInsert.append((forecastID, parserID,
value.timestamp,
value.temperature,
minMax["minTemperature"],
minMax["maxTemperature"],
value.rh,
minMax["minRH"],
minMax["maxRH"],
value.wind,
value.solarRad,
value.skyCover,
value.rain,
value.et0,
value.pop,
value.qpf,
value.condition,
value.pressure,
value.dewPoint,
value.userData))
self.clearHistory(parserID, False)
self.database.executeMany("INSERT INTO parserData(forecastID, parserID, timestamp, "\
"temperature, minTemperature, maxTemperature, rh, minRh, maxRh, "\
"wind, solarRad, skyCover, rain, et0, pop, qpf, "\
"condition, pressure, dewPoint, userData) "\
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", valuesToInsert)
self.database.commit()
def parseStationYesterdayDataNoKey(self, data):
#daily summary for yesterday
try:
l = RMWeatherDataLimits()
temperature = self.__toFloat(data["TemperatureAvgC"])
mintemp = self.__toFloat(data["TemperatureLowC"])
maxtemp = self.__toFloat(data["TemperatureHighC"])
rh = self.__toFloat(data["HumidityAvg"])
minrh = self.__toFloat(data["HumidityLow"])
maxrh = self.__toFloat(data["HumidityHigh"])
dewpoint = self.__toFloat(data["DewpointAvgC"])
wind = self.__toFloat(data["WindSpeedAvgKMH"])
maxpressure = self.__toFloat(data["PressureMaxhPa"])
minpressure = self.__toFloat(data["PressureMinhPa"])
rain = self.__toFloat(
data["PrecipitationSumCM"]) * 10.0 # from cm to mm
if wind is not None:
wind = wind / 3.6 # converted from kmetersph to mps
if maxpressure is not None:
maxpressure = l.sanitize(RMWeatherDataType.PRESSURE,
maxpressure /
10.0) # converted to from hpa to kpa
if minpressure is not None:
minpressure = l.sanitize(RMWeatherDataType.PRESSURE,
minpressure / 10.0)
pressure = None
if maxpressure is not None and minpressure is not None:
pressure = (maxpressure + minpressure) / 2.0
#log.info("rh:%s minrh: %s maxrh: %s pressure: %s temp: %s mintemp: %s maxtemp: %s" % (rh, minrh, maxrh, pressure, temperature, mintemp, maxtemp))
timestamp = rmCurrentDayTimestamp()
timestamp = rmGetStartOfDay(timestamp - 12 * 3600)
self.addValue(RMParser.dataType.TEMPERATURE, timestamp,
temperature, False)
self.addValue(RMParser.dataType.MINTEMP, timestamp, mintemp, False)
self.addValue(RMParser.dataType.MAXTEMP, timestamp, maxtemp, False)
self.addValue(RMParser.dataType.RH, timestamp, rh, False)
self.addValue(RMParser.dataType.MINRH, timestamp, minrh, False)
self.addValue(RMParser.dataType.MAXRH, timestamp, maxrh, False)
self.addValue(RMParser.dataType.WIND, timestamp, wind, False)
self.addValue(RMParser.dataType.RAIN, timestamp, rain, False)
self.addValue(RMParser.dataType.DEWPOINT, timestamp, dewpoint,
False)
self.addValue(RMParser.dataType.PRESSURE, timestamp, pressure,
False)
except Exception, e:
self.lastKnownError = "ERROR: Failed to get historical data"
log.error("%s: %s" % (self.lastKnownError, e))
def getMinMax(self, parserID, dayTimestamp):
### Min and Max are computed only from the last forecast for that day.
results = {
"minTemperature": None,
"maxTemperature": None,
"minRH": None,
"maxRH": None
}
if self.database.isOpen():
minForecastTimestamp = None
maxForecastTimestamp = None
maxDayTimestamp = dayTimestamp + 86400
rows = self.database.execute("SELECT f.timestamp, pd.timestamp, pd.temperature, pd.minTemperature, pd.maxTemperature, pd.rh, pd.minRh, pd.maxRh "\
"FROM forecast f, parserData pd "\
"WHERE pd.parserID=? AND ?<=pd.timestamp AND pd.timestamp<=? AND pd.forecastID=f.ID ORDER BY pd.forecastID DESC",
(parserID, dayTimestamp - 2 * 86400, dayTimestamp + 2 * 86400))
for row in rows:
timestamp = rmGetStartOfDay(row[1])
if dayTimestamp <= timestamp < maxDayTimestamp:
forecastTimestamp = rmGetStartOfDay(row[0])
if maxForecastTimestamp is None:
minForecastTimestamp = forecastTimestamp
maxForecastTimestamp = forecastTimestamp + 86400
if minForecastTimestamp <= forecastTimestamp < maxForecastTimestamp:
minTemp = self.__val(row[3], row[2])
maxTemp = self.__val(row[4], row[2])
minRH = self.__val(row[6], row[5])
maxRH = self.__val(row[7], row[5])
results["minTemperature"] = self.__min(results["minTemperature"], minTemp)
results["maxTemperature"] = self.__max(results["maxTemperature"], maxTemp)
results["minRH"] = self.__min(results["minRH"], minRH)
results["maxRH"] = self.__max(results["maxRH"], maxRH)
return results
def perform(self):
# Try to connect, and if it fail try to auto discover the device
if self._connect() or self._discover():
# Successfully connected to the GW1000 device, let's retrieve live data
live_data = self._get_live_data()
if self.startOfDayTimestamp == 0:
# First usage, initialization of the start of the day variable
self.startOfDayTimestamp = rmGetStartOfDay(
self.currentTimestamp)
# Check if the live data is for a new day
elif rmGetStartOfDay(
self.currentTimestamp) != self.startOfDayTimestamp:
# Report historical data of yesterday
self._report_observations()
# Reset the observations data for a new day
self._reset_observations()
# Parser live data and add observations
self._parse_live_data(live_data)
# A new observation, increment the observation counter
self.observation_counter += 1
self._report_observations()
def __getSimpleForecast(self):
try:
tuple = datetime.datetime.fromtimestamp(int(
time.time())).timetuple()
dayTimestamp = int(
datetime.datetime(tuple.tm_year, tuple.tm_mon,
tuple.tm_mday).strftime("%s"))
maxDayTimestamp = dayTimestamp + globalSettings.parserDataSizeInDays * 86400
simpleForecast = self.jsonResponse["forecast"]["simpleforecast"][
"forecastday"]
timestamp = []
temperatureMax = []
temperatureMin = []
wind = []
humidity = []
qpf = []
condition = []
for dayF in simpleForecast:
tt = self.__toInt(dayF["date"]["epoch"])
tt = rmGetStartOfDay(tt)
if tt > maxDayTimestamp:
break
timestamp.append(self.__toInt(tt))
temperatureMax.append(self.__toFloat(dayF["high"]["celsius"]))
temperatureMin.append(self.__toFloat(dayF["low"]["celsius"]))
windValue = self.__toFloat(dayF["avewind"]["kph"])
if windValue is not None:
wind.append(windValue /
3.6) # convertred from kmetersph to meterps
humidity.append(self.__toFloat(dayF["avehumidity"]))
qpf.append(self.__toFloat(dayF["qpf_allday"]["mm"]))
condition.append(self.conditionConvert(dayF["conditions"]))
temperatureMax = zip(timestamp, temperatureMax)
temperatureMin = zip(timestamp, temperatureMin)
wind = zip(timestamp, wind)
humidity = zip(timestamp, humidity)
qpf = zip(timestamp, qpf)
condition = zip(timestamp, condition)
self.addValues(RMParser.dataType.RH, humidity)
self.addValues(RMParser.dataType.MAXTEMP, temperatureMax)
self.addValues(RMParser.dataType.MINTEMP, temperatureMin)
self.addValues(RMParser.dataType.QPF, qpf)
self.addValues(RMParser.dataType.WIND, wind)
self.addValues(RMParser.dataType.CONDITION, condition)
except:
log.error("Failed to get simple forecast")
def importFromSprinklerV1Db(self, filePath):
if self.database.isOpen():
v1DB = RMDatabase(filePath)
if not v1DB.open():
return False
result = False
try:
data = OrderedDict()
valuesToInsert = []
rows = v1DB.execute(
"SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag FROM %s ORDER BY ts_started, zid"
% self._tableName
)
for row in rows:
dayTimestamp = rmGetStartOfDay(int(row[0]))
dayData = data.get(dayTimestamp, None)
if dayData is None:
dayData = {"token": uuid.uuid4().hex, "tokenTimestamp": dayTimestamp}
data[dayTimestamp] = dayData
valuesToInsert.append(
[
row[0],
row[1],
row[2],
row[3],
row[4],
row[5],
row[6],
dayData["token"],
dayData["tokenTimestamp"],
]
)
if valuesToInsert:
self.database.execute("DELETE FROM %s" % self._tableName)
self.database.executeMany(
"INSERT INTO %s(ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp) "
"VALUES(?,?,?,?,?,?,?,?,?)" % self._tableName,
valuesToInsert,
)
self.database.commit()
result = True
except Exception, e:
log.exception(e)
v1DB.close()
return result
def getRecordsByParserName(self, parserName):
results = OrderedDict()
if self.database.isOpen():
#SELECT f.timestamp, f.processed, pd.* FROM parser p, forecast f, parserData pd WHERE p.name='ForecastIO Parser' AND p.id == pd.parserID AND f.id == pd.forecastID ORDER BY f.id DESC, pd.timestamp DESC;
records = self.database.execute("SELECT f.timestamp, f.processed, pd.* FROM parser p, forecast f, parserData pd "\
"WHERE p.name=? AND p.id == pd.parserID AND f.id == pd.forecastID "\
"ORDER BY f.id DESC, pd.timestamp ASC", (parserName, ))
for row in records:
forecast = RMForecastInfo(row[2], row[0], row[1])
weatherData = RMWeatherData(row[4])
weatherData.temperature = row[5]
weatherData.minTemperature = row[6]
weatherData.maxTemperature = row[7]
weatherData.rh = row[8]
weatherData.minRh = row[9]
weatherData.maxRh = row[10]
weatherData.wind = row[11]
weatherData.solarRad = row[12]
weatherData.skyCover = row[13]
weatherData.rain = row[14]
weatherData.et0 = row[15]
weatherData.pop = row[16]
weatherData.qpf = row[17]
weatherData.condition = row[18]
weatherData.pressure = row[19]
weatherData.dewPoint = row[20]
weatherData.userData = row[21]
dayTimestamp = rmGetStartOfDay(weatherData.timestamp)
forecastValues = None
if forecast in results:
forecastValues = results[forecast]
else:
forecastValues = OrderedDict()
results[forecast] = forecastValues
dailyValues = None
if dayTimestamp in forecastValues:
dailyValues = forecastValues[dayTimestamp]
else:
dailyValues = []
forecastValues[dayTimestamp] = dailyValues
dailyValues.append([weatherData, ])
return results
def importFromSprinklerV1Db(self, filePath):
if (self.database.isOpen()):
v1DB = RMDatabase(filePath)
if not v1DB.open():
return False
result = False
try:
data = OrderedDict()
valuesToInsert = []
rows = v1DB.execute(
"SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag FROM %s ORDER BY ts_started, zid"
% self._tableName)
for row in rows:
dayTimestamp = rmGetStartOfDay(int(row[0]))
dayData = data.get(dayTimestamp, None)
if dayData is None:
dayData = {
"token": uuid.uuid4().hex,
"tokenTimestamp": dayTimestamp
}
data[dayTimestamp] = dayData
valuesToInsert.append([
row[0], row[1], row[2], row[3], row[4], row[5], row[6],
dayData["token"], dayData["tokenTimestamp"]
])
if valuesToInsert:
self.database.execute("DELETE FROM %s" % self._tableName)
self.database.executeMany("INSERT INTO %s(ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp) "\
"VALUES(?,?,?,?,?,?,?,?,?)" % self._tableName, valuesToInsert)
self.database.commit()
result = True
except Exception, e:
log.exception(e)
v1DB.close()
return result
def deleteRecordsHistoryByDayThreshold(self, parserID, minDayTimestampThresold, maxDayTimestampThresold, commit = True):
if(self.database.isOpen()):
# Delete very old data
rows = self.database.execute("DELETE FROM parserData WHERE timestamp<?", (minDayTimestampThresold, ))
#SELECT f.ID, f.timestamp, p.rowid, p.timestamp, p.temperature, p.rh, p.wind, p.solarRad, p.skyCover, p.rain, p.et0, p.pop, p.qpf, p.condition, p.pressure, p.dewPoint, p.archived FROM parserData p, forecast f WHERE f.ID=p.forecastID ORDER BY p.timestamp DESC, p.forecastID DESC
# Compute new data
query = "SELECT f.ID, f.timestamp, p.rowid, p.timestamp, p.temperature, p.rh, p.wind, p.solarRad, p.skyCover, p.rain, p.et0, p.pop, p.qpf, p.condition, p.pressure, p.dewPoint, p.archived "\
"FROM parserData p, forecast f WHERE f.ID=p.forecastID AND p.parserID=? ORDER BY p.timestamp DESC, p.forecastID DESC"
rows = self.database.execute(query, (parserID, ))
tempData = OrderedDict()
rowIdsToDelete = []
rowIdsToKeep = []
newData = []
lastDayTimestamp = None
lastForecastID = None
for row in rows:
forecastID = row[0]
forecastDayTimestamp = rmGetStartOfDay(row[1])
dayTimestamp = rmGetStartOfDay(row[3])
if lastDayTimestamp != dayTimestamp: # Enter a new day
lastDayTimestamp = dayTimestamp
lastForecastID = forecastID
if maxDayTimestampThresold <= forecastDayTimestamp:
continue
if lastForecastID != forecastID: # We want only the last forecast for this day
rowIdsToDelete.append(str(row[2]))
continue
timestamp = rmNormalizeTimestamp(row[3])
#timestampOffset = timestamp - dayTimestamp
dayData = tempData.get(dayTimestamp, None)
if dayData is None:
tempData[dayTimestamp] = dayData = {}
dayData["count"] = 0
dayData["data"] = {
"rowid": row[2],
"forecastID": forecastID,
"condition": None,
"archived": row[16]
}
rowIdsToKeep.append(str(row[2]))
else:
rowIdsToDelete.append(str(row[2]))
dayData["count"] += 1
dayData = dayData["data"]
dayData["temperature"] = self.__sum(dayData.get("temperature", None), row[4])
dayData["rh"] = self.__sum(dayData.get("rh", None), row[5])
dayData["wind"] = self.__sum(dayData.get("wind", None), row[6])
dayData["solarRad"] = self.__sum(dayData.get("solarRad", None), row[7])
dayData["skyCover"] = self.__sum(dayData.get("skyCover", None), row[8])
dayData["rain"] = self.__sum(dayData.get("rain", None), row[9])
dayData["et0"] = row[10]
dayData["pop"] = self.__sum(dayData.get("pop", None), row[11])
dayData["qpf"] = self.__sum(dayData.get("qpf", None), row[12])
dayData["pressure"] = self.__sum(dayData.get("pressure", None), row[14])
dayData["dewPoint"] = self.__sum(dayData.get("dewPoint", None), row[15])
if row[13]: # and 43200 <= timestampOffset <= 50400: # between 12-14.
dayData["condition"] = row[13]
for dayTimestamp in tempData:
dayData = tempData[dayTimestamp]
count = dayData["count"]
dayData = dayData["data"]
if not dayData["archived"]:
if count > 1:
dayData["temperature"] = self.__avg(dayData.get("temperature", None), count)
dayData["rh"] = self.__avg(dayData.get("rh", None), count)
dayData["wind"] = self.__avg(dayData.get("wind", None), count)
dayData["solarRad"] = self.__avg(dayData.get("solarRad", None), count)
dayData["skyCover"] = self.__avg(dayData.get("skyCover", None), count)
dayData["rain"] = self.__avg(dayData.get("rain", None), count)
dayData["pop"] = self.__avg(dayData.get("pop", None), count)
dayData["pressure"] = self.__avg(dayData.get("pressure", None), count)
dayData["dewPoint"] = self.__avg(dayData.get("dewPoint", None), count)
newData.append((dayTimestamp, dayData["temperature"], dayData["rh"], dayData["wind"], dayData["solarRad"],
dayData["skyCover"], dayData["rain"], dayData["et0"], dayData["pop"], dayData["qpf"],
dayData["condition"], dayData["pressure"], dayData["dewPoint"], dayData["rowid"]))
# Delete unnecessary data
if rowIdsToDelete:
self.database.execute("DELETE FROM parserData WHERE rowid IN(%s)" % ",".join(rowIdsToDelete))
# Update computed data
if newData:
query = "UPDATE parserData SET timestamp=?, temperature=?, rh=?, wind=?, solarRad=?, skyCover=?, rain=?, et0=?, pop=?, qpf=?, condition=?, pressure=?, dewPoint=?, archived=1 WHERE rowid=?"
self.database.executeMany(query, newData)
self.database.execute("DELETE FROM forecast WHERE processed <> 0 AND ID NOT IN (SELECT DISTINCT forecastID FROM parserData)")
if commit:
self.database.commit()
def parseStationDataWithKey(self, jsonData):
# daily summary for yesterday
tsToday = rmCurrentDayTimestamp()
tsYesterDay = rmDeltaDayFromTimestamp(tsToday, -1)
l = RMWeatherDataLimits()
try:
dailysummary = jsonData['summaries']
for observation in dailysummary:
tsDay = observation.get('epoch', None)
tsDay = rmGetStartOfDay(tsDay)
if tsDay == tsYesterDay:
temperature = self.__toFloat(
observation['metric']['tempAvg'])
mintemp = self.__toFloat(observation['metric']['tempLow'])
maxtemp = self.__toFloat(observation['metric']['tempHigh'])
rh = self.__toFloat(observation["humidityAvg"])
minrh = self.__toFloat(observation["humidityLow"])
maxrh = self.__toFloat(observation["humidityHigh"])
dewpoint = self.__toFloat(
observation['metric']["dewptAvg"])
wind = self.__toFloat(
observation['metric']["windspeedAvg"])
if wind is not None:
wind = wind / 3.6 # converted from kmetersph to mps
maxpressure = self.__toFloat(
observation['metric']["pressureMax"])
minpressure = self.__toFloat(
observation['metric']["pressureMin"])
if maxpressure is not None:
maxpressure = l.sanitize(
RMWeatherDataType.PRESSURE,
maxpressure / 10.0) # converted to from hpa to kpa
if minpressure is not None:
minpressure = l.sanitize(RMWeatherDataType.PRESSURE,
minpressure / 10.0)
pressure = None
if maxpressure is not None and minpressure is not None:
pressure = (maxpressure + minpressure) / 2.0
rain = self.__toFloat(observation['metric']["precipTotal"])
self.addValue(RMParser.dataType.TEMPERATURE, tsYesterDay,
temperature, False)
self.addValue(RMParser.dataType.MINTEMP, tsYesterDay,
mintemp, False)
self.addValue(RMParser.dataType.MAXTEMP, tsYesterDay,
maxtemp, False)
self.addValue(RMParser.dataType.RH, tsYesterDay, rh, False)
self.addValue(RMParser.dataType.MINRH, tsYesterDay, minrh,
False)
self.addValue(RMParser.dataType.MAXRH, tsYesterDay, maxrh,
False)
self.addValue(RMParser.dataType.WIND, tsYesterDay, wind,
False)
self.addValue(RMParser.dataType.RAIN, tsYesterDay, rain,
False)
self.addValue(RMParser.dataType.DEWPOINT, tsYesterDay,
dewpoint, False)
self.addValue(RMParser.dataType.PRESSURE, tsYesterDay,
pressure, False)
return True
return False
except:
self.lastKnownError = "Warning: Failed to get yesterday data summary"
log.info(self.lastKnownError)
return False
def _reset_observations(self):
self.observations = dict.fromkeys(self.observations, None)
self.observation_counter = 0
self.startOfDayTimestamp = rmGetStartOfDay(self.currentTimestamp)
def getRecordsEx(self,
minTimestamp,
maxTimestamp,
withManualPrograms=False):
if (self.database.isOpen()):
#if withManualPrograms:
# sqlCondition = " AND usersch_id != 0 "
# sqlConditionForced = " WHERE usersch_id != 0"
#else:
# sqlCondition = ""
# sqlConditionForced = ""
if minTimestamp and maxTimestamp:
records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "\
"WHERE ?<=tokenTimestamp AND tokenTimestamp<? GROUP BY tokenTimestamp "\
"ORDER BY tokenTimestamp" % self._tableName, (minTimestamp, maxTimestamp)
)
elif minTimestamp:
records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "\
"WHERE ?<=tokenTimestamp GROUP BY tokenTimestamp "\
"ORDER BY tokenTimestamp" % self._tableName, (minTimestamp, )
)
elif maxTimestamp:
records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec), SUM(user_sec) userDuration duration, usersch_id FROM %s "\
"WHERE tokenTimestamp<? GROUP BY tokenTimestamp "\
"ORDER BY tokenTimestamp" % self._tableName, (maxTimestamp, )
)
else:
records = self.database.execute("SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "\
"GROUP BY tokenTimestamp ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName
)
tempResults = OrderedDict()
for row in records:
if not withManualPrograms and int(row[3]) == 0:
continue
dayTimestamp = rmGetStartOfDay(int(row[0]))
totalDurations = tempResults.get(
dayTimestamp, None) # List with real and user durations
if totalDurations is None:
totalDurations = [int(row[1]), int(row[2])]
else:
totalDurations[0] += int(row[1])
totalDurations[1] += int(row[2])
tempResults[dayTimestamp] = totalDurations
results = {"days": []}
for dayTimestamp in tempResults:
date = rmTimestampToDateAsString(dayTimestamp, "%Y-%m-%d")
day = {
"dayTimestamp": dayTimestamp,
"date": date,
"realDuration": tempResults.get(dayTimestamp)[0],
"userDuration": tempResults.get(dayTimestamp)[1]
}
results["days"].append(day)
return results
return None
def getRecords(self, minTimestamp, maxTimestamp):
if (self.database.isOpen()):
if minTimestamp and maxTimestamp:
records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\
"WHERE ?<=tokenTimestamp AND tokenTimestamp<? "\
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (minTimestamp, maxTimestamp)
)
elif minTimestamp:
records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\
"WHERE ?<=tokenTimestamp "\
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (minTimestamp, )
)
elif maxTimestamp:
records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\
"WHERE tokenTimestamp<? "\
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName, (maxTimestamp, )
)
else:
records = self.database.execute("SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "\
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName
)
tempResults = OrderedDict()
for row in records:
token = row[7]
dayTimestamp = rmGetStartOfDay(int(row[8]))
dayGroup = tempResults.get(dayTimestamp, None)
if dayGroup is None:
dayGroup = OrderedDict()
tempResults[dayTimestamp] = dayGroup
programGroup = dayGroup.get(token, None)
if programGroup is None:
programGroup = OrderedDict()
dayGroup[token] = programGroup
zones = programGroup.get(row[1], None)
if zones is None:
zones = OrderedDict()
programGroup[row[1]] = zones
zone = zones.get(row[2], None)
if zone is None:
zone = OrderedDict()
zone["uid"] = row[2]
zone["flag"] = row[6]
zone["cycles"] = []
zones[row[2]] = zone
cycles = zone["cycles"]
info = OrderedDict()
info["id"] = len(cycles) + 1
info["startTime"] = rmTimestampToDateAsString(row[0])
info["startTimestamp"] = row[0]
info["userDuration"] = row[3]
info["machineDuration"] = row[4]
info["realDuration"] = row[5]
cycles.append(info)
results = {"days": []}
for dayTimestamp in tempResults:
programs = []
day = {
"date": rmTimestampToDateAsString(dayTimestamp,
"%Y-%m-%d"),
"dateTimestamp": dayTimestamp,
"programs": programs
}
results["days"].append(day)
dayGroup = tempResults[dayTimestamp]
for token in dayGroup:
tempPrograms = dayGroup[token]
for programId in tempPrograms:
zones = []
program = OrderedDict()
program["id"] = programId
program["zones"] = zones
programs.append(program)
tempZones = tempPrograms[programId]
for zoneId in tempZones:
zones.append(tempZones[zoneId])
return results
return None
def getRecordsEx(self, minTimestamp, maxTimestamp, withManualPrograms=False):
if self.database.isOpen():
# if withManualPrograms:
# sqlCondition = " AND usersch_id != 0 "
# sqlConditionForced = " WHERE usersch_id != 0"
# else:
# sqlCondition = ""
# sqlConditionForced = ""
if minTimestamp and maxTimestamp:
records = self.database.execute(
"SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "
"WHERE ?<=tokenTimestamp AND tokenTimestamp<? GROUP BY tokenTimestamp "
"ORDER BY tokenTimestamp" % self._tableName,
(minTimestamp, maxTimestamp),
)
elif minTimestamp:
records = self.database.execute(
"SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "
"WHERE ?<=tokenTimestamp GROUP BY tokenTimestamp "
"ORDER BY tokenTimestamp" % self._tableName,
(minTimestamp,),
)
elif maxTimestamp:
records = self.database.execute(
"SELECT tokenTimestamp, SUM(real_sec), SUM(user_sec) userDuration duration, usersch_id FROM %s "
"WHERE tokenTimestamp<? GROUP BY tokenTimestamp "
"ORDER BY tokenTimestamp" % self._tableName,
(maxTimestamp,),
)
else:
records = self.database.execute(
"SELECT tokenTimestamp, SUM(real_sec) realDuration, SUM(user_sec) userDuration, usersch_id FROM %s "
"GROUP BY tokenTimestamp ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName
)
tempResults = OrderedDict()
for row in records:
if not withManualPrograms and int(row[3]) == 0:
continue
dayTimestamp = rmGetStartOfDay(int(row[0]))
totalDurations = tempResults.get(dayTimestamp, None) # List with real and user durations
if totalDurations is None:
totalDurations = [int(row[1]), int(row[2])]
else:
totalDurations[0] += int(row[1])
totalDurations[1] += int(row[2])
tempResults[dayTimestamp] = totalDurations
results = {"days": []}
for dayTimestamp in tempResults:
date = rmTimestampToDateAsString(dayTimestamp, "%Y-%m-%d")
day = {
"dayTimestamp": dayTimestamp,
"date": date,
"realDuration": tempResults.get(dayTimestamp)[0],
"userDuration": tempResults.get(dayTimestamp)[1],
}
results["days"].append(day)
return results
return None
def getRecords(self, minTimestamp, maxTimestamp):
if self.database.isOpen():
if minTimestamp and maxTimestamp:
records = self.database.execute(
"SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "
"WHERE ?<=tokenTimestamp AND tokenTimestamp<? "
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName,
(minTimestamp, maxTimestamp),
)
elif minTimestamp:
records = self.database.execute(
"SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "
"WHERE ?<=tokenTimestamp "
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName,
(minTimestamp,),
)
elif maxTimestamp:
records = self.database.execute(
"SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "
"WHERE tokenTimestamp<? "
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName,
(maxTimestamp,),
)
else:
records = self.database.execute(
"SELECT ts_started, usersch_id, zid, user_sec, machine_sec, real_sec, flag, token, tokenTimestamp FROM %s "
"ORDER BY tokenTimestamp, usersch_id, zid" % self._tableName
)
tempResults = OrderedDict()
for row in records:
token = row[7]
dayTimestamp = rmGetStartOfDay(int(row[8]))
dayGroup = tempResults.get(dayTimestamp, None)
if dayGroup is None:
dayGroup = OrderedDict()
tempResults[dayTimestamp] = dayGroup
programGroup = dayGroup.get(token, None)
if programGroup is None:
programGroup = OrderedDict()
dayGroup[token] = programGroup
zones = programGroup.get(row[1], None)
if zones is None:
zones = OrderedDict()
programGroup[row[1]] = zones
zone = zones.get(row[2], None)
if zone is None:
zone = OrderedDict()
zone["uid"] = row[2]
zone["flag"] = row[6]
zone["cycles"] = []
zones[row[2]] = zone
cycles = zone["cycles"]
info = OrderedDict()
info["id"] = len(cycles) + 1
info["startTime"] = rmTimestampToDateAsString(row[0])
info["startTimestamp"] = row[0]
info["userDuration"] = row[3]
info["machineDuration"] = row[4]
info["realDuration"] = row[5]
cycles.append(info)
results = {"days": []}
for dayTimestamp in tempResults:
programs = []
day = {
"date": rmTimestampToDateAsString(dayTimestamp, "%Y-%m-%d"),
"dateTimestamp": dayTimestamp,
"programs": programs,
}
results["days"].append(day)
dayGroup = tempResults[dayTimestamp]
for token in dayGroup:
tempPrograms = dayGroup[token]
for programId in tempPrograms:
zones = []
program = OrderedDict()
program["id"] = programId
program["zones"] = zones
programs.append(program)
tempZones = tempPrograms[programId]
for zoneId in tempZones:
zones.append(tempZones[zoneId])
return results
return None
def perform(self):
station = self.params.get("station", None)
if station is None or station == "":
station = "10637"
log.debug("No station set, using Frankfurt am Main (%s)" % station)
url = "http://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/single_stations/" + str(
station) + "/kml/MOSMIX_L_LATEST_" + str(station) + ".kmz"
try:
datafile = self.openURL(url)
if datafile is None:
self.lastKnownError = "Cannot read data from DWD Service."
return
else:
log.debug("Successfully loaded the KML file")
kmz = zipfile.ZipFile(BufferedRandomReader(datafile), 'r')
for name in kmz.namelist():
kml = kmz.read(name)
root = ElementTree.fromstring(kml)
ns = {
'xmlns':
"http://www.opengis.net/kml/2.2",
'dwd':
'https://opendata.dwd.de/weather/lib/pointforecast_dwd_extension_V1_0.xsd'
}
# temporary storage of forecast values
tmp = dict()
forecastDict = dict()
timestamps = []
forecasts = dict()
# Find all forecasts
for element in root.findall('.//dwd:Forecast', ns):
forecasts.update({
element.attrib['{https://opendata.dwd.de/weather/lib/pointforecast_dwd_extension_V1_0.xsd}elementName']:
element[0].text.split()
})
# Find all timestamps
for element in root.findall('.//dwd:TimeStep', namespaces=ns):
timestamps.append(element.text)
for timestep in timestamps:
for measure, values in forecasts.iteritems():
tmp.update({measure: values.pop(0)})
forecastDict.update({timestep: dict(tmp)})
# Add retreived data to DB
for time, forecast in forecastDict.iteritems():
timestamp = rmTimestampFromDateAsString(
time[:-5], "%Y-%m-%dT%H:%M:%S")
yesterdayTimestamp = rmGetStartOfDay(timestamp -
12 * 60 * 60)
if timestamp is None:
log.debug(
"Cannot convert timestamp: %s to unix timestamp" %
time)
continue
# Temperature
if forecast['TTT'] != '-':
TTT = float(forecast['TTT']) - 273.15
self.addValue(RMParser.dataType.TEMPERATURE, timestamp,
TTT)
# Minimum temperature last 24h
if forecast['TN'] != '-':
TN = float(forecast['TN']) - 273.15
self.addValue(RMParser.dataType.MINTEMP,
timestamp - 12 * 60 * 60, TN)
# Maximum temperature last 24h
if forecast['TX'] != '-':
TX = float(forecast['TX']) - 273.15
self.addValue(RMParser.dataType.MINTEMP,
timestamp - 12 * 60 * 60, TX)
# Windspeed
if forecast['FF'] != '-':
FF = float(forecast['FF'])
self.addValue(RMParser.dataType.WIND, timestamp, FF)
# Precipation last 24h
if forecast['RRdc'] != '-':
RRdc = float(forecast['RRdc'])
self.addValue(RMParser.dataType.QPF,
yesterdayTimestamp, RRdc)
# Atmospheric pressure
if forecast['PPPP'] != '-':
PPPP = float(forecast['PPPP']) / 1000
self.addValue(RMParser.dataType.PRESSURE, timestamp,
PPPP)
# Dewpoint
if forecast['Td'] != '-':
Td = float(forecast['Td']) - 273.15
self.addValue(RMParser.dataType.DEWPOINT, timestamp,
Td)
except Exception as e:
log.error("*** Error running DWD parser")
log.exception(e)
def wfUDPData(self):
air_count = 0
sky_count = 0
temp_total = 0
humd_total = 0
pres_total = 0
wind_total = 0
srad_total = 0
rain_total = 0
dewp_total = 0
bufferSize = 1024 # whatever you need
port = 50222
day_of_year = 0
log.debug("Start listening on port %d" % port)
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('0.0.0.0', port))
except:
log.Error("Socket failure")
log.debug("Start recieve loop")
while self.started:
hub = s.recvfrom(bufferSize)
data = json.loads(hub[0]) # hub is a truple (json, ip, port)
now = datetime.now()
# Check if this is a new day, if so
if day_of_year != now.timetuple().tm_yday:
# clear counters
air_count = 0
sky_count = 0
temp_total = 0
humd_total = 0
pres_total = 0
wind_total = 0
srad_total = 0
rain_total = 0
dewp_total = 0
self.report = {
'temperature': 0,
'humidity': 0,
'pressure': 0,
'dewpoint': 0,
'wind': 0,
'srad': 0,
'rain': 0,
'max_temp': -100,
'min_temp': 100,
'max_humid': 0,
'min_humid': 100
}
day_of_year = now.timetuple().tm_yday
self.parserData[1] = self.parserData[0]
self.newDay += 1 # signal that just rolled into a new day, need to send a yesterday summary onetime
# reset yesterday's timestamp to start of day
if 'ts' in self.parserData[1]:
self.parserData[1]['ts'] = rmGetStartOfDay(
self.parserData[1]['ts'])
#log.debug("type = %s broadcast s/n = %s TEMPEST target: %s" % (data["type"], data["serial_number"], self.params["TempestSerialNum"]))
debugMsg = "Observation: "
if (((data["type"] == "obs_air") and
(data["serial_number"] == self.params["AirSerialNumber"])) or
((data["type"] == "obs_st") and
(data["serial_number"] == self.params["TempestSerialNum"]))):
if data["type"] == "obs_air":
pres_idx = 1 # UDP packet data indexes for Air device
temp_idx = 2
humd_idx = 3
else:
pres_idx = 6 # UDP packet data indexes for TEMPEST device
temp_idx = 7
humd_idx = 8
air_count += 1
temp_total += data["obs"][0][temp_idx]
self.report["temperature"] = float(temp_total) / float(
air_count)
humd_total += data["obs"][0][humd_idx]
self.report["humidity"] = float(humd_total) / float(air_count)
# report pressure in hpa so convert from mb to hpa
pres_total += (data["obs"][0][pres_idx] / 10.0)
self.report["pressure"] = float(pres_total) / float(air_count)
# Calculate dewpoint
b = (17.625 * data["obs"][0][temp_idx]) / (
243.04 + data["obs"][0][temp_idx])
rh = float(data["obs"][0][humd_idx]) / 100.0
c = math.log(rh)
dewpoint = (243.04 * (c + b)) / (17.625 - c - b)
dewp_total += dewpoint
self.report["dewpoint"] = dewp_total / air_count
# Track Min/Max
if (data["obs"][0][temp_idx] > self.report["max_temp"]):
self.report["max_temp"] = data["obs"][0][temp_idx]
if (data["obs"][0][temp_idx] < self.report["min_temp"]):
self.report["min_temp"] = data["obs"][0][temp_idx]
if (data["obs"][0][humd_idx] > self.report["max_humid"]):
self.report["max_humid"] = data["obs"][0][humd_idx]
if (data["obs"][0][humd_idx] < self.report["min_humid"]):
self.report["min_humid"] = data["obs"][0][humd_idx]
debugMsg += "Temp (dF)= %.2f, " % (
(float(data["obs"][0][temp_idx]) * 9 / 5) + 32
) # convert degC to degF
debugMsg += "Humid = %.2f, " % data["obs"][0][humd_idx]
debugMsg += "Press (inHg) = %.2f, " % (
float(data["obs"][0][pres_idx]) / 33.8639)
if (((data["type"] == "obs_sky") and
(data["serial_number"] == self.params["SkySerialNumber"])) or
((data["type"] == "obs_st") and
(data["serial_number"] == self.params["TempestSerialNum"]))):
if data["type"] == "obs_sky":
wind_idx = 5 # UDP Packet data indexes for Sky device
srad_idx = 10
rain_idx = 11
else:
wind_idx = 2 # UDP packet data indexes for TEMPEST device
srad_idx = 11
rain_idx = 12
sky_count += 1
wind_total += data["obs"][0][wind_idx]
self.report["wind"] = float(wind_total) / float(sky_count)
srad_total += data["obs"][0][srad_idx]
self.report["srad"] = float(srad_total) / float(sky_count)
rain_total += data["obs"][0][rain_idx]
self.report["rain"] = rain_total
debugMsg += "Cumulative Rain (inch) = %.2f" % (rain_total /
25.4)
log.debug(debugMsg)
ts = mod_time.mktime(now.timetuple()) + now.microsecond / 1e6
self.parserData[0] = {'ts': ts, 'report': self.report}
log.debug("Receive thread exiting")
s.close()
self.started = False
def wfUDPData(self):
air_count = 0
sky_count = 0
temp_total = 0
humd_total = 0
pres_total = 0
wind_total = 0
srad_total = 0
rain_total = 0
dewp_total = 0
prev_air_hour = 0
prev_sky_hour = 0
prev_air_day = 0
prev_sky_day = 0
bufferSize = 1024 # whatever you need
port = 50222
day_of_year = 0
log.debug("Start listening on port %d" % port)
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('0.0.0.0', port))
except:
log.Error("Socket failure")
log.debug("Start recieve loop")
while self.started:
hub = s.recvfrom(bufferSize)
data = json.loads(hub[0]) # hub is a truple (json, ip, port)
now = datetime.now()
# Check if this is a new day, if so
if day_of_year != now.timetuple().tm_yday:
# clear counters
air_count = 0
sky_count = 0
temp_total = 0
humd_total = 0
pres_total = 0
dewp_total = 0
srad_total = 0
rain_total = 0
self.report = {
'temperature': 0,
'humidity': 0,
'pressure': 0,
'dewpoint': 0,
'wind': 0,
'srad': 0,
'rain': 0,
'max_temp': -100,
'min_temp': 100,
'max_humid': 0,
'min_humid': 100
}
day_of_year = now.timetuple().tm_yday
self.parserData[1] = self.parserData[0]
# reset yesterday's timestamp to start of day
if 'ts' in self.parserData[1]:
self.parserData[1]['ts'] = rmGetStartOfDay(self.parserData[1]['ts'])
#log.debug("type = %s s/n = %s" % (data["type"], data["serial_number"]))
if (data["type"] == "obs_air") and (data["serial_number"] == self.params["AirSerialNumber"]):
air_count += 1
temp_total += data["obs"][0][2]
self.report["temperature"] = float(temp_total) / float(air_count)
humd_total += data["obs"][0][3]
self.report["humidity"] = float(humd_total) / float(air_count)
# report pressure in hpa so convert from mb to hpa
pres_total += (data["obs"][0][1] / 10.0)
self.report["pressure"] = float(pres_total) / float(air_count)
# Calculate dewpoint
b = (17.625 * data["obs"][0][2]) / (243.04 + data["obs"][0][2])
rh = float(data["obs"][0][3]) / 100.0
c = math.log(rh)
dewpoint = (243.04 * (c + b)) / (17.625 - c - b)
dewp_total += dewpoint
self.report["dewpoint"] = dewp_total / air_count
# Track Min/Max
if (data["obs"][0][2] > self.report["max_temp"]):
self.report["max_temp"] = data["obs"][0][2]
if (data["obs"][0][2] < self.report["min_temp"]):
self.report["min_temp"] = data["obs"][0][2]
if (data["obs"][0][3] > self.report["max_humid"]):
self.report["max_humid"] = data["obs"][0][3]
if (data["obs"][0][3] < self.report["min_humid"]):
self.report["min_humid"] = data["obs"][0][3]
if (data["type"] == "obs_sky") and (data["serial_number"] == self.params["SkySerialNumber"]):
sky_count += 1
wind_total += data["obs"][0][5]
self.report["wind"] = float(wind_total) / float(sky_count)
srad_total += data["obs"][0][10]
self.report["srad"] = float(srad_total) / float(sky_count)
rain_total += data["obs"][0][3]
self.report["rain"] = rain_total
ts = mod_time.mktime(now.timetuple()) + now.microsecond / 1e6
self.parserData[0] = {
'ts': ts,
'report':self.report
}
log.debug("Receive thread exiting")
s.close()
self.started = False
