def getMeasure(self, moduleID, deviceID, measure):
postParams = {
"access_token": self.accessToken,
"module_id": moduleID,
"device_id": deviceID,
"scale": "1day",
"type": measure,
"date_begin": rmCurrentDayTimestamp() - 24 * 3600,
"date_end": rmCurrentDayTimestamp() - 1,
"real_time": True
}
jsonData = self.postRequest(self.getMeasureUrl, postParams)
return jsonData
def deleteRecordsByHistory(self, commit=True):
if (self.database.isOpen()):
threshold = rmCurrentDayTimestamp(
) - globalSettings.waterLogHistorySize * 86400
self.database.execute(
"DELETE FROM %s WHERE tokenTimestamp < ?" % self._tableName,
(threshold, ))
if commit:
self.database.commit()
def parseStationYesterdayDataNoKey(self, data):
#daily summary for yesterday
try:
l = RMWeatherDataLimits()
temperature = self.__toFloat(data["TemperatureAvgC"])
mintemp = self.__toFloat(data["TemperatureLowC"])
maxtemp = self.__toFloat(data["TemperatureHighC"])
rh = self.__toFloat(data["HumidityAvg"])
minrh = self.__toFloat(data["HumidityLow"])
maxrh = self.__toFloat(data["HumidityHigh"])
dewpoint = self.__toFloat(data["DewpointAvgC"])
wind = self.__toFloat(data["WindSpeedAvgKMH"])
maxpressure = self.__toFloat(data["PressureMaxhPa"])
minpressure = self.__toFloat(data["PressureMinhPa"])
rain = self.__toFloat(
data["PrecipitationSumCM"]) * 10.0 # from cm to mm
if wind is not None:
wind = wind / 3.6 # converted from kmetersph to mps
if maxpressure is not None:
maxpressure = l.sanitize(RMWeatherDataType.PRESSURE,
maxpressure /
10.0) # converted to from hpa to kpa
if minpressure is not None:
minpressure = l.sanitize(RMWeatherDataType.PRESSURE,
minpressure / 10.0)
pressure = None
if maxpressure is not None and minpressure is not None:
pressure = (maxpressure + minpressure) / 2.0
#log.info("rh:%s minrh: %s maxrh: %s pressure: %s temp: %s mintemp: %s maxtemp: %s" % (rh, minrh, maxrh, pressure, temperature, mintemp, maxtemp))
timestamp = rmCurrentDayTimestamp()
timestamp = rmGetStartOfDay(timestamp - 12 * 3600)
self.addValue(RMParser.dataType.TEMPERATURE, timestamp,
temperature, False)
self.addValue(RMParser.dataType.MINTEMP, timestamp, mintemp, False)
self.addValue(RMParser.dataType.MAXTEMP, timestamp, maxtemp, False)
self.addValue(RMParser.dataType.RH, timestamp, rh, False)
self.addValue(RMParser.dataType.MINRH, timestamp, minrh, False)
self.addValue(RMParser.dataType.MAXRH, timestamp, maxrh, False)
self.addValue(RMParser.dataType.WIND, timestamp, wind, False)
self.addValue(RMParser.dataType.RAIN, timestamp, rain, False)
self.addValue(RMParser.dataType.DEWPOINT, timestamp, dewpoint,
False)
self.addValue(RMParser.dataType.PRESSURE, timestamp, pressure,
False)
except Exception, e:
self.lastKnownError = "ERROR: Failed to get historical data"
log.error("%s: %s" % (self.lastKnownError, e))
def clearHistory(self, parserID, commit):
if self.database.isOpen():
if globalSettings.parserHistorySize > 0:
maxDayTimestamp = rmCurrentDayTimestamp()
minDayTimestamp = maxDayTimestamp - globalSettings.parserHistorySize * 86400
self.deleteRecordsHistoryByDayThreshold(parserID, minDayTimestamp, maxDayTimestamp, False)
else:
self.database.execute("DELETE FROM parserData WHERE parserID=?", (parserID, ))
if commit:
self.database.commit()
def perform(self): # The function that will be executed must have this name
# Accessing system location settings
#lat = self.settings.location.latitude
log.info("Hello History")
# Other location settings
#self.zip
#self.name
#self.state
#self.latitude
#self.longitude
#self.address
#self.elevation
#self.gmtOffset
#self.dstOffset
#self.stationID
#self.stationName
#self.et0Average
station = self.params.get("StationID", None)
if station is None or station == "":
station = "02860"
log.debug("No station set, using (%s)" % station)
#url = "https://opendata.dwd.de/climate/observations_germany/climate/hourly/precipitation/recent/stundenwerte_RR_" + str(station) + "_akt.zip"
url = "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/now/1minutenwerte_nieder_" + str(station) + "_now.zip"
URLParams = [
("User-Agent", "RainMachine v2")
]
try:
req = urllib2.Request(url)
response = urllib2.urlopen(req)
raw = response.read()
zipFile = ZipFile(StringIO.StringIO(raw))
dataFile = None
for fileInfo in zipFile.filelist:
if fileInfo.filename.startswith("produkt_ein_now_"):
dataFile = fileInfo
if dataFile == None:
log.error("Unable to find data file.")
return
content = zipFile.read(dataFile)
reader = csv.reader(StringIO.StringIO(content), delimiter=';')
next(reader)
#minDate = datetime.datetime.today() - datetime.timedelta(days=30)
minDate = rmDeltaDayFromTimestamp(rmCurrentDayTimestamp(), -30)
epochDate = datetime.datetime(1970,1,1)
log.debug("minDate: %s" % minDate)
lastHour = None
currentHour = None
totalHour = 0
for row in reader:
timeStamp = row[1]
#log.debug("Timestamp: %s" % timeStamp)
myDate = datetime.datetime.strptime(row[1], "%Y%m%d%H%M")
myEpoch = (int)((myDate - epochDate).total_seconds())
#date = rmTimestampFromDateAsString(timeStamp, "%Y%m%d%H")
if myEpoch is None:
log.debug("Cannot convert timestamp: %s to unix timestamp" % timeStamp)
continue
if myEpoch < minDate:
continue
value = parseString(row[3])
if value == None:
continue
currentHour = myEpoch - (myEpoch % 3600)
if currentHour != lastHour:
if lastHour != None:
# log.debug("Adding value %s" % value)
self.addValue(RMParser.dataType.RAIN, lastHour, totalHour)
totalHour = value
lastHour = currentHour
else:
totalHour += value
log.info("Done")
except Exception, e:
log.error("*** Error running DWD parser")
log.exception(e)
def parseStationDataWithKey(self, jsonData):
# daily summary for yesterday
tsToday = rmCurrentDayTimestamp()
tsYesterDay = rmDeltaDayFromTimestamp(tsToday, -1)
l = RMWeatherDataLimits()
try:
dailysummary = jsonData['summaries']
for observation in dailysummary:
tsDay = observation.get('epoch', None)
tsDay = rmGetStartOfDay(tsDay)
if tsDay == tsYesterDay:
temperature = self.__toFloat(
observation['metric']['tempAvg'])
mintemp = self.__toFloat(observation['metric']['tempLow'])
maxtemp = self.__toFloat(observation['metric']['tempHigh'])
rh = self.__toFloat(observation["humidityAvg"])
minrh = self.__toFloat(observation["humidityLow"])
maxrh = self.__toFloat(observation["humidityHigh"])
dewpoint = self.__toFloat(
observation['metric']["dewptAvg"])
wind = self.__toFloat(
observation['metric']["windspeedAvg"])
if wind is not None:
wind = wind / 3.6 # converted from kmetersph to mps
maxpressure = self.__toFloat(
observation['metric']["pressureMax"])
minpressure = self.__toFloat(
observation['metric']["pressureMin"])
if maxpressure is not None:
maxpressure = l.sanitize(
RMWeatherDataType.PRESSURE,
maxpressure / 10.0) # converted to from hpa to kpa
if minpressure is not None:
minpressure = l.sanitize(RMWeatherDataType.PRESSURE,
minpressure / 10.0)
pressure = None
if maxpressure is not None and minpressure is not None:
pressure = (maxpressure + minpressure) / 2.0
rain = self.__toFloat(observation['metric']["precipTotal"])
self.addValue(RMParser.dataType.TEMPERATURE, tsYesterDay,
temperature, False)
self.addValue(RMParser.dataType.MINTEMP, tsYesterDay,
mintemp, False)
self.addValue(RMParser.dataType.MAXTEMP, tsYesterDay,
maxtemp, False)
self.addValue(RMParser.dataType.RH, tsYesterDay, rh, False)
self.addValue(RMParser.dataType.MINRH, tsYesterDay, minrh,
False)
self.addValue(RMParser.dataType.MAXRH, tsYesterDay, maxrh,
False)
self.addValue(RMParser.dataType.WIND, tsYesterDay, wind,
False)
self.addValue(RMParser.dataType.RAIN, tsYesterDay, rain,
False)
self.addValue(RMParser.dataType.DEWPOINT, tsYesterDay,
dewpoint, False)
self.addValue(RMParser.dataType.PRESSURE, tsYesterDay,
pressure, False)
return True
return False
except:
self.lastKnownError = "Warning: Failed to get yesterday data summary"
log.info(self.lastKnownError)
return False
def perform(self):
if self.username is None:
self.username = self.params["username"]
self.password = self.params["password"]
if self.password is None or self.username is None:
log.info("Cannot login: no username or password provided")
self.lastKnownError = "Error: Invalid username or password"
return
if self.username is not self.params["username"]:
self.username = self.params["username"]
self.password = self.params["password"]
self.clientOauth()
if self.accessToken is None:
self.clientOauth()
else:
self.renewAccesTokenIfNeeded()
if self.accessToken is None:
log.info("Cannot login: invalid oauth")
self.lastKnownError = "Error: Invalid username or password"
return
self.getData()
tsStartOfDayUTC = rmCurrentDayTimestamp()
specifiedModules = []
if self.params["useSpecifiedModules"]:
modulesString = self.params["specificModules"]
specifiedModules = modulesString.split(',')
specifiedModules = [item.strip() for item in specifiedModules]
for device in self.jsonData["body"]["devices"][0:1]:
name = device["station_name"] #put as output parameter?
[llat, llon] = device["place"]["location"] #use for max distance
modules = device["modules"]
rh = 0
temp = 0
maxTemp = 0
minTemp = 0
rain = 0
wind = 0
tsTemp = None
tsWind = None
tsRain = None
idxTemp = 0
idxWind = 0
idxRain = 0
self.params["_availableModules"] = []
for module in modules:
moduleName = 'unnamed'
try:
moduleName = module["module_name"]
except:
pass
moduleID = module["_id"]
self.params["_availableModules"].append([moduleName , moduleID] )
moduleDataType = module["data_type"]
if self.params["useSpecifiedModules"]:
if moduleID not in specifiedModules:
continue
elif "outdoor" not in moduleName.lower() and ("Rain" not in moduleDataType) and ("Wind" not in moduleDataType):
continue
try:
recordedRain = self.__toFloat(module["dashboard_data"]["Rain"]) #measured in C
tsRecordedRain = self.__toInt(module["dashboard_data"]["time_utc"])
if tsRecordedRain < tsStartOfDayUTC:
continue
tsRain = max(tsRecordedRain, tsRain)
rain += recordedRain
idxRain+=1
except:
pass
try:
recordedWind = self.__toFloat(module["dashboard_data"]["WindStrength"])
tsRecordedWind = self.__toInt(module["dashboard_data"]["time_utc"])
if tsRecordedWind < tsStartOfDayUTC:
continue
tsWind = max(recordedWind, tsWind)
wind += recordedWind
idxWind+=1
except:
pass
try:
recordedTemp = self.__toFloat(module["dashboard_data"]["Temperature"])
tsRecordedTemp = self.__toInt(module["dashboard_data"]["time_utc"])
if tsRecordedTemp < tsStartOfDayUTC :
continue
tsTemp = max(tsRecordedTemp, tsTemp)
maxTemp += self.__toFloat(module["dashboard_data"]["max_temp"])
minTemp += self.__toFloat(module["dashboard_data"]["min_temp"])
rh += self.__toFloat(module["dashboard_data"]["Humidity"]) #measured in %
temp += recordedTemp
idxTemp+=1
except:
pass
if idxTemp > 0 and tsTemp > tsStartOfDayUTC:
self.addValue(RMParser.dataType.TEMPERATURE, tsStartOfDayUTC, temp/idxTemp)
self.addValue(RMParser.dataType.MINTEMP, tsStartOfDayUTC, minTemp/idxTemp)
self.addValue(RMParser.dataType.MAXTEMP, tsStartOfDayUTC, maxTemp/idxTemp)
self.addValue(RMParser.dataType.RH, tsStartOfDayUTC, rh/idxTemp)
if idxWind > 0 and tsWind > tsStartOfDayUTC:
self.addValue(RMParser.dataType.WIND, tsStartOfDayUTC, wind/idxWind)
if idxRain > 0 and tsRain > tsStartOfDayUTC:
self.addValue(RMParser.dataType.RAIN, tsStartOfDayUTC, rain/idxRain)
for parserCfg in RMParserManager.instance.parsers:
if self.parserName is parserCfg.name:
RMParserManager.instance.setParserParams(parserCfg.dbID, self.params)
break
def perform(self):
self.clientSecret = ""
self.clientID = ""
if self.username is None:
self.username = self.params["username"]
self.password = self.params["password"]
if self.password is None or self.username is None:
log.info("Cannot login: no username or password provided")
self.lastKnownError = "Error: Invalid username or password"
return
if self.username is not self.params["username"]:
self.username = self.params["username"]
self.password = self.params["password"]
self.clientOauth()
if self.accessToken is None:
self.clientOauth()
else:
self.renewAccesTokenIfNeeded()
if self.accessToken is None:
log.info("Cannot login: invalid oauth")
self.lastKnownError = "Error: Invalid username or password"
return
self.getData()
tsStartOfDayUTC = rmCurrentDayTimestamp()
specifiedModules = []
if self.params["useSpecifiedModules"]:
modulesString = self.params["specificModules"]
specifiedModules = modulesString.split(',')
specifiedModules = [item.strip() for item in specifiedModules]
for device in self.jsonData["body"]["devices"][0:1]:
name = device["station_name"] #put as output parameter?
[llat, llon] = device["place"]["location"] #use for max distance
modules = device["modules"]
rh = 0
temp = 0
maxTemp = 0
minTemp = 0
rain = 0
wind = 0
tsTemp = None
tsWind = None
tsRain = None
idxTemp = 0
idxWind = 0
idxRain = 0
self.params["_availableModules"] = []
for module in modules:
moduleName = 'unnamed'
try:
moduleName = module["module_name"]
except:
pass
moduleID = module["_id"]
self.params["_availableModules"].append([moduleName, moduleID])
moduleDataType = module["data_type"]
if self.params["useSpecifiedModules"]:
if moduleID not in specifiedModules:
continue
elif "outdoor" not in moduleName.lower() and (
"Rain"
not in moduleDataType) and ("Wind"
not in moduleDataType):
continue
try:
recordedRain = self.__toFloat(
module["dashboard_data"]["Rain"]) #measured in C
tsRecordedRain = self.__toInt(
module["dashboard_data"]["time_utc"])
if tsRecordedRain < tsStartOfDayUTC:
continue
tsRain = max(tsRecordedRain, tsRain)
rain += recordedRain
idxRain += 1
except:
pass
try:
recordedWind = self.__toFloat(
module["dashboard_data"]["WindStrength"])
tsRecordedWind = self.__toInt(
module["dashboard_data"]["time_utc"])
if tsRecordedWind < tsStartOfDayUTC:
continue
tsWind = max(recordedWind, tsWind)
wind += recordedWind
idxWind += 1
except:
pass
try:
recordedTemp = self.__toFloat(
module["dashboard_data"]["Temperature"])
tsRecordedTemp = self.__toInt(
module["dashboard_data"]["time_utc"])
if tsRecordedTemp < tsStartOfDayUTC:
continue
tsTemp = max(tsRecordedTemp, tsTemp)
maxTemp += self.__toFloat(
module["dashboard_data"]["max_temp"])
minTemp += self.__toFloat(
module["dashboard_data"]["min_temp"])
rh += self.__toFloat(
module["dashboard_data"]["Humidity"]) #measured in %
temp += recordedTemp
idxTemp += 1
except:
pass
if idxTemp > 0 and tsTemp > tsStartOfDayUTC:
self.addValue(RMParser.dataType.TEMPERATURE, tsStartOfDayUTC,
temp / idxTemp)
self.addValue(RMParser.dataType.MINTEMP, tsStartOfDayUTC,
minTemp / idxTemp)
self.addValue(RMParser.dataType.MAXTEMP, tsStartOfDayUTC,
maxTemp / idxTemp)
self.addValue(RMParser.dataType.RH, tsStartOfDayUTC,
rh / idxTemp)
if idxWind > 0 and tsWind > tsStartOfDayUTC:
self.addValue(RMParser.dataType.WIND, tsStartOfDayUTC,
wind / idxWind)
if idxRain > 0 and tsRain > tsStartOfDayUTC:
self.addValue(RMParser.dataType.RAIN, tsStartOfDayUTC,
rain / idxRain)
for parserCfg in RMParserManager.instance.parsers:
if self.parserName is parserCfg.name:
RMParserManager.instance.setParserParams(
parserCfg.dbID, self.params)
break
def deleteRecordsByHistory(self, commit=True):
if self.database.isOpen():
threshold = rmCurrentDayTimestamp() - globalSettings.waterLogHistorySize * 86400
self.database.execute("DELETE FROM %s WHERE tokenTimestamp < ?" % self._tableName, (threshold,))
if commit:
self.database.commit()
def perform(self):
self.clientSecret = ""
self.clientID = ""
if self.username is None:
self.username = self.params["username"]
self.password = self.params["password"]
if self.password is None or self.username is None:
self.lastKnownError = "Error: Invalid username or password."
log.error(self.lastKnownError)
return
if self.username != self.params["username"]:
self.username = self.params["username"]
self.password = self.params["password"]
self.clientOauth()
if self.accessToken is None:
self.clientOauth()
else:
self.renewAccesTokenIfNeeded()
if self.accessToken is None:
self.lastKnownError = "Error: Cannot retrieve OAuth token."
log.error(self.lastKnownError)
return
self.getData()
tsStartOfDayUTC = rmCurrentDayTimestamp()
specifiedModules = []
if self.params["useSpecifiedModules"]:
modulesString = self.params["specificModules"]
specifiedModules = modulesString.split(',')
specifiedModules = [item.strip() for item in specifiedModules]
# Only use the first reported device which should be user device, others are devices owned by other users
# and shared with current user
if len(self.jsonData["body"]["devices"]) == 0:
self.lastKnownError = "No NetAtmo devices found"
log.error(self.lastKnownError)
return
device = self.jsonData["body"]["devices"][0]
name = device["station_name"] #put as output parameter?
[llat, llon] = device["place"]["location"] #use for max distance
deviceID = device["_id"]
modules = device["modules"]
minRH = 0
maxRH = 0
# minPress = 0
# maxPress = 0
maxTemp = 0
minTemp = 0
rain = 0
wind = 0
tsTemp = None
tsWind = None
tsRain = None
idxTemp = 0
idxRH = 0
# idxPress = 0
idxWind = 0
idxRain = 0
self.params["_availableModules"] = []
for module in modules:
moduleName = 'unnamed'
try:
moduleName = module["module_name"]
except:
pass
moduleID = module["_id"]
self.params["_availableModules"].append([moduleName, moduleID])
moduleDataType = module["data_type"]
if self.params["useSpecifiedModules"]:
if moduleID not in specifiedModules:
continue
elif "outdoor" not in moduleName.lower() and (
"Rain" not in moduleDataType) and ("Wind"
not in moduleDataType):
continue
if "Rain" in moduleDataType:
try:
rainJson = self.getMeasure(moduleID, deviceID, "sum_rain")
recordedRain = self.__toFloat(
rainJson["body"][0]["value"][0][0])
tsRain = self.__toInt(rainJson["body"][0]["beg_time"])
rain += recordedRain
idxRain += 1
except Exception, e:
log.error("Error reading rain gauge module: %s" % e)
if "Wind" in moduleDataType:
try:
windJson = self.getMeasure(moduleID, deviceID,
"WindStrength")
recorderWind = self.__toFloat(
windJson["body"][0]["value"][0][0])
tsWind = self.__toInt(windJson["body"][0]["beg_time"])
wind += recorderWind
idxWind += 1
except Exception, e:
log.error("Error reading wind module: %s" % e)
def perform(self):
self.clientSecret = ""
self.clientID = ""
if self.username is None:
self.username = self.params["username"]
self.password = self.params["password"]
if self.password is None or self.username is None:
self.lastKnownError = "Error: Invalid username or password."
log.error(self.lastKnownError)
return
if self.username != self.params["username"]:
self.username = self.params["username"]
self.password = self.params["password"]
self.clientOauth()
# Try to refresh the access token first if a previous one exists
if self.accessToken is not None:
self.renewAccesTokenIfNeeded()
if self.accessToken is None:
log.info("Doing full auth")
self.clientOauth()
if self.accessToken is None:
self.lastKnownError = "Error: Authentication failure"
log.error(self.lastKnownError)
return
self.getData()
tsStartOfDayUTC = rmCurrentDayTimestamp()
if len(self.jsonData["body"]["devices"]) == 0:
self.lastKnownError = "No NetAtmo devices found"
log.error(self.lastKnownError)
return
# Get available modules from returned json
self.buildAvailableModules()
# Build the list of user specified modules
specifiedModules = []
if self.params["useSpecifiedModules"]:
modulesString = self.params["specificModules"]
specifiedModules = modulesString.split(',')
specifiedModules = [item.strip() for item in specifiedModules]
# Only use the first reported device which should be user device if user didn't specify modules
# Otherwise we will have multiple reports that we need to collate data from multiple devices
# and multiple modules
if self.params["useSpecifiedModules"]:
for device in self.jsonData["body"]["devices"]:
self.getDeviceData(device, specifiedModules)
else:
self.getDeviceData(self.jsonData["body"]["devices"][0],
specifiedModules)
