def render(self, data={}, context={}): try: assert self.id is not None, "'wunderground.id' must be set" assert self.password is not None, "'wunderground.password' must be set" assert self.period is not None, "'wunderground.period' must be set" self.real_time = self.real_time and self.period < 30 rtfreq = None if self.real_time: rtfreq = self.period self.logger.info( "Initializing Wunderground publisher (station %s)" % self.id) import weather.services self.publisher = weather.services.Wunderground( self.id, self.password, rtfreq) self.alive = True if not self.real_time: accu = AccumulatorDatasource() accu.slice = 'day' accu.span = 1 accu.storage = self.storage accu.formulas = { 'current': { 'temp': LastFormula('temp'), 'dew_point': LastFormula('dew_point'), 'hum': LastFormula('hum'), 'pressure': LastFormula('pressure'), 'wind': LastFormula('wind'), 'wind_deg': LastFormula('wind_dir'), 'gust': LastFormula('wind_gust'), 'gust_deg': LastFormula('wind_gust_dir'), 'rain_rate': LastFormula('rain_rate'), 'rain_fall': SumFormula('rain'), 'utctime': LastFormula('utctime') } } while self.alive: try: data = accu.execute()['current']['series'] index = len(data['lbl']) - 1 params = { # <float> pressure: in inches of Hg 'pressure': HPaToInHg(data['pressure'][index]), # <float> dewpoint: in Fahrenheit 'dewpoint': CToF(data['dew_point'][index]), # <float> humidity: between 0.0 and 100.0 inclusive 'humidity': data['hum'][index], # <float> tempf: in Fahrenheit 'tempf': CToF(data['temp'][index]), # <float> rainin: inches/hour of rain 'rainin': MmToIn(data['rain_rate'][index]), # <float> rainday: total rainfall in day (localtime) 'rainday': MmToIn(data['rain_fall'][index]), # <string> dateutc: date "YYYY-MM-DD HH:MM:SS" in GMT timezone 'dateutc': data['utctime'][index].strftime( '%Y-%m-%d %H:%M:%S'), # <float> windspeed: in mph 'windspeed': MpsToMph(data['wind'][index]), # <float> winddir: in degrees, between 0.0 and 360.0 'winddir': data['wind_deg'][index], # <float> windgust: in mph 'windgust': MpsToMph(data['gust'][index]), # <float> windgustdir: in degrees, between 0.0 and 360.0 'windgustdir': data['gust_deg'][index] } # Do not send parameters that are null (None). # from above only dateutc is a mandatory parameter. params = dict( filter(lambda (p, v): v, [(p, v) for p, v in params.iteritems()])) self.logger.info( "Publishing Wunderground data (normal server): %s " % str(params)) self.publisher.set(**params) response = self.publisher.publish() self.logger.info('Result Wunderground publisher: %s' % str(response)) except Exception, e: self.logger.exception(e) time.sleep(self.period) else:
def render(self, data={}, context={}): try: assert self.username is not None, "'openweathermap.id' must be set" assert self.password is not None, "'openweathermap.password' must be set" assert self.name is not None, "'openweathermap.name' must be set" assert self.latitude is not None, "'openweathermap.latitude' must be set" assert self.longitude is not None, "'openweathermap.longitude' must be set" assert self.altitude is not None, "'openweathermap.altitude' must be set" self.logger.info("Initializing openweathermap.com (user %s)" % self.username) self.alive = True accu = AccumulatorDatasource() accu.slice = 'day' accu.span = 1 accu.storage = self.storage accu.formulas = {'current': { 'temp' : LastFormula('temp'), 'hum' : LastFormula('hum'), 'pressure' : LastFormula('pressure'), 'dew_point' : LastFormula('dew_point'), 'wind' : LastFormula('wind'), 'wind_gust' : LastFormula('wind_gust'), 'wind_deg' : LastFormula('wind_dir'), 'rain' : SumFormula('rain'), 'utctime' : LastFormula('utctime') } } if self.send_uv: accu.formulas['current']['uv'] = LastFormula('uv') if self.send_radiation: accu.formulas['current']['solar_rad'] = LastFormula('solar_rad') accu24h = AccumulatorDatasource() accu24h.slice = 'hour' accu24h.span = 24 accu24h.storage = self.storage accu24h.formulas = {'current': {'rain': SumFormula('rain')} } accu60min = AccumulatorDatasource() accu60min.slice = 'minute' accu60min.span = 60 accu60min.storage = self.storage accu60min.formulas = {'current': {'rain': SumFormula('rain')} } last_timestamp = None while self.alive: try: data = accu.execute()['current']['series'] index = len(data['lbl'])-1 rain_1h = sum(map(lambda x: x if x is not None else 0, accu60min.execute()['current']['series']['rain'][:60])) rain_24h = sum(map(lambda x: x if x is not None else 0, accu24h.execute()['current']['series']['rain'][:24])) if last_timestamp == None or last_timestamp < data['utctime'][index]: last_timestamp = data['utctime'][index] args = { 'wind_dir': int(round(data['wind_deg'][index])), # grad 'wind_speed': str(data['wind'][index]), # mps 'wind_gust': str(data['wind_gust'][index]), # mps 'temp': str(data['temp'][index]), # grad C #'dewpoint': str(data['dew_point'][index]), # NOT WORKING PROPERLY 'humidity': int(round(data['hum'][index])), # relative humidity % 'pressure': str(data['pressure'][index]), # mb 'rain_1h': rain_1h, # mm 'rain_24h': rain_24h, # mm 'rain_today': str(data['rain'][index]), # mm 'lat': self.latitude, 'long': self.longitude, 'alt': self.altitude, 'name': self.name } if self.send_uv: args['uv'] = str(data['uv'][index]) if self.send_radiation: args['lum'] = str(data['solar_rad'][index]) self.logger.debug("Publishing openweathermap data: %s " % urllib.urlencode(args)) response = self._publish(args, 'openweathermap.org', '/data/post') if response[0] == 200: self.logger.info('Data published successfully') self.logger.debug('Code: %s Status: %s Answer: %s' % response) else: self.logger.error('Error publishing data. Code: %s Status: %s Answer: %s' % response) except Exception, e: if (str(e) == "'NoneType' object has no attribute 'strftime'") or (str(e) == "a float is required"): self.logger.error('Could not publish: no valid values at this time. Retry next run...') else: self.logger.exception(e) time.sleep(60) # each minute we check for new records to send to openweathermap except Exception, e: self.logger.exception(e) raise
class AccumulatorDatasource(object): ''' Calculates data from a storage in an iterative way by traversing only recently added data. [ Properties ] storage [storage]: The underlying storage to get samples. slice [year|month|week|day|hour|minute] (optional): The unit of grouping for the calculated series. Defaults to 'hour'' span [numeric] (optional): Number of slices in the resulting series. Defaults to 24. period [numeric] (optional): Number of seconds between two refreshes of the calculated data. DEfaults to 120. format [string or list of strings] (optional): Date/time format string for labels. See Python strftime function. It can be a single string (only 1 label) or a list of formats. First label is 'lbl' and the rest are 'lbl2', lbl3', etc. formulas [dict] (optional): Specify what and how to calculate. Defines the structure of the resulting data. Dictionary keyed by the measure names ('temp', 'hum', ...). Values are dictionaries keyed by the serie names ('avg', 'min', ...) and containing 'formula' objects. caching [true|false] (optional): Enable/disable caching for normal requests. Defaults to true. ''' storage = None slice = 'hour' span = 23 format = None formats = { 'year': ['%y', '%Y'], 'month': ['%m', '%Y/%m'], 'week': ['%d/%m', '%Y/%m/%d'], 'day': ['%d/%m', '%Y/%m/%d'], 'hour': ['%H', '%Y/%m/%d %H'], 'minute': ['%H:%M', '%Y/%m/%d %H:%M'] } period = 120 default_formulas = { 'temp': { 'avg' : AverageFormula('temp'), 'min' : MinFormula('temp'), 'max' : MaxFormula('temp') }, 'dew' : { 'avg': AverageFormula('dew_point') }, 'hum' : { 'avg' : AverageFormula('hum'), 'min' : MinFormula('hum'), 'max' : MaxFormula('hum') }, 'press' : { 'avg' : AverageFormula('pressure'), 'min' : MinFormula('pressure'), 'max' : MaxFormula('pressure') }, 'wind' : { 'avg' : AverageFormula('wind'), 'max' : MaxFormula('wind_gust'), 'deg,dir' : PredominantWindFormula('wind') }, 'sectors' : { 'avg' : WindSectorAverageFormula('wind'), 'max' : WindSectorMaxFormula('wind_gust'), 'freq' : WindSectorFrequencyFormula('wind') }, 'rain' : { 'rate' : MaxFormula('rain_rate'), 'fall' : SumFormula('rain') }, 'uv' : { 'index' : MaxFormula('uv_index') } } formulas = default_formulas caching = True logger = logging.getLogger("datasource.accumulator") last_timestamp = datetime.datetime.fromtimestamp(0) cached_slices = None cached_series = None lock = threading.Lock() class Slice(object): def __init__(self, formulas, from_time, to_time, keys): self.formulas = copy.deepcopy(formulas) self.from_time = from_time self.to_time = to_time # replace string keys with index for performance for serie in self.formulas.values(): for formula in serie.values(): if type(formula.index)==str: formula.index = keys.index(formula.index) # Index can be a list of indexes (e.g. heatIndex or WindChill) elif type(formula.index)==list: formula.index = map(lambda x: keys.index(x), formula.index) def add_sample(self, sample): for serie in self.formulas.values(): for formula in serie.values(): formula.append(sample) def get_slice_duration(self): if self.slice == 'minute': return datetime.timedelta(0, 60) elif self.slice == 'hour': return datetime.timedelta(0, 3600) elif self.slice == 'day': return datetime.timedelta(1) elif self.slice == 'week': return datetime.timedelta(7) elif self.slice == 'month': return datetime.timedelta(30) elif self.slice == 'year': return datetime.timedelta(365) def get_slice_start(self, time): if self.slice == 'minute': return datetime.datetime(time.year, time.month, time.day, time.hour, time.minute) elif self.slice == 'hour': return datetime.datetime(time.year, time.month, time.day, time.hour) elif self.slice == 'day': return datetime.datetime(time.year, time.month, time.day) elif self.slice == 'week': (year, week, dayweek) = time.isocalendar() return iso_to_gregorian(year, week, 1) elif self.slice == 'month': return datetime.datetime(time.year, time.month, 1) elif self.slice == 'year': return datetime.datetime(time.year, 1, 1) def get_next_slice_start(self, time): if self.slice == 'minute': return time+datetime.timedelta(0,60) elif self.slice == 'hour': return time+datetime.timedelta(0,3600) elif self.slice == 'day': return time+datetime.timedelta(1,0) elif self.slice == 'week': (year, week, dayweek) = time.isocalendar() return iso_to_gregorian(year, week, 1)+datetime.timedelta(7) elif self.slice == 'month': if time.month == 12: return datetime.datetime(time.year + 1, 1, 1) else: return datetime.datetime(time.year, time.month + 1, 1) elif self.slice == 'year': return datetime.datetime(time.year + 1, 1, 1) def get_labels(self, slices): if self.format is not None: if type(self.format) == str: format_list = [self.format] else: format_list = self.format else: format_list = self.formats[self.slice] return [[slice.from_time.strftime(format) for slice in slices] for format in format_list] def update_slices(self, slices, from_time, to_time, context, last_timestamp=None): if len(slices) > 0: slice_from_time = slices[-1].to_time else: slice_from_time = from_time # Create the necessary slices t = self.get_slice_start(slice_from_time) keys = self.storage.keys(context=context) while t < to_time: end = self.get_next_slice_start(t) self.logger.debug("Creating slice %s - %s", t, end) slice = self.Slice(self.formulas, t, end, keys) slices.append(slice) t = end # Fill them with samples if last_timestamp: update_from_time = max(last_timestamp + datetime.timedelta(seconds=1), from_time) # Add 1 sec to last_timestamp so that the same sample is not retrieved twice else: update_from_time = from_time self.logger.debug("Update from %s ", update_from_time) s = 0 to_delete = 0 localtime_index = keys.index('localtime') for sample in self.storage.samples(update_from_time, to_time, context=context): # find the first slice receiving the samples sample_localtime = sample[localtime_index] while slices[s].to_time < sample_localtime: if slices[s].to_time < from_time: # count of obsolete slices to delete to_delete=s s = s + 1 slices[s].add_sample(sample) last_timestamp = sample_localtime return last_timestamp, to_delete def get_series(self, slices): result = {} for k,v in self.formulas.iteritems(): result[k]={} result[k]['series']={} for key in v.keys(): subkeys = key.split(',') for subkey in subkeys: result[k]['series'][subkey]=[] i = 1 for labels in self.get_labels(slices): literal = 'lbl%d' % i if i > 1 else 'lbl' i += 1 result[k]['series'][literal]=labels for slice in slices: for k,v in slice.formulas.iteritems(): for key,formula in v.iteritems(): value = formula.value() subkeys = key.split(',') if len(subkeys) == 1: value = [ value ] for i in range(len(subkeys)): result[k]['series'][subkeys[i]].append(value[i]) return result def execute(self,data={}, context={}): if data.has_key('time_end'): to_time = parse(data['time_end']) use_cache = False else: to_time = datetime.datetime.now() use_cache = self.caching duration = self.get_slice_duration() times = (self.span - 1) delta= duration * times from_time = self.get_slice_start(to_time - delta) if use_cache: self.logger.debug("Last timestamp: %s", self.last_timestamp) self.lock.acquire() try: if self.last_timestamp < to_time - datetime.timedelta(0,self.period) or self.cached_series is None: if self.cached_slices is None: self.cached_slices = [] last_timestamp, to_delete = self.update_slices(self.cached_slices, from_time, to_time, context, self.last_timestamp) self.cached_slices = self.cached_slices[to_delete:] self.logger.debug('Deleted %s slices', to_delete) self.logger.debug("Last timestamp: %s", self.last_timestamp) self.last_timestamp = last_timestamp self.cached_series = self.get_series(self.cached_slices) finally: self.lock.release() return self.cached_series else: # use_cache == False slices = [] self.update_slices(slices, from_time, to_time, context) return self.get_series(slices)
def render(self, data={}, context={}): try: assert self.username is not None, "'MetofficeWOW.siteid' must be set" assert self.password is not None, "'MetofficeWOW.siteAuthenticationKey' must be set" assert self.period is not None, "'MetofficeWOW.period' must be set" self.logger.info("Initializing MetOffice WOW Upload (user %s)" % self.username) self.alive = True accu = AccumulatorDatasource() accu.slice = 'hour' accu.span = 2 accu.storage = self.storage accu.formulas = { 'current': { 'temp': LastFormula('temp'), 'hum': LastFormula('hum'), 'pressure': LastFormula('pressure'), 'wind': LastFormula('wind'), 'wind_deg': LastFormula('wind_dir'), 'rain': SumFormula('rain'), 'utctime': LastFormula('utctime') } } while self.alive: try: data = accu.execute()['current']['series'] index = len(data['lbl']) - 1 args = { 'dateutc': data['utctime'][index].strftime('%Y-%m-%d %H:%M:%S'), # Some ARGs are hashed out here as Metoffice WOW needs them in the correct order or it will reject the post # I found that if I used the args they are in a random order so define them on the URL encode instead #'siteAuthenticationKey': str(self.password), #'softwaretype': "Wfrog", 'humidity': int(round(data['hum'][index])), 'tempf': str(CToF(data['temp'][index])), #'siteid': str(self.username), 'winddir': int(round(data['wind_deg'][index])), 'windspeedmph': str(MpsToMph(data['wind'][index])), 'baromin': str(HPaToInHg(data['pressure'][index])), 'rainin': str(MmToIn(data['rain'][index])) } self.logger.info("Publishing Metoffice WOW data: %s " % urlencode(args)) self._publish(args, 'wow.metoffice.gov.uk', '/automaticreading') except Exception, e: if (str(e) == "'NoneType' object has no attribute 'strftime'" ) or (str(e) == "a float is required"): self.logger.error( 'Could not publish: no valid values at this time. Retry next run...' ) else: self.logger.error( 'Got unexpected error. Retry next run. Error: %s' % e) time.sleep(self.period) except Exception, e: self.logger.exception(e) raise
def render(self, data={}, context={}): try: assert self.id is not None, "'meteoclimatic.id' must be set" assert self.storage is not None, "'meteoclimatic.storage' must be set" if self.accuD == None: self.logger.info("Initializing accumulators") # Accumulator for yearly data self.accuY = AccumulatorDatasource() self.accuY.slice = 'year' self.accuY.span = 1 self.accuY.caching = True self.accuY.storage = self.storage self.accuY.formulas = { 'data': { 'max_temp': MaxFormula('temp'), 'min_temp': MinFormula('temp'), 'max_hum': MaxFormula('hum'), 'min_hum': MinFormula('hum'), 'max_pressure': MaxFormula('pressure'), 'min_pressure': MinFormula('pressure'), 'max_gust': MaxFormula('wind_gust'), 'rain_fall': SumFormula('rain') } } # Accumulator for monthly data self.accuM = AccumulatorDatasource() self.accuM.slice = 'month' self.accuM.span = 1 self.accuM.storage = self.storage self.accuM.caching = True self.accuM.formulas = { 'data': { 'max_temp': MaxFormula('temp'), 'min_temp': MinFormula('temp'), 'max_hum': MaxFormula('hum'), 'min_hum': MinFormula('hum'), 'max_pressure': MaxFormula('pressure'), 'min_pressure': MinFormula('pressure'), 'max_gust': MaxFormula('wind_gust'), 'rain_fall': SumFormula('rain') } } # Accumulator for daily and current data self.accuD = AccumulatorDatasource() self.accuD.slice = 'day' self.accuD.span = 1 self.accuD.storage = self.storage self.accuD.caching = True self.accuD.formulas = { 'data': { 'max_temp': MaxFormula('temp'), 'min_temp': MinFormula('temp'), 'max_hum': MaxFormula('hum'), 'min_hum': MinFormula('hum'), 'max_pressure': MaxFormula('pressure'), 'min_pressure': MinFormula('pressure'), 'max_gust': MaxFormula('wind_gust'), 'rain_fall': SumFormula('rain') }, 'current': { 'temp': LastFormula('temp'), 'hum': LastFormula('hum'), 'pressure': LastFormula('pressure'), 'gust': LastFormula('wind_gust'), 'wind_deg': LastFormula('wind_dir'), 'time': LastFormula('localtime') } } if 'solar_rad' in self.storage.keys(): self.accuD.formulas['current']['solar_rad'] = LastFormula( 'solar_rad') self.logger.info("Calculating ...") template = "*VER=DATA2*COD=%s*%s*%s*%s*%s*EOT*" % ( self.id, self._calculateCurrentData( self.accuD), self._calculateAggregData('D', self.accuD), self._calculateAggregData('M', self.accuM), self._calculateAggregData('Y', self.accuY)) self.lastTemplate = template self.logger.info("Template calculated: %s" % template) return ['text/plain', template] except Exception, e: self.logger.warning("Error rendering meteoclimatic data: %s" % str(e)) if self.lastTemplate == None: return ['text/plain', "*VER=DATA2*COD=%s*EOT*" % self.id] else: return ['text/plain', self.lastTemplate]
def render(self, data={}, context={}): try: import Image import ImageDraw import ImageColor assert self.storage is not None, "'sticker.storage' must be set" # Initialize accumulators if self.accuD == None: self.logger.info("Initializing accumulators") # Accumulator for yearly data self.accuY = AccumulatorDatasource() self.accuY.slice = 'year' self.accuY.span = 1 self.accuY.caching = True self.accuY.storage = self.storage self.accuY.formulas = { 'data': { 'max_temp': MaxFormula('temp'), 'min_temp': MinFormula('temp'), 'max_gust': MaxFormula('wind_gust'), 'rain_fall': SumFormula('rain') } } # Accumulator for monthly data self.accuM = AccumulatorDatasource() self.accuM.slice = 'month' self.accuM.span = 1 self.accuM.storage = self.storage self.accuM.caching = True self.accuM.formulas = { 'data': { 'max_temp': MaxFormula('temp'), 'min_temp': MinFormula('temp'), 'max_gust': MaxFormula('wind_gust'), 'rain_fall': SumFormula('rain') } } # Accumulator for daily and current data self.accuD = AccumulatorDatasource() self.accuD.slice = 'day' self.accuD.span = 1 self.accuD.storage = self.storage self.accuD.caching = True self.accuD.formulas = { 'data': { 'max_temp': MaxFormula('temp'), 'min_temp': MinFormula('temp'), 'max_gust': MaxFormula('wind_gust'), 'rain_fall': SumFormula('rain') }, 'current': { 'temp': LastFormula('temp'), 'hum': LastFormula('hum'), 'pressure': LastFormula('pressure'), 'gust': LastFormula('wind_gust'), 'wind_deg': LastFormula('wind_dir'), 'time': LastFormula('localtime') } } # Calculate data self.logger.info("Calculating ...") current = self._calculateCurrentData(self.accuD) # Create Sticker green = ImageColor.getrgb("#007000") wheat = ImageColor.getrgb("#F5DEB3") dark_wheat = ImageColor.getrgb("#8D7641") black = ImageColor.getrgb("#000000") width = 260 height = 100 corner = 10 im = Image.new('RGBA', (width, height), wheat) draw = ImageDraw.Draw(im) # 1) Transparency mask = Image.new('L', im.size, color=0) mdraw = ImageDraw.Draw(mask) mdraw.rectangle((corner, 0, width - corner, height), fill=255) mdraw.rectangle((0, corner, width, height - corner), fill=255) mdraw.chord((0, 0, corner * 2, corner * 2), 0, 360, fill=255) mdraw.chord((0, height - corner * 2 - 1, corner * 2, height - 1), 0, 360, fill=255) mdraw.chord((width - corner * 2 - 1, 0, width - 1, corner * 2), 0, 360, fill=255) mdraw.chord((width - corner * 2 - 1, height - corner * 2 - 1, width - 1, height - 1), 0, 360, fill=255) im.putalpha(mask) # 2) Borders draw.arc((0, 0, corner * 2, corner * 2), 180, 270, fill=dark_wheat) draw.arc((0, height - corner * 2 - 1, corner * 2, height - 1), 90, 180, fill=dark_wheat) draw.arc((width - corner * 2 - 1, 0, width, corner * 2), 270, 360, fill=dark_wheat) draw.arc((width - corner * 2 - 1, height - corner * 2 - 1, width - 1, height - 1), 0, 90, fill=dark_wheat) draw.line((corner, 0, width - corner - 1, 0), fill=dark_wheat) draw.line((corner, height - 1, width - corner - 1, height - 1), fill=dark_wheat) draw.line((0, corner, 0, height - corner - 1), fill=dark_wheat) draw.line((width - 1, corner, width - 1, height - corner - 1), fill=dark_wheat) # 3) Logo logo = Image.open(self.logo_file) im.paste(logo, (4, 3), logo) # using the same image with transparencies as mask # 4) Current data draw.text((65, 5), self.station_name, fill=green) draw.text((65, 25), "%0.1fC %d%% %0.1fKm/h %dmb" % current[0], fill=black) draw.text((65, 38), current[1], fill=dark_wheat) draw.text((6, 60), " Today: %4.1fC-%4.1fC %4.1fKm/h %5.1fl." % self._calculateAggregData(self.accuD), fill=dark_wheat) draw.text((6, 72), " Monthly: %4.1fC-%4.1fC %4.1fKm/h %5.1fl." % self._calculateAggregData(self.accuM), fill=dark_wheat) draw.text((6, 84), " Yearly: %4.1fC-%4.1fC %4.1fKm/h %5.1fl." % self._calculateAggregData(self.accuY), fill=dark_wheat) # Save sticker im.save(self.filename) self.logger.info("Sticker generated") f = open(self.filename, "rb") d = f.read() f.close() return ['image/png', d] except Exception, e: self.logger.warning("Error rendering sticker: %s" % str(e)) return None