def main(): counter = 0 timer = statsd.timer('lisa.process_ip', rate=0.01) # 1% sample rate while True: if KILLED: log.info('Shutdown successful') return 0 try: ip_info = redis.brpop(rkeys.IPLOGS) except RedisError as e: log.error('Error with Redis: {}'.format(e)) return 1 # don't start above redis call as it will block to wait timer.start() log.debug('Got log data: ' + ip_info[1]) try: rtype, ip = ip_info[1].split(',') except ValueError: continue timestamp = get_epoch_minute() if rate_limit_ip(ip, timestamp): continue record = geo.get(ip) if record: # everything goes for total count and map process_map(record, timestamp) # only shares get more processing if rtype != data_types.DOWNLOAD: process_share(record, rtype) timer.stop() statsd.incr('lisa.process_ip', rate=0.01) # 1% sample rate if args.verbose: sys.stdout.write('.') sys.stdout.flush() # using a counter and if statement here instead of the # `rate` param on the gauge to avoid getting the length # of the Redis list every time. counter += 1 if counter >= 1000: counter = 0 statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))
def process_map(geo_data): """Add download aggregate data to redis.""" redis.incr(rkeys.MAP_TOTAL) try: # rounding to aid in geo aggregation location = { 'lat': round_map_coord(geo_data['location']['latitude']), 'lon': round_map_coord(geo_data['location']['longitude']), } except (KeyError, TypeError): # this appears to mostly happen with anonymous proxies log.info('Geo data contained no location.') log.debug(geo_data) return geo_key = '{lat}:{lon}'.format(**location) log.debug('Got location: ' + geo_key) unix_min = get_epoch_minute() time_key = rkeys.MAP_GEO.format(unix_min) log.debug('Got timestamp: %s' % unix_min) redis.hincrby(time_key, geo_key, 1) # store the timestamp used in a sorted set for use in milhouse redis.zadd(rkeys.MAP_TIMESTAMPS, unix_min, unix_min)
def get_context_data(self, **kwargs): if redis: timestamp = int(redis.get(rkeys.LATEST_TIMESTAMP) or 0) else: timestamp = get_epoch_minute() - 600 # 10 min ago context = super(GlowView, self).get_context_data(**kwargs) context.update({ 'data_timestamp': timestamp, 'share_map_twitter': get_tw_share_url( url='http://mzl.la/1g5k6OK', text=_('Join millions of Firefox users around the world ' 'who are shaping the future of the Web:'), hashtags='firefox', ), 'share_stats_twitter': get_tw_share_url( url='http://mzl.la/1n0x8lA', text=_('Join millions of Firefox users around the world ' 'who are shaping the future of the Web:'), hashtags='firefox', ), 'share_video_twitter': get_tw_share_url( url='http://mzl.la/1rtoRsE', text=_('Watch the next generation of Internet users talk ' 'about the Web they want!'), hashtags='firefox', ), 'share_stats_twitter_privacy': get_tw_share_url( url='http://mzl.la/1i6jEol', text='#Firefox {}' .format(_('fights for government surveillance reform ' 'and was the only major browser not targeted by the NSA scandal.')), ), 'share_stats_twitter_opportunity': get_tw_share_url( url='http://mzl.la/1nEMbBx', text='#Firefox {}' .format(_('is made by a global volunteer community 6,000 strong, ' 'open to participation from anyone.')), ), 'share_stats_twitter_access': get_tw_share_url( url='http://mzl.la/1k45Dae', text='#Firefox {}' .format(_('disrupted the mobile industry with the first Web-based OS to ' 'help bring the next billion people online')), ), 'share_stats_twitter_freedom': get_tw_share_url( url='http://mzl.la/1rfuE59', text='#Firefox {}' .format(_('protested against SOPA and PIPA, dangerous copyright legislation ' 'that threatened the freedom of the Web.')), ), 'share_stats_twitter_learning': get_tw_share_url( url='http://mzl.la/1ty5Ffu', text='#Firefox {}' .format(_('teaches digital skills to millions of people to help them move ' 'from using the Web to actively making it.')), ), 'share_stats_twitter_control': get_tw_share_url( url='http://mzl.la/Qvn21j', text='#Firefox {}' .format(_('leads the way in giving users greater control online. ' 'They pioneered features like Do Not Track and Lightbeam.')), ), 'share_map_facebook': get_fb_share_url('http://mzl.la/1oKbBCb'), 'share_video_facebook': get_fb_share_url('http://mzl.la/1lNAxrc'), 'share_stats_facebook': get_fb_share_url('http://mzl.la/1sxET6z'), 'share_stats_facebook_privacy': get_fb_share_url('http://mzl.la/1kXIpU8'), 'share_stats_facebook_opportunity': get_fb_share_url('http://mzl.la/1jEWeE7'), 'share_stats_facebook_access': get_fb_share_url('http://mzl.la/1ty5r7W'), 'share_stats_facebook_freedom': get_fb_share_url('http://mzl.la/1icscuJ'), 'share_stats_facebook_learning': get_fb_share_url('http://mzl.la/1rkanJU'), 'share_stats_facebook_control': get_fb_share_url('http://mzl.la/1k45KCq'), 'count_footnote': COUNT_FOOTNOTE.format(_('What does this number mean?')), 'countries_list': self.get_countries_list(), }) return context
def get_context_data(self, **kwargs): if redis: timestamp = int(redis.get(rkeys.LATEST_TIMESTAMP) or 0) else: timestamp = get_epoch_minute() - 600 # 10 min ago context = super(GlowView, self).get_context_data(**kwargs) context.update({ 'data_timestamp': timestamp, 'share_map_twitter': get_tw_share_url( url='http://mzl.la/1g5k6OK', text=_('Join millions of Firefox users around the world ' 'who are shaping the future of the Web:'), hashtags='firefox', ), 'share_stats_twitter': get_tw_share_url( url='http://mzl.la/1n0x8lA', text=_('Join millions of Firefox users around the world ' 'who are shaping the future of the Web:'), hashtags='firefox', ), 'share_video_twitter': get_tw_share_url( url='http://mzl.la/1rtoRsE', text=_('Watch the next generation of Internet users talk ' 'about the Web they want!'), hashtags='firefox', ), 'share_stats_twitter_privacy': get_tw_share_url( url='http://mzl.la/1i6jEol', text='#Firefox {}'.format( _('fights for government surveillance reform ' 'and was the only major browser not targeted by the NSA scandal.' )), ), 'share_stats_twitter_opportunity': get_tw_share_url( url='http://mzl.la/1nEMbBx', text='#Firefox {}'.format( _('is made by a global volunteer community 6,000 strong, ' 'open to participation from anyone.')), ), 'share_stats_twitter_access': get_tw_share_url( url='http://mzl.la/1k45Dae', text='#Firefox {}'.format( _('disrupted the mobile industry with the first Web-based OS to ' 'help bring the next billion people online')), ), 'share_stats_twitter_freedom': get_tw_share_url( url='http://mzl.la/1rfuE59', text='#Firefox {}'.format( _('protested against SOPA and PIPA, dangerous copyright legislation ' 'that threatened the freedom of the Web.')), ), 'share_stats_twitter_learning': get_tw_share_url( url='http://mzl.la/1ty5Ffu', text='#Firefox {}'.format( _('teaches digital skills to millions of people to help them move ' 'from using the Web to actively making it.')), ), 'share_stats_twitter_control': get_tw_share_url( url='http://mzl.la/Qvn21j', text='#Firefox {}'.format( _('leads the way in giving users greater control online. ' 'They pioneered features like Do Not Track and Lightbeam.' )), ), 'share_map_facebook': get_fb_share_url('http://mzl.la/1oKbBCb'), 'share_video_facebook': get_fb_share_url('http://mzl.la/1lNAxrc'), 'share_stats_facebook': get_fb_share_url('http://mzl.la/1sxET6z'), 'share_stats_facebook_privacy': get_fb_share_url('http://mzl.la/1kXIpU8'), 'share_stats_facebook_opportunity': get_fb_share_url('http://mzl.la/1jEWeE7'), 'share_stats_facebook_access': get_fb_share_url('http://mzl.la/1ty5r7W'), 'share_stats_facebook_freedom': get_fb_share_url('http://mzl.la/1icscuJ'), 'share_stats_facebook_learning': get_fb_share_url('http://mzl.la/1rkanJU'), 'share_stats_facebook_control': get_fb_share_url('http://mzl.la/1k45KCq'), 'count_footnote': COUNT_FOOTNOTE.format(_('What does this number mean?')), 'countries_list': self.get_countries_list(), 'is_long_headline': self.is_long_headline(), }) return context
def main(): global counter timer = statsd.timer('lisa.process_ip', rate=0.01) # 1% sample rate pipe = redis.pipeline() while True: if KILLED: pipe.execute() log.info('Shutdown successful') return 0 try: if args.benchmark: ip_info = redis.rpop(rkeys.IPLOGS) else: ip_info = redis.brpop(rkeys.IPLOGS)[1] except RedisError as e: log.error('Error with Redis: {}'.format(e)) pipe.execute() return 1 if ip_info is None: # benchmark run is over pipe.execute() return 0 # don't start above redis call as it will block to wait timer.start() log.debug('Got log data: ' + ip_info) try: rtype, ip = ip_info.split(',') except ValueError: continue timestamp = get_epoch_minute() if rate_limit_ip(ip): continue record = geo.get(ip) if record: # everything goes for total count and map process_map(record, timestamp, pipe) # only shares get more processing if rtype != data_types.DOWNLOAD: process_share(record, rtype, pipe) timer.stop() statsd.incr('lisa.process_ip', rate=0.01) # 1% sample rate if args.verbose: sys.stdout.write('.') sys.stdout.flush() # using a counter and if statement here instead of the # `rate` param on the gauge to avoid getting the length # of the Redis list every time. counter += 1 if args.benchmark: if not counter % 1000: pipe.execute() else: if counter >= 1000: pipe.execute() counter = 0 statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))