def main():
counter = 0
timer = statsd.timer('lisa.process_ip', rate=0.01) # 1% sample rate
while True:
if KILLED:
log.info('Shutdown successful')
return 0
try:
ip_info = redis.brpop(rkeys.IPLOGS)
except RedisError as e:
log.error('Error with Redis: {}'.format(e))
return 1
# don't start above redis call as it will block to wait
timer.start()
log.debug('Got log data: ' + ip_info[1])
try:
rtype, ip = ip_info[1].split(',')
except ValueError:
continue
timestamp = get_epoch_minute()
if rate_limit_ip(ip, timestamp):
continue
record = geo.get(ip)
if record:
# everything goes for total count and map
process_map(record, timestamp)
# only shares get more processing
if rtype != data_types.DOWNLOAD:
process_share(record, rtype)
timer.stop()
statsd.incr('lisa.process_ip', rate=0.01) # 1% sample rate
if args.verbose:
sys.stdout.write('.')
sys.stdout.flush()
# using a counter and if statement here instead of the
# `rate` param on the gauge to avoid getting the length
# of the Redis list every time.
counter += 1
if counter >= 1000:
counter = 0
statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))
def process_map(geo_data):
"""Add download aggregate data to redis."""
redis.incr(rkeys.MAP_TOTAL)
try:
# rounding to aid in geo aggregation
location = {
'lat': round_map_coord(geo_data['location']['latitude']),
'lon': round_map_coord(geo_data['location']['longitude']),
}
except (KeyError, TypeError):
# this appears to mostly happen with anonymous proxies
log.info('Geo data contained no location.')
log.debug(geo_data)
return
geo_key = '{lat}:{lon}'.format(**location)
log.debug('Got location: ' + geo_key)
unix_min = get_epoch_minute()
time_key = rkeys.MAP_GEO.format(unix_min)
log.debug('Got timestamp: %s' % unix_min)
redis.hincrby(time_key, geo_key, 1)
# store the timestamp used in a sorted set for use in milhouse
redis.zadd(rkeys.MAP_TIMESTAMPS, unix_min, unix_min)
def get_context_data(self, **kwargs):
if redis:
timestamp = int(redis.get(rkeys.LATEST_TIMESTAMP) or 0)
else:
timestamp = get_epoch_minute() - 600 # 10 min ago
context = super(GlowView, self).get_context_data(**kwargs)
context.update({
'data_timestamp': timestamp,
'share_map_twitter': get_tw_share_url(
url='http://mzl.la/1g5k6OK',
text=_('Join millions of Firefox users around the world '
'who are shaping the future of the Web:'),
hashtags='firefox',
),
'share_stats_twitter': get_tw_share_url(
url='http://mzl.la/1n0x8lA',
text=_('Join millions of Firefox users around the world '
'who are shaping the future of the Web:'),
hashtags='firefox',
),
'share_video_twitter': get_tw_share_url(
url='http://mzl.la/1rtoRsE',
text=_('Watch the next generation of Internet users talk '
'about the Web they want!'),
hashtags='firefox',
),
'share_stats_twitter_privacy': get_tw_share_url(
url='http://mzl.la/1i6jEol',
text='#Firefox {}'
.format(_('fights for government surveillance reform '
'and was the only major browser not targeted by the NSA scandal.')),
),
'share_stats_twitter_opportunity': get_tw_share_url(
url='http://mzl.la/1nEMbBx',
text='#Firefox {}'
.format(_('is made by a global volunteer community 6,000 strong, '
'open to participation from anyone.')),
),
'share_stats_twitter_access': get_tw_share_url(
url='http://mzl.la/1k45Dae',
text='#Firefox {}'
.format(_('disrupted the mobile industry with the first Web-based OS to '
'help bring the next billion people online')),
),
'share_stats_twitter_freedom': get_tw_share_url(
url='http://mzl.la/1rfuE59',
text='#Firefox {}'
.format(_('protested against SOPA and PIPA, dangerous copyright legislation '
'that threatened the freedom of the Web.')),
),
'share_stats_twitter_learning': get_tw_share_url(
url='http://mzl.la/1ty5Ffu',
text='#Firefox {}'
.format(_('teaches digital skills to millions of people to help them move '
'from using the Web to actively making it.')),
),
'share_stats_twitter_control': get_tw_share_url(
url='http://mzl.la/Qvn21j',
text='#Firefox {}'
.format(_('leads the way in giving users greater control online. '
'They pioneered features like Do Not Track and Lightbeam.')),
),
'share_map_facebook': get_fb_share_url('http://mzl.la/1oKbBCb'),
'share_video_facebook': get_fb_share_url('http://mzl.la/1lNAxrc'),
'share_stats_facebook': get_fb_share_url('http://mzl.la/1sxET6z'),
'share_stats_facebook_privacy': get_fb_share_url('http://mzl.la/1kXIpU8'),
'share_stats_facebook_opportunity': get_fb_share_url('http://mzl.la/1jEWeE7'),
'share_stats_facebook_access': get_fb_share_url('http://mzl.la/1ty5r7W'),
'share_stats_facebook_freedom': get_fb_share_url('http://mzl.la/1icscuJ'),
'share_stats_facebook_learning': get_fb_share_url('http://mzl.la/1rkanJU'),
'share_stats_facebook_control': get_fb_share_url('http://mzl.la/1k45KCq'),
'count_footnote': COUNT_FOOTNOTE.format(_('What does this number mean?')),
'countries_list': self.get_countries_list(),
})
return context
def get_context_data(self, **kwargs):
if redis:
timestamp = int(redis.get(rkeys.LATEST_TIMESTAMP) or 0)
else:
timestamp = get_epoch_minute() - 600 # 10 min ago
context = super(GlowView, self).get_context_data(**kwargs)
context.update({
'data_timestamp':
timestamp,
'share_map_twitter':
get_tw_share_url(
url='http://mzl.la/1g5k6OK',
text=_('Join millions of Firefox users around the world '
'who are shaping the future of the Web:'),
hashtags='firefox',
),
'share_stats_twitter':
get_tw_share_url(
url='http://mzl.la/1n0x8lA',
text=_('Join millions of Firefox users around the world '
'who are shaping the future of the Web:'),
hashtags='firefox',
),
'share_video_twitter':
get_tw_share_url(
url='http://mzl.la/1rtoRsE',
text=_('Watch the next generation of Internet users talk '
'about the Web they want!'),
hashtags='firefox',
),
'share_stats_twitter_privacy':
get_tw_share_url(
url='http://mzl.la/1i6jEol',
text='#Firefox {}'.format(
_('fights for government surveillance reform '
'and was the only major browser not targeted by the NSA scandal.'
)),
),
'share_stats_twitter_opportunity':
get_tw_share_url(
url='http://mzl.la/1nEMbBx',
text='#Firefox {}'.format(
_('is made by a global volunteer community 6,000 strong, '
'open to participation from anyone.')),
),
'share_stats_twitter_access':
get_tw_share_url(
url='http://mzl.la/1k45Dae',
text='#Firefox {}'.format(
_('disrupted the mobile industry with the first Web-based OS to '
'help bring the next billion people online')),
),
'share_stats_twitter_freedom':
get_tw_share_url(
url='http://mzl.la/1rfuE59',
text='#Firefox {}'.format(
_('protested against SOPA and PIPA, dangerous copyright legislation '
'that threatened the freedom of the Web.')),
),
'share_stats_twitter_learning':
get_tw_share_url(
url='http://mzl.la/1ty5Ffu',
text='#Firefox {}'.format(
_('teaches digital skills to millions of people to help them move '
'from using the Web to actively making it.')),
),
'share_stats_twitter_control':
get_tw_share_url(
url='http://mzl.la/Qvn21j',
text='#Firefox {}'.format(
_('leads the way in giving users greater control online. '
'They pioneered features like Do Not Track and Lightbeam.'
)),
),
'share_map_facebook':
get_fb_share_url('http://mzl.la/1oKbBCb'),
'share_video_facebook':
get_fb_share_url('http://mzl.la/1lNAxrc'),
'share_stats_facebook':
get_fb_share_url('http://mzl.la/1sxET6z'),
'share_stats_facebook_privacy':
get_fb_share_url('http://mzl.la/1kXIpU8'),
'share_stats_facebook_opportunity':
get_fb_share_url('http://mzl.la/1jEWeE7'),
'share_stats_facebook_access':
get_fb_share_url('http://mzl.la/1ty5r7W'),
'share_stats_facebook_freedom':
get_fb_share_url('http://mzl.la/1icscuJ'),
'share_stats_facebook_learning':
get_fb_share_url('http://mzl.la/1rkanJU'),
'share_stats_facebook_control':
get_fb_share_url('http://mzl.la/1k45KCq'),
'count_footnote':
COUNT_FOOTNOTE.format(_('What does this number mean?')),
'countries_list':
self.get_countries_list(),
'is_long_headline':
self.is_long_headline(),
})
return context
def main():
global counter
timer = statsd.timer('lisa.process_ip', rate=0.01) # 1% sample rate
pipe = redis.pipeline()
while True:
if KILLED:
pipe.execute()
log.info('Shutdown successful')
return 0
try:
if args.benchmark:
ip_info = redis.rpop(rkeys.IPLOGS)
else:
ip_info = redis.brpop(rkeys.IPLOGS)[1]
except RedisError as e:
log.error('Error with Redis: {}'.format(e))
pipe.execute()
return 1
if ip_info is None:
# benchmark run is over
pipe.execute()
return 0
# don't start above redis call as it will block to wait
timer.start()
log.debug('Got log data: ' + ip_info)
try:
rtype, ip = ip_info.split(',')
except ValueError:
continue
timestamp = get_epoch_minute()
if rate_limit_ip(ip):
continue
record = geo.get(ip)
if record:
# everything goes for total count and map
process_map(record, timestamp, pipe)
# only shares get more processing
if rtype != data_types.DOWNLOAD:
process_share(record, rtype, pipe)
timer.stop()
statsd.incr('lisa.process_ip', rate=0.01) # 1% sample rate
if args.verbose:
sys.stdout.write('.')
sys.stdout.flush()
# using a counter and if statement here instead of the
# `rate` param on the gauge to avoid getting the length
# of the Redis list every time.
counter += 1
if args.benchmark:
if not counter % 1000:
pipe.execute()
else:
if counter >= 1000:
pipe.execute()
counter = 0
statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))