def main():
counter = 0
timer = statsd.timer('lisa.process_ip', rate=0.01) # 1% sample rate
while True:
if KILLED:
log.info('Shutdown successful')
return 0
try:
ip_info = redis.brpop(rkeys.IPLOGS)
except RedisError as e:
log.error('Error with Redis: {}'.format(e))
return 1
# don't start above redis call as it will block to wait
timer.start()
log.debug('Got log data: ' + ip_info[1])
try:
rtype, ip = ip_info[1].split(',')
except ValueError:
continue
timestamp = get_epoch_minute()
if rate_limit_ip(ip, timestamp):
continue
record = geo.get(ip)
if record:
# everything goes for total count and map
process_map(record, timestamp)
# only shares get more processing
if rtype != data_types.DOWNLOAD:
process_share(record, rtype)
timer.stop()
statsd.incr('lisa.process_ip', rate=0.01) # 1% sample rate
if args.verbose:
sys.stdout.write('.')
sys.stdout.flush()
# using a counter and if statement here instead of the
# `rate` param on the gauge to avoid getting the length
# of the Redis list every time.
counter += 1
if counter >= 1000:
counter = 0
statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))
def main():
counter = 0
while True:
if KILLED:
log.info('Shutdown successful')
return 0
try:
ip_info = redis.brpop(rkeys.IPLOGS)
except RedisError as e:
log.error('Error with Redis: {}'.format(e))
return 1
log.debug('Got log data: ' + ip_info[1])
try:
rtype, ip = ip_info[1].split(',')
except ValueError:
continue
record = geo.get(ip)
if record:
# everything goes for total count and map
process_map(record)
# only shares get more processing
if rtype != data_types.DOWNLOAD:
process_share(record, rtype)
if args.verbose:
sys.stdout.write('.')
sys.stdout.flush()
if statsd:
counter += 1
if counter >= 1000:
counter = 0
statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))
def main():
global counter
timer = statsd.timer('lisa.process_ip', rate=0.01) # 1% sample rate
pipe = redis.pipeline()
while True:
if KILLED:
pipe.execute()
log.info('Shutdown successful')
return 0
try:
if args.benchmark:
ip_info = redis.rpop(rkeys.IPLOGS)
else:
ip_info = redis.brpop(rkeys.IPLOGS)[1]
except RedisError as e:
log.error('Error with Redis: {}'.format(e))
pipe.execute()
return 1
if ip_info is None:
# benchmark run is over
pipe.execute()
return 0
# don't start above redis call as it will block to wait
timer.start()
log.debug('Got log data: ' + ip_info)
try:
rtype, ip = ip_info.split(',')
except ValueError:
continue
timestamp = get_epoch_minute()
if rate_limit_ip(ip):
continue
record = geo.get(ip)
if record:
# everything goes for total count and map
process_map(record, timestamp, pipe)
# only shares get more processing
if rtype != data_types.DOWNLOAD:
process_share(record, rtype, pipe)
timer.stop()
statsd.incr('lisa.process_ip', rate=0.01) # 1% sample rate
if args.verbose:
sys.stdout.write('.')
sys.stdout.flush()
# using a counter and if statement here instead of the
# `rate` param on the gauge to avoid getting the length
# of the Redis list every time.
counter += 1
if args.benchmark:
if not counter % 1000:
pipe.execute()
else:
if counter >= 1000:
pipe.execute()
counter = 0
statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))
pipe.execute()
counter = 0
statsd.gauge('queue.geoip', redis.llen(rkeys.IPLOGS))
if __name__ == '__main__':
register_signals(handle_signals)
try:
geo = maxminddb.Reader(args.file)
except IOError:
log.error('ERROR: Can\'t find MaxMind Database file (%s). '
'Try setting the --file flag.' % args.file)
sys.exit(1)
if args.benchmark:
bench_count = redis.llen(rkeys.IPLOGS)
if not bench_count:
print 'No IPs to process'
sys.exit(1)
print 'Starting benchmark of {} records'.format(bench_count)
bench_start = time.time()
return_code = main()
if args.benchmark:
bench_time = time.time() - bench_start
print 'Total Processed: {}'.format(counter)
print 'Total Time: {}s'.format(bench_time)
print 'IPs per minute: {}'.format(counter / (bench_time / 60))
sys.exit(return_code)
