def parse_message(self):
"""
Workers. Consume messages from inq and produce parameters on outq
"""
for line in iter(self.inq.get, "STOP"):
if line == '':
continue
try:
dt, user_agent, path, query_params = line.split('|')
params = get_params(dt,query_params,user_agent=user_agent )
self.outq.put(params)
except:
traceback.print_exc(file=sys.stdout)
def transform(line):
count, query_params, minute, spider = line.strip().split('\t')
params = get_params(minute + ":00", query_params)
params['count'] = count #here the data is aggregated already
params['spider'] = (spider == 'true')
return params
def transform(line):
count, query_params, minute, spider = line.strip().split('\t')
params = get_params(minute+":00", query_params)
params['count'] = count #here the data is aggregated already
params['spider'] = (spider == 'true')
return params