def parse_message(self): """ Workers. Consume messages from inq and produce parameters on outq """ for line in iter(self.inq.get, "STOP"): if line == '': continue try: dt, user_agent, path, query_params = line.split('|') params = get_params(dt,query_params,user_agent=user_agent ) self.outq.put(params) except: traceback.print_exc(file=sys.stdout)
def transform(line): count, query_params, minute, spider = line.strip().split('\t') params = get_params(minute + ":00", query_params) params['count'] = count #here the data is aggregated already params['spider'] = (spider == 'true') return params
def transform(line): count, query_params, minute, spider = line.strip().split('\t') params = get_params(minute+":00", query_params) params['count'] = count #here the data is aggregated already params['spider'] = (spider == 'true') return params